mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2026-10-28 16:18:23 +08:00
Alexey Milovidov
f5c5dd1d4d
`resolve_symlink_chain` returned the resolved path with `echo`, but the installer runs under `/bin/sh`, which is `dash` on Debian-based systems, and `dash`'s `echo` expands backslash escapes: a dangling destination whose target contains `\t` or `\n` was resolved to a mangled path, so the installation either failed or created the file at the wrong place. Every `echo` inside the generated script is now `printf '%s\n'` - the `ls -ld` listing fed to `awk` in `keep_destination_inode` carries the file name too, and the diagnostics should print paths byte-for-byte as well. Added `Install tgz over a dangling symlink with a backslash in the target` to `ci/jobs/install_check.py`. https://github.com/ClickHouse/ClickHouse/pull/112766
287 lines
11 KiB
Python
Executable File
287 lines
11 KiB
Python
Executable File
#!/usr/bin/env bash
|
|
|
|
set -e
|
|
set -o pipefail
|
|
|
|
# Avoid dependency on locale
|
|
LC_ALL=C
|
|
|
|
PKG_ROOT='root'
|
|
|
|
DEB_ARCH=${DEB_ARCH:-amd64}
|
|
SANITIZER=${SANITIZER:-""}
|
|
SOURCE=${SOURCE:-$PKG_ROOT}
|
|
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
HELP="${0} [--test] [--rpm] [-h|--help]
|
|
--test - adds '+test' prefix to version
|
|
--apk - build APK packages
|
|
--archlinux - build archlinux packages
|
|
--rpm - build RPM packages
|
|
--tgz - build tarball package
|
|
--deb - build deb package
|
|
--help - show this help and exit
|
|
|
|
Used envs:
|
|
DEB_ARCH='${DEB_ARCH}'
|
|
OUTPUT_DIR='${OUTPUT_DIR}' - where the artifact will be placed
|
|
SANITIZER='${SANITIZER}' - if any sanitizer is used, affects version string
|
|
SOURCE='${SOURCE}' - directory with sources tree
|
|
VERSION_STRING='${VERSION_STRING}' - the package version to overwrite
|
|
"
|
|
|
|
CLICKHOUSE_VERSION_STRING=${VERSION_STRING}
|
|
export CLICKHOUSE_VERSION_STRING
|
|
|
|
|
|
while [[ $1 == --* ]]
|
|
do
|
|
case "$1" in
|
|
--test )
|
|
VERSION_POSTFIX+='+test'
|
|
shift ;;
|
|
--deb )
|
|
MAKE_DEB=1
|
|
shift ;;
|
|
--apk )
|
|
MAKE_APK=1
|
|
shift ;;
|
|
--archlinux )
|
|
MAKE_ARCHLINUX=1
|
|
shift ;;
|
|
--rpm )
|
|
MAKE_RPM=1
|
|
shift ;;
|
|
--tgz )
|
|
MAKE_TGZ=1
|
|
shift ;;
|
|
--help )
|
|
echo "$HELP"
|
|
exit ;;
|
|
* )
|
|
echo "Unknown option $1"
|
|
exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
function deb2tgz {
|
|
local FILE PKG_NAME PKG_DIR PKG_PATH TARBALL
|
|
FILE=$1
|
|
PKG_NAME=${FILE##*/}; PKG_NAME=${PKG_NAME%%_*}
|
|
PKG_DIR="$PKG_NAME-$CLICKHOUSE_VERSION_STRING"
|
|
PKG_PATH="$OUTPUT_DIR/$PKG_DIR"
|
|
TARBALL="$OUTPUT_DIR/$PKG_DIR-$DEB_ARCH.tgz"
|
|
rm -rf "$PKG_PATH"
|
|
dpkg-deb -R "$FILE" "$PKG_PATH"
|
|
mkdir -p "$PKG_PATH/install"
|
|
cat > "$PKG_PATH/install/doinst.sh" << 'EOF'
|
|
#!/bin/sh
|
|
set -e
|
|
|
|
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
|
|
|
|
# Moving a file replaces the destination inode with the one from the unpack tree, so the
|
|
# installed file inherits the SELinux label of the temporary directory instead of the one
|
|
# the final path should have. Restore the labels from the policy where SELinux is active.
|
|
RESTORECON=""
|
|
if command -v selinuxenabled > /dev/null 2>&1 && selinuxenabled > /dev/null 2>&1 && command -v restorecon > /dev/null 2>&1; then
|
|
RESTORECON="restorecon"
|
|
fi
|
|
|
|
# Extended attributes live on the inode as well, and unlike the SELinux label they cannot be
|
|
# recreated from a policy, so a destination carrying them is written through instead of being
|
|
# renamed over. `getfattr` reports them where it is available.
|
|
GETFATTR=""
|
|
if command -v getfattr > /dev/null 2>&1; then
|
|
GETFATTR="getfattr"
|
|
fi
|
|
|
|
# Whether the file already at the destination has to survive the installation as the same
|
|
# inode. Everything else - and that is the common case, including every fresh installation -
|
|
# is replaced by a rename, which needs no additional disk space.
|
|
keep_destination_inode () {
|
|
# An existing symlink at the destination is written through, the way copying did, so that
|
|
# an installation keeping its configs or binaries elsewhere and linking to them keeps the
|
|
# indirection after an upgrade. A symlink with no target yet is handled separately in the
|
|
# loop below, because there is no inode to write through in that case.
|
|
if [ -L "$1" ] && [ -e "$1" ]; then
|
|
return 0
|
|
fi
|
|
if [ ! -e "$1" ]; then
|
|
return 1
|
|
fi
|
|
# Only a regular file can be written through.
|
|
if [ ! -f "$1" ]; then
|
|
return 1
|
|
fi
|
|
listing=$(ls -ld -- "$1")
|
|
# Another hard link to the destination would keep pointing at the pre-upgrade contents
|
|
# after a rename, while writing through updates every link.
|
|
if [ "$(printf '%s\n' "$listing" | awk '{print $2}')" -gt 1 ]; then
|
|
return 0
|
|
fi
|
|
# An ACL is reported by `ls` as a `+` after the permissions.
|
|
case "$(printf '%s\n' "$listing" | awk '{print $1}')" in
|
|
*+) return 0 ;;
|
|
esac
|
|
if [ -z "$GETFATTR" ]; then
|
|
# Without `getfattr` there is no portable way to tell whether the destination carries
|
|
# extended attributes, so an existing regular file conservatively keeps its inode.
|
|
# The disk-space saving is unaffected: it comes from the fresh installation, where no
|
|
# destination exists and the rename path is taken above.
|
|
return 0
|
|
fi
|
|
# The SELinux label is skipped: it is not carried over by a rename either, but it is
|
|
# recreated from the policy by `restorecon` below, and on an SELinux system every
|
|
# single file has one, which would make the rename path unreachable.
|
|
attributes=$($GETFATTR --absolute-names --match=- --dump -- "$1" 2> /dev/null | grep -v -e '^#' -e '^$' -e '^security\.selinux=' || true)
|
|
if [ -n "$attributes" ]; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# The path a chain of symlinks finally points at. Every hop is followed, and a relative hop is
|
|
# resolved against the directory of the link it was read from, so that a destination pointing
|
|
# at another symlink keeps every level of the indirection instead of only the first one.
|
|
resolve_symlink_chain () {
|
|
path="$1"
|
|
hops=0
|
|
while [ -L "$path" ]; do
|
|
hops=$((hops + 1))
|
|
if [ "$hops" -gt 40 ]; then
|
|
printf '%s\n' "Too many levels of symbolic links at $1" >&2
|
|
return 1
|
|
fi
|
|
target=$(readlink -- "$path")
|
|
case "$target" in
|
|
/*) path="$target" ;;
|
|
*) path="$(dirname -- "$path")/$target" ;;
|
|
esac
|
|
done
|
|
printf '%s\n' "$path"
|
|
}
|
|
|
|
for filepath in `find $SCRIPTPATH/.. -type f -or -type l | grep -v "\.\./install/"`; do
|
|
destpath=${filepath##$SCRIPTPATH/..}
|
|
mkdir -p $(dirname "$destpath")
|
|
if [ ! -L "$filepath" ] && [ -L "$destpath" ] && [ ! -e "$destpath" ]; then
|
|
# A symlink whose target does not exist yet has to survive the installation as well,
|
|
# but there is no destination inode to write through: a shell redirection would create
|
|
# the target with the `umask` of the installation and with no relation to the mode of
|
|
# the packaged file, which would install a binary non-executable. Rename the packaged
|
|
# file to where the link points instead, so that the link keeps its indirection and the
|
|
# installed file gets the mode, the ownership and the label every installed file has.
|
|
# The whole chain is followed: a link pointing at another link has to keep both, the
|
|
# way writing through the destination did, so only the final referent is created.
|
|
linktarget=$(resolve_symlink_chain "$destpath")
|
|
linkparent=$(dirname -- "$linktarget")
|
|
if [ ! -d "$linkparent" ]; then
|
|
# The directory the link points into is not created here. It would be created
|
|
# with the `umask` of the installation and with no defined ownership, so the
|
|
# installed file could end up unreachable while the installation reports
|
|
# success. Writing through the link failed loudly when that directory was
|
|
# missing, and pointing a destination at a directory that does not exist stays
|
|
# an error the administrator has to see and fix.
|
|
printf '%s\n' "Cannot install $destpath: it points at $linktarget, but the directory $linkparent does not exist" >&2
|
|
exit 1
|
|
fi
|
|
mv -f "$filepath" "$linktarget"
|
|
chown -h root:root "$linktarget"
|
|
if [ -n "$RESTORECON" ]; then
|
|
$RESTORECON "$linktarget"
|
|
fi
|
|
elif [ ! -L "$filepath" ] && keep_destination_inode "$destpath"; then
|
|
# The destination keeps its inode, so its ownership, permissions, attributes and every
|
|
# other hard link to it stay valid after the upgrade, as they did when copying.
|
|
cat "$filepath" > "$destpath"
|
|
rm -f "$filepath"
|
|
else
|
|
# Move instead of copy: copying requires twice the disk space, which is several
|
|
# gigabytes for the packages with debug symbols. Moving also succeeds when the
|
|
# destination is a running executable, while copying over it fails with ETXTBSY.
|
|
mv -f "$filepath" "$destpath"
|
|
# The tarball carries the ownership of the user who built it, while the installed
|
|
# files have to belong to root, as they do in the deb and rpm packages.
|
|
chown -h root:root "$destpath"
|
|
if [ -n "$RESTORECON" ]; then
|
|
$RESTORECON "$destpath"
|
|
fi
|
|
fi
|
|
done
|
|
EOF
|
|
chmod +x "$PKG_PATH/install/doinst.sh"
|
|
if [ -f "$PKG_PATH/DEBIAN/postinst" ]; then
|
|
# we don't need debconf source in doinst in any case
|
|
tail +2 "$PKG_PATH/DEBIAN/postinst" | grep -v debconf/confmodule >> "$PKG_PATH/install/doinst.sh"
|
|
fi
|
|
rm -rf "$PKG_PATH/DEBIAN"
|
|
if [ -f "/usr/bin/pigz" ]; then
|
|
tar --use-compress-program=pigz -cf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR"
|
|
else
|
|
tar -czf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR"
|
|
fi
|
|
|
|
# Cut the $OUTPUT_DIR/ from the sha512sum output to make it universal
|
|
sha512sum "$TARBALL" | sed "s|$OUTPUT_DIR/||" > "$TARBALL".sha512
|
|
|
|
rm -r "$PKG_PATH"
|
|
}
|
|
|
|
# Build options
|
|
if [ -n "$SANITIZER" ]; then
|
|
if [[ "$SANITIZER" == "address" ]]; then VERSION_POSTFIX+="+asan"
|
|
elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan"
|
|
elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan"
|
|
elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan"
|
|
else
|
|
echo "Unknown value of SANITIZER variable: $SANITIZER"
|
|
exit 3
|
|
fi
|
|
elif [[ $BUILD_TYPE == 'debug' ]]; then
|
|
VERSION_POSTFIX+="+debug"
|
|
elif [[ $BUILD_TYPE =~ 'coverage' ]]; then
|
|
VERSION_POSTFIX+="+coverage"
|
|
elif [[ $BUILD_TYPE == 'cfi' ]]; then
|
|
VERSION_POSTFIX+="+cfi"
|
|
fi
|
|
|
|
if [[ "$PKG_ROOT" != "$SOURCE" ]]; then
|
|
# packages are built only from PKG_SOURCE
|
|
rm -rf "./$PKG_ROOT"
|
|
ln -sf "$SOURCE" "$PKG_SOURCE"
|
|
fi
|
|
|
|
CLICKHOUSE_VERSION_STRING+=$VERSION_POSTFIX
|
|
echo -e "\nCurrent version is $CLICKHOUSE_VERSION_STRING"
|
|
|
|
for config in clickhouse*.yaml; do
|
|
if [ -n "$MAKE_DEB" ] || [ -n "$MAKE_TGZ" ]; then
|
|
echo "Building deb package for $config"
|
|
PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/stderr | grep "created package:" | sed 's/.*created package: //')
|
|
fi
|
|
|
|
if [ -n "$MAKE_APK" ]; then
|
|
echo "Building apk package for $config"
|
|
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager apk
|
|
fi
|
|
|
|
if [ -n "$MAKE_ARCHLINUX" ]; then
|
|
echo "Building archlinux package for $config"
|
|
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager archlinux
|
|
fi
|
|
|
|
if [ -n "$MAKE_RPM" ]; then
|
|
echo "Building rpm package for $config"
|
|
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager rpm
|
|
fi
|
|
|
|
if [ -n "$MAKE_TGZ" ]; then
|
|
echo "Building tarball for $config"
|
|
deb2tgz "$PKG_PATH"
|
|
fi
|
|
done
|
|
|
|
# vim: ts=4: sw=4: sts=4: expandtab
|