mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2026-10-28 16:18:23 +08:00
Azat KhuzhinandClaude Fable 5.1
821f1007c9
Each server restart resets jemalloc's dump sequence, so sorting by it could render a stale snapshot. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
480 lines
22 KiB
Bash
480 lines
22 KiB
Bash
#!/bin/bash
|
|
|
|
set -ex
|
|
|
|
OK="\tOK\t\\N\t"
|
|
FAIL="\tFAIL\t\\N\t"
|
|
|
|
FAILURE_CONTEXT_LINES=100
|
|
FAILURE_CONTEXT_MAX_LINE_WIDTH=3000
|
|
START_SERVER_READY_TIMEOUT=120
|
|
# Where MALLOC_CONF (set by stress_job.py for jemalloc builds) points prof_prefix.
|
|
JEMALLOC_PROFILES_DIR=/var/log/clickhouse-server/jemalloc_profiles
|
|
|
|
function escaped()
|
|
{
|
|
# That's the simplest way I found to escape a string in bash. Yep, bash is the most convenient programming language.
|
|
# Also limit lines width just in case (too long lines are not really useful usually)
|
|
clickhouse local -S 's String' --input-format=LineAsString -q "select substr(s, 1, $FAILURE_CONTEXT_MAX_LINE_WIDTH)
|
|
from table format CustomSeparated settings format_custom_row_after_delimiter='\\\\\\\\n'"
|
|
}
|
|
|
|
function head_escaped()
|
|
{
|
|
head -n "$FAILURE_CONTEXT_LINES" "$1" | escaped
|
|
}
|
|
|
|
function unts()
|
|
{
|
|
grep -Po "[0-9][0-9]:[0-9][0-9] \K.*"
|
|
}
|
|
|
|
function trim_server_logs()
|
|
{
|
|
head -n "$FAILURE_CONTEXT_LINES" "/test_output/$1" | grep -Eo " \[ [0-9]+ \] \{.*" | escaped
|
|
}
|
|
|
|
function run_with_retry()
|
|
{
|
|
local retries="$1" && shift
|
|
|
|
local retry=0
|
|
until [ "$retry" -ge "$retries" ]; do
|
|
if "$@"; then
|
|
return
|
|
else
|
|
retry=$((retry + 1))
|
|
sleep 5
|
|
fi
|
|
done
|
|
|
|
echo "Command '$*' failed after $retries retries, exiting" >&2
|
|
return 1
|
|
}
|
|
|
|
# Only the upgrade check needs this: it installs a released version, and a release is
|
|
# a set of packages. The stress test itself takes the plain binary - see `install_binary`.
|
|
function install_packages()
|
|
{
|
|
dpkg -i "$1"/clickhouse-common-static_*.deb
|
|
dpkg -i "$1"/clickhouse-common-static-dbg_*.deb
|
|
dpkg -i "$1"/clickhouse-server_*.deb
|
|
dpkg -i "$1"/clickhouse-client_*.deb
|
|
}
|
|
|
|
# Set up the same paths the packages own, but from a single `clickhouse` binary, which is
|
|
# built and uploaded for every build type anyway. Packages are assembled only for the
|
|
# release builds, the ones users install and the ones the `Install packages` check covers.
|
|
#
|
|
# The binary keeps the place it was downloaded to and everything else points at it: the
|
|
# artifact is a self-extracting archive that unpacks over itself, so it needs the room for
|
|
# the unpacked binary once, on the volume it already occupies.
|
|
function install_binary()
|
|
{
|
|
local binary
|
|
binary="$(realpath "$1")"
|
|
|
|
chmod +x "$binary"
|
|
local name
|
|
for name in clickhouse ch chc chl \
|
|
clickhouse-benchmark clickhouse-client clickhouse-compressor \
|
|
clickhouse-disks clickhouse-extract-from-config clickhouse-format clickhouse-keeper \
|
|
clickhouse-keeper-client clickhouse-keeper-converter clickhouse-local \
|
|
clickhouse-obfuscator clickhouse-server clickhouse-chdig chdig
|
|
do
|
|
ln -sf "$binary" "/usr/bin/$name"
|
|
done
|
|
|
|
# The `google/*.proto` files that the packages carry in `/usr/share/clickhouse/protos`
|
|
# come with `protoc` in the image, the same way the functional tests take them.
|
|
mkdir -p /usr/share/clickhouse
|
|
ln -sf /usr/local/include /usr/share/clickhouse/protos
|
|
|
|
# `configure` replaces most of the config tree below, but the server does not start
|
|
# without these two, and they are what the server package installs.
|
|
cp /repo/programs/server/config.xml /repo/programs/server/users.xml /etc/clickhouse-server/
|
|
|
|
# Unpack the self-extracting archive here, before anything else reaches for the binary.
|
|
# Decompressing a sanitizer build takes a while, and left to the first server start it
|
|
# would count against the wait for the pid file, while any `clickhouse local` running
|
|
# meanwhile would race a half-written binary.
|
|
clickhouse-server --version
|
|
}
|
|
|
|
function configure()
|
|
{
|
|
# install test configs
|
|
export USE_DATABASE_ORDINARY=1
|
|
export EXPORT_S3_STORAGE_POLICIES=1
|
|
|
|
# --no-remote-database-disk - stress tests does not enable remote database
|
|
# disk since server left some garbage files in the root, and clickhouse failed
|
|
# to start after with:
|
|
#
|
|
# DB::Exception: Incorrect file extension: V.sql.move_from.X in metadata directory store/346/346b09d6-907d-4b6b-a1e3-361ecfdab8a4/. (INCORRECT_FILE_NAME)
|
|
#
|
|
/repo/tests/config/install.sh /etc/clickhouse-server /etc/clickhouse-client --no-keeper-inject-auth --no-remote-database-disk "$@"
|
|
|
|
# jemalloc does not create the prof_prefix directory itself.
|
|
mkdir -p "$JEMALLOC_PROFILES_DIR"
|
|
|
|
# These jobs mix config drop-ins and binaries of different versions (the upgrade
|
|
# check boots the previous release with the current repo's configs) - an unknown
|
|
# config option is expected here and must not prevent the server from starting.
|
|
echo "<clickhouse><skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings></clickhouse>" \
|
|
> /etc/clickhouse-server/config.d/skip_check_for_incorrect_settings.xml
|
|
|
|
# avoid too slow startup
|
|
sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
|
|
| sed "s|<snapshot_distance>100000</snapshot_distance>|<snapshot_distance>10000</snapshot_distance>|" \
|
|
> /etc/clickhouse-server/config.d/keeper_port.xml.tmp
|
|
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
|
|
|
|
function randomize_config_boolean_value {
|
|
local value=$((RANDOM % 2))
|
|
echo "Replacing $1 with $value"
|
|
sudo cat "/etc/clickhouse-server/config.d/$2.xml" \
|
|
| sed "s|<$1>[01]</$1>|<$1>$value</$1>|" \
|
|
> "/etc/clickhouse-server/config.d/$2.xml.tmp"
|
|
sudo mv "/etc/clickhouse-server/config.d/$2.xml.tmp" "/etc/clickhouse-server/config.d/$2.xml"
|
|
}
|
|
|
|
if [[ -n "$RANDOMIZE_KEEPER_FEATURE_FLAGS" ]] && [[ "$RANDOMIZE_KEEPER_FEATURE_FLAGS" -eq 1 ]]; then
|
|
# transactions.xml is always installed here, and transactions require Keeper to advertise
|
|
# FILTERED_LIST, MULTI_READ, LIST_WITH_STAT_AND_DATA and CHECK_STAT (see TransactionManager).
|
|
randomize_config_boolean_value check_not_exists keeper_port
|
|
randomize_config_boolean_value create_if_not_exists keeper_port
|
|
randomize_config_boolean_value remove_recursive keeper_port
|
|
fi
|
|
|
|
sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
|
|
sudo chgrp clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
|
|
|
|
# Always randomize in zookeeper.xml (the base config)
|
|
randomize_config_boolean_value use_compression zookeeper
|
|
randomize_config_boolean_value use_xid_64 zookeeper
|
|
|
|
# for clickhouse-server (via service)
|
|
echo "ASAN_OPTIONS='malloc_context_size=10 verbosity=1 allocator_release_to_os_interval_ms=10000'" >> /etc/environment
|
|
# for clickhouse-client
|
|
export ASAN_OPTIONS='malloc_context_size=10 allocator_release_to_os_interval_ms=10000'
|
|
|
|
# since we run clickhouse from root
|
|
sudo chown root: /var/lib/clickhouse
|
|
|
|
# Cores must land in the host-mounted /cores (stress_job.py collects only it).
|
|
# Written here, not in configure_limits: install.sh wipes config.d on every
|
|
# configure call, and the upgrade runner never calls configure_limits.
|
|
cat > /etc/clickhouse-server/config.d/core_path.xml <<EOL
|
|
<clickhouse>
|
|
<core_path>/cores</core_path>
|
|
</clickhouse>
|
|
EOL
|
|
}
|
|
|
|
function configure_limits()
|
|
{
|
|
local total_mem
|
|
total_mem=$(awk '/MemTotal/ { print $(NF-1) }' /proc/meminfo) # KiB
|
|
total_mem=$(( total_mem*1024 )) # bytes
|
|
|
|
# NOTE: the total server memory usage is capped to the memory usage of the
|
|
# whole cgroup, so, there is no need to take into account that we run
|
|
# multiple servers, so static 80% will be just fine.
|
|
|
|
max_server_memory_usage_to_ram_ratio=0.8
|
|
echo "Setting max_server_memory_usage_to_ram_ratio to ${max_server_memory_usage_to_ram_ratio}"
|
|
cat > /etc/clickhouse-server/config.d/max_server_memory_usage.xml <<EOL
|
|
<clickhouse>
|
|
<max_server_memory_usage_to_ram_ratio>${max_server_memory_usage_to_ram_ratio}</max_server_memory_usage_to_ram_ratio>
|
|
</clickhouse>
|
|
EOL
|
|
|
|
local max_users_mem
|
|
max_users_mem=$((total_mem*30/100)) # 30%
|
|
# Similar to docker/test/fuzzer/query-fuzzer-tweaks-users.xml
|
|
echo "Setting max_memory_usage_for_user=$max_users_mem and max_memory_usage for queries to 10G"
|
|
# The heredoc delimiter is unquoted so that ${max_users_mem} expands. That also means backticks
|
|
# and $(...) would be evaluated, and its XML comments cannot hold a double hyphen, so the
|
|
# rationale for the `allow_experimental_window_view` constraint below lives here: the upgrade
|
|
# check replays the previous release's test suite under its own binary, which still creates
|
|
# window views, and `clickhouse-test --upgrade-check` passes `--fake-drop`, so they stay in the
|
|
# metadata for the new binary to fail on. Keep it readonly, rather than set to `false`, until
|
|
# that suite stops creating them: assigning an obsolete setting makes the new binary warn.
|
|
cat > /etc/clickhouse-server/users.d/stress_test_tweaks-users.xml <<EOL
|
|
<clickhouse>
|
|
<profiles>
|
|
<default>
|
|
<database_replicated_allow_replicated_engine_arguments>1</database_replicated_allow_replicated_engine_arguments>
|
|
<max_execution_time>60</max_execution_time>
|
|
<max_memory_usage>10G</max_memory_usage>
|
|
<max_memory_usage_for_user>${max_users_mem}</max_memory_usage_for_user>
|
|
|
|
<table_function_remote_max_addresses>200</table_function_remote_max_addresses>
|
|
|
|
<ast_fuzzer_runs>5</ast_fuzzer_runs>
|
|
<ast_fuzzer_any_query>true</ast_fuzzer_any_query>
|
|
|
|
<constraints>
|
|
<max_execution_time>
|
|
<max>60</max>
|
|
</max_execution_time>
|
|
|
|
<max_memory_usage>
|
|
<max>10G</max>
|
|
</max_memory_usage>
|
|
|
|
<table_function_remote_max_addresses>
|
|
<max>200</max>
|
|
</table_function_remote_max_addresses>
|
|
|
|
<!-- Prevent stack overflow -->
|
|
<max_ast_depth>
|
|
<readonly/>
|
|
</max_ast_depth>
|
|
|
|
<allow_experimental_window_view>
|
|
<readonly/>
|
|
</allow_experimental_window_view>
|
|
|
|
</constraints>
|
|
</default>
|
|
</profiles>
|
|
</clickhouse>
|
|
EOL
|
|
|
|
cat > /etc/clickhouse-server/config.d/core.xml <<EOL
|
|
<clickhouse>
|
|
|
|
<remote_servers>
|
|
<test_cluster_database_replicated>
|
|
<shard>
|
|
<replica>
|
|
<host>localhost</host>
|
|
<port>9000</port>
|
|
</replica>
|
|
</shard>
|
|
</test_cluster_database_replicated>
|
|
</remote_servers>
|
|
|
|
</clickhouse>
|
|
EOL
|
|
}
|
|
|
|
function stop_server()
|
|
{
|
|
local max_tries=${1:-300}
|
|
local check_hang=${2:-true}
|
|
local pid
|
|
# Preserve the pid, since the server can hung after the PID will be deleted.
|
|
pid="$(cat /var/run/clickhouse-server/clickhouse-server.pid)"
|
|
|
|
# FIXME: workaround for slow shutdown of Distributed tables (due to
|
|
# sequential tables shutdown), since flush_on_detach=0 will not help if the
|
|
# send is already in progress.
|
|
#
|
|
# Refs: https://github.com/ClickHouse/ClickHouse/issues/72557
|
|
#
|
|
# `--receive_timeout` is inert until Hello completes (the client holds the
|
|
# 300s handshake timeout until then), so `timeout` is what bounds an attempt.
|
|
for i in {1..30}; do
|
|
if timeout -s KILL 15 clickhouse client --handshake_timeout_ms=10000 --receive_timeout=10 -q "SYSTEM STOP DISTRIBUTED SENDS"; then
|
|
break
|
|
fi
|
|
done
|
|
clickhouse stop --max-tries "$max_tries" --do-not-kill && return
|
|
|
|
if [ "$check_hang" == true ]
|
|
then
|
|
# We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces.
|
|
# Add a special status just in case, so it will be possible to find in the CI DB
|
|
echo -e "Warning: server did not stop yet$OK" >> /test_output/test_results.tsv
|
|
kill -TERM "$(pidof gdb)" ||:
|
|
|
|
# We are waiting 60 seconds until the server will be terminated, since
|
|
# - gdb detach can be slow
|
|
# - server may do some work at shutdown
|
|
# - main use case it not to mark failure as hung check when server exits abnormally with sanitizer/crash
|
|
kill -s CONT "$pid" ||:
|
|
kill -s 0 "$pid" && sleep 60
|
|
|
|
# The server could finally stop while we were terminating gdb, let's recheck if it's still running
|
|
kill -s 0 "$pid" || return 0
|
|
echo -e "Possible deadlock on shutdown (see gdb.log)$FAIL" >> /test_output/test_results.tsv
|
|
echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log
|
|
timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$pid" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log
|
|
clickhouse stop --force
|
|
else
|
|
echo -e "Warning: server did not stop yet$OK" >> /test_output/test_results.tsv
|
|
clickhouse stop --force
|
|
fi
|
|
}
|
|
|
|
function wait_server_port_free()
|
|
{
|
|
local port=$1
|
|
local timeout=${2:-120}
|
|
local elapsed=0
|
|
while ss -tlnp | grep -q ":${port}\b" && [ "$elapsed" -lt "$timeout" ]; do
|
|
sleep 1
|
|
elapsed=$((elapsed + 1))
|
|
done
|
|
if [ "$elapsed" -ge "$timeout" ]; then
|
|
echo "Warning: port $port is still in use after ${timeout}s"
|
|
fi
|
|
}
|
|
|
|
function wait_server_ports_free()
|
|
{
|
|
# Wait for all ClickHouse listen ports to become available.
|
|
# In debug+ThreadFuzzer builds, shutdown is slow and the kernel
|
|
# may still hold sockets from the previous server instance.
|
|
# 8123 HTTP 9010 TCP with proxy protocol
|
|
# 8443 HTTPS 9022 SSH
|
|
# 9000 Native TCP 9100 gRPC
|
|
# 9004 MySQL compat 9181 Keeper
|
|
# 9005 PostgreSQL compat 9234 Keeper Raft
|
|
# 9009 Interserver HTTP 9440 Native TCP TLS
|
|
# 9988 Prometheus
|
|
for port in 8123 8443 9000 9004 9005 9009 9010 9022 9100 9181 9234 9440 9988; do
|
|
wait_server_port_free "$port"
|
|
done
|
|
}
|
|
|
|
function report_start_failure()
|
|
{
|
|
echo "Cannot start clickhouse-server"
|
|
rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt ||:
|
|
echo -e "Cannot start clickhouse-server$FAIL$1$(trim_server_logs application_errors.txt)" >> /test_output/test_results.tsv
|
|
cat /var/log/clickhouse-server/stdout.log
|
|
tail -n100 /var/log/clickhouse-server/stderr.log
|
|
tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e '<Warning> RaftInstance:' -e '<Information> RaftInstance' | tail -n100
|
|
}
|
|
|
|
function start_server()
|
|
{
|
|
# After safeExit (forceful shutdown), the kernel may still be cleaning up sockets,
|
|
# especially under sanitizer builds. Wait for the server ports to become available.
|
|
wait_server_ports_free
|
|
|
|
counter=0
|
|
max_attempt=${1:-6}
|
|
local ready_timeout=${2:-$START_SERVER_READY_TIMEOUT}
|
|
# The client port opens only after the databases are loaded, so a running server is bounded
|
|
# by time and only a server that exited is bounded by the number of starts.
|
|
local wait_start=$SECONDS
|
|
until clickhouse-client --handshake_timeout_ms=30000 --receive_timeout 30 --query "SELECT 1"
|
|
do
|
|
# Only wait for ports and restart if the previous daemon actually exited.
|
|
# If the process is still alive, the ports are bound by a healthy but
|
|
# slow-starting server — just retry the readiness check.
|
|
if [ "$counter" -eq 0 ] || ! pgrep -x clickhouse-serv > /dev/null 2>&1
|
|
then
|
|
if [ "$counter" -gt "$max_attempt" ]
|
|
then
|
|
report_start_failure "server process is not running after $counter start attempts"
|
|
return 1
|
|
fi
|
|
[ "$counter" -gt 0 ] && wait_server_ports_free
|
|
clickhouse start --user root >/var/log/clickhouse-server/stdout.log 2>>/var/log/clickhouse-server/stderr.log
|
|
counter=$((counter + 1))
|
|
# The deadline covers one daemon instance, so each start re-anchors it.
|
|
wait_start=$SECONDS
|
|
elif [ "$SECONDS" -ge "$((wait_start + ready_timeout))" ]
|
|
then
|
|
report_start_failure "a clickhouse-server process is running but did not become ready in $((SECONDS - wait_start))s after start attempt $counter completed"
|
|
return 1
|
|
fi
|
|
sleep 10
|
|
done
|
|
|
|
# Positive evidence in the job log that the core limit is usable (the sudoers
|
|
# rlimit_core drop-in worked and the server re-raised its soft limit).
|
|
server_pid="$(cat /var/run/clickhouse-server/clickhouse-server.pid 2>/dev/null)" \
|
|
&& grep -H "^Max core file size" "/proc/${server_pid}/limits" ||:
|
|
}
|
|
|
|
function check_server_start()
|
|
{
|
|
clickhouse-client --receive_timeout 30 --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
|
|
|| (rg --text "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \
|
|
&& echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(trim_server_logs application_errors.txt)" \
|
|
>> /test_output/test_results.tsv)
|
|
|
|
# Remove file application_errors.txt if it's empty
|
|
[ -s /test_output/application_errors.txt ] || rm -f /test_output/application_errors.txt
|
|
}
|
|
|
|
function check_logs_for_critical_errors()
|
|
{
|
|
# ignore:
|
|
# - a.myext which is used in 02724_database_s3.sh and does not exist
|
|
# - "DistributedCacheTCPHandler" and "caller id: None:DistribCache" because they happen inside distributed cache server
|
|
# - "ReadBuffer is canceled by the exception" -- this is a normal situation when the buffer is canceled, the message neded for debug purposes
|
|
# - "ReadBufferFromDistributedCache", "AsynchronousBoundedReadBuffer", "ReadBufferFromS3", "ReadBufferFromAzureBlobStorage"
|
|
# exceptions printed internally by a buffer, exception will be rethrown and handled correctly
|
|
# - "Error during background download" printed by the filesystem cache background download worker (CacheMetadata):
|
|
# the prefetched object may be concurrently removed (DROP/mutation/part removal), the download is just marked failed and the data is re-read directly
|
|
# The matched keys' upload and delete lines are appended to the artifact. The status of the
|
|
# group is the trailing echo's, so a failed collection cannot turn a real failure into "No lost s3 keys".
|
|
rg --text "Code: 499.*The specified key does not exist" /var/log/clickhouse-server/clickhouse-server*.log | grep -v -e "a.myext" -e "ReadBuffer is canceled by the exception" -e "DistributedCacheTCPHandler" -e "ReadBufferFromDistributedCache" -e "ReadBufferFromS3" -e "ReadBufferFromAzureBlobStorage" -e "AsynchronousBoundedReadBuffer" -e "Error during background download" -e "caller id: None:DistribCache" > /test_output/no_such_key_errors.txt \
|
|
&& { python3 /repo/ci/jobs/scripts/s3_key_lifecycle.py /test_output/no_such_key_errors.txt /var/log/clickhouse-server >> /test_output/no_such_key_errors.txt \
|
|
|| echo "--- lifecycle collection FAILED, see the runner log for the traceback ---" >> /test_output/no_such_key_errors.txt ; \
|
|
echo -e "S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)$FAIL$(trim_server_logs no_such_key_errors.txt)" >> /test_output/test_results.tsv ; } \
|
|
|| echo -e "No lost s3 keys$OK" >> /test_output/test_results.tsv
|
|
|
|
rg -Fa "it is lost forever" /var/log/clickhouse-server/clickhouse-server*.log | grep 'SharedMergeTreePartCheckThread' > /dev/null \
|
|
&& echo -e "Lost forever for SharedMergeTree$FAIL" >> /test_output/test_results.tsv \
|
|
|| echo -e "No SharedMergeTree lost forever in clickhouse-server.log$OK" >> /test_output/test_results.tsv
|
|
|
|
# Remove file no_such_key_errors.txt if it's empty
|
|
[ -s /test_output/no_such_key_errors.txt ] || rm -f /test_output/no_such_key_errors.txt
|
|
|
|
dmesg -T > /test_output/dmesg.log
|
|
}
|
|
|
|
# Arguments: logs of the server(s) that produced the top-level profiles, consulted to decide
|
|
# whether rendering is worth it.
|
|
function collect_jemalloc_profiles()
|
|
{
|
|
if [[ -z "$(find "$JEMALLOC_PROFILES_DIR" -name '*.heap' -print -quit)" ]]; then
|
|
echo "No jemalloc profiles were dumped"
|
|
return 0
|
|
fi
|
|
|
|
# Symbolizing a debug build takes minutes, so render only the latest profile and only when
|
|
# memory is in question: the total limit was hit, or the server was killed (OOM killer).
|
|
# The raw .heap files always ship and can be rendered offline with jeprof and the binary.
|
|
# Only top-level profiles belong to the installed binary (see upgrade_runner.sh). Newest by
|
|
# mtime: the servers are restarted several times and each restarts jemalloc's dump sequence.
|
|
local latest
|
|
latest=$(find "$JEMALLOC_PROFILES_DIR" -maxdepth 1 -name '*.heap' -printf '%T@ %p\n' | sort -n | tail -n1 | cut -d' ' -f2-)
|
|
if [[ -n "$latest" ]] \
|
|
&& grep -a -q -F -e 'after total memory exceeded' -e '(total) memory limit exceeded' -e 'Child process was terminated by signal 9' "$@"
|
|
then
|
|
local collapsed="${latest%.heap}.collapsed"
|
|
if timeout --verbose --signal=TERM --kill-after=60 600 \
|
|
jeprof "$(readlink -f "$(command -v clickhouse)")" "$latest" --collapsed > "$collapsed" 2> "${collapsed}.log"
|
|
then
|
|
flamegraph.pl --color mem --width 2560 < "$collapsed" > "${latest%.heap}.svg" || rm -f "${latest%.heap}.svg"
|
|
else
|
|
echo "Failed to symbolize $latest"
|
|
rm -f "$collapsed"
|
|
fi
|
|
fi
|
|
|
|
tar -C "$(dirname "$JEMALLOC_PROFILES_DIR")" -cf - "$(basename "$JEMALLOC_PROFILES_DIR")" | zstd --threads=0 -f -o /test_output/jemalloc.tar.zst \
|
|
|| echo "Failed to archive jemalloc profiles"
|
|
}
|
|
|
|
function collect_query_and_trace_logs()
|
|
{
|
|
for table in query_log trace_log metric_log aggregated_zookeeper_log
|
|
do
|
|
# Don't ignore errors here, it leads to ignore sanitizer reports when running clickhouse-local
|
|
clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst
|
|
done
|
|
}
|
|
|
|
# vi: ft=bash
|