#!/usr/bin/env bash
# End-to-end checks for v5 Nodes on the QEMU development environment of this checkout.
#
# Usage:
#   scripts/node-e2e [--only <scenario>[,<scenario>]] [--list] [--build]
#
#   --list    list the scenarios and exit
#   --only    run only these scenarios, in the given order
#   --build   build Sentinel (host binary) and Flux from $SENTINEL_DIR and install them first
#
# Runs every scenario except offline-removal, which is destructive and runs only when named
# with --only. Each scenario prints PASS or FAIL with the reason and restores what it changed
# (Coolify, VMs, workload, firewall rule, CA rotation, ingress), also when it fails. Exits non-zero
# when a scenario fails.
#
# Needs the instance of this checkout (./scripts/dev start) with the KVM workers node-worker-a
# and node-worker-b in the cluster "Development QEMU mesh" and one workload on worker A, plus
# jq and virsh on the host. Coolify-side steps run through `php artisan dev:node-e2e`.
#
# Env: SENTINEL_DIR=/root/devel/sentinel  NODE_E2E_WORK=<dir for build artifacts and the SSH key>
#      NODE_E2E_SENTINEL_VERSION=<version baked into the Sentinel build>
#
set -euo pipefail
set -o errtrace

cd "$(dirname "$0")/.."
ROOT="$(pwd -P)"

SCENARIOS=(preflight coolify-down-dns ingress reboot-recovery stop-marker flux-restart offline-node ca-rotation ca-rotation-offline offline-removal)
DEFAULT_SCENARIOS=(preflight coolify-down-dns ingress reboot-recovery stop-marker flux-restart offline-node ca-rotation ca-rotation-offline)
declare -A DESCRIPTIONS=(
  [preflight]='stack up, both Nodes connected, cluster active, workload running, DNS on both Nodes'
  [coolify-down-dns]='Coolify down for 150 s: DNS keeps resolving and endpoint expiry keeps moving'
  [ingress]='HTTP ingress on both Nodes: domain served, stop gives 502, start serves, Coolify down 150 s still serves'
  [reboot-recovery]='Coolify down, reboot both VMs: network, Corrosion, DNS, and workload come back'
  [stop-marker]='a stopped workload stays stopped across a reboot; a started one comes back'
  [flux-restart]='restart Flux during a deployment: no operation stays active, workload runs'
  [offline-node]='Node B offline: A converges, B pending, cluster degraded, notifications, B catches up'
  [ca-rotation]='full staged Flux CA rotation with both Nodes online, then system.ping'
  [ca-rotation-offline]='CA rotation with Node B offline during distribution: B blocks the switch, then catches up'
  [offline-removal]='(opt-in, destructive) remove offline Node B, deferred leave on boot, add B back'
)

SENTINEL_DIR="${SENTINEL_DIR:-/root/devel/sentinel}"
WORK="${NODE_E2E_WORK:-$HOME/.cache/coolify-node-e2e}"
GIT_COMMON_DIR="$(git rev-parse --path-format=absolute --git-common-dir)"
ENV_DIR="$(dirname "$GIT_COMMON_DIR")/.dev-instances"
PROJECT="$(./scripts/dev container)"
INSTANCE="${PROJECT#coolify-dev-}"
APP="$PROJECT"
FLUX="$PROJECT-flux"
ENV_FILE="$ENV_DIR/$INSTANCE.env"
FLUX_OVERRIDE="$ENV_DIR/$INSTANCE-flux-override.yml"
RESTART_DIR="$ROOT/storage/app/node-e2e"
KEY="$WORK/ssh-key"
SSH_OPTS=(-i "$KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 -o BatchMode=yes)
declare -A NODE_IP=() NODE_WG=()
WORKLOAD_UUID=''
WORKLOAD_DNS_NAME=''
CONTAINER_NAME=''
WORKLOAD_IP=''
ROTATION_START_VERSION=''

usage() {
  awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0"
}

log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*" >&2; }

# ---------------------------------------------------------------------------
# Scenario bookkeeping: fail() records the reason, note() adds a detail to the result line.

REASON_FILE=''
NOTES_FILE=''
RESTORE_STEPS=()

fail() {
  [[ -s "$REASON_FILE" ]] || printf '%s' "$*" >"$REASON_FILE"
  exit 1
}

note() {
  log "$*"
  printf '%s; ' "$*" >>"$NOTES_FILE"
}

# on_exit <command...>: run when the scenario ends, newest first, also after a failure.
on_exit() { RESTORE_STEPS+=("$*"); }

run_restore_steps() {
  local index
  for ((index = ${#RESTORE_STEPS[@]} - 1; index >= 0; index--)); do
    log "restore: ${RESTORE_STEPS[index]}"
    (eval "${RESTORE_STEPS[index]}") || log "restore step failed: ${RESTORE_STEPS[index]}"
  done
  log 'restore: baseline'
  restore_baseline || log 'restore: baseline is not reached'
}

# wait_until <seconds> <description> <command...>: poll every 5 s.
wait_until() {
  local timeout="$1" description="$2" deadline
  shift 2
  deadline=$((SECONDS + timeout))
  while ! "$@" >/dev/null 2>&1; do
    if ((SECONDS >= deadline)); then
      fail "timeout after ${timeout}s: $description"
    fi
    sleep 5
  done
}

# Same as wait_until, but returns 1 instead of failing the scenario.
try_until() {
  local timeout="$1" deadline
  shift
  deadline=$((SECONDS + timeout))
  while ! "$@" >/dev/null 2>&1; do
    ((SECONDS < deadline)) || return 1
    sleep 5
  done
}

# ---------------------------------------------------------------------------
# Coolify, Flux, VMs, Nodes

artisan() { docker exec -i "$APP" php artisan dev:node-e2e "$@"; }

status_json() { artisan status; }

# st <jq filter>: one value from the status JSON.
st() { status_json | jq -r "$1"; }

st_is() { [[ "$(st "$1" 2>/dev/null)" == "$2" ]]; }

coolify_running() {
  [[ "$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' "$APP" 2>/dev/null)" == 'running healthy' ]] &&
    artisan status >/dev/null 2>&1
}

coolify_stop() {
  log "stopping Coolify ($APP)"
  docker stop "$APP" >/dev/null
}

coolify_start() {
  if [[ "$(docker inspect --format '{{.State.Status}}' "$APP")" != running ]]; then
    log "starting Coolify ($APP)"
    docker start "$APP" >/dev/null
  fi
  try_until 300 coolify_running || {
    log 'Coolify did not become healthy within 5 minutes'
    return 1
  }
}

vm_domain() { echo "coolify-dev-$INSTANCE--node-worker-$1"; }

vm_state() { virsh domstate "$(vm_domain "$1")" 2>/dev/null | head -1; }

node_ssh() {
  local node="$1"
  shift
  ssh "${SSH_OPTS[@]}" "root@${NODE_IP[$node]}" "$@"
}

boot_id() { node_ssh "$1" cat /proc/sys/kernel/random/boot_id 2>/dev/null; }

booted_since() { local current; current="$(boot_id "$1")" && [[ -n "$current" && "$current" != "$2" ]]; }

sentinel_active() { node_ssh "$1" systemctl is-active --quiet sentinel; }

vm_shutdown() {
  log "shutting down VM $(vm_domain "$1")"
  virsh shutdown "$(vm_domain "$1")" >/dev/null
  wait_until 180 "VM $1 shuts off" vm_is_off "$1"
}

vm_is_off() { [[ "$(vm_state "$1")" == 'shut off' ]]; }

vm_start() {
  if [[ "$(vm_state "$1")" != running ]]; then
    log "starting VM $(vm_domain "$1")"
    virsh start "$(vm_domain "$1")" >/dev/null
  fi
  try_until 240 sentinel_active "$1"
}

# vm_reboot <node>...: reboot and wait until SSH answers with a new boot ID.
vm_reboot() {
  local node
  declare -A before=()
  for node in "$@"; do
    before[$node]="$(boot_id "$node")"
    log "rebooting VM $(vm_domain "$node")"
    node_ssh "$node" 'systemctl reboot' >/dev/null 2>&1 || true
  done
  for node in "$@"; do
    wait_until 240 "VM $node boots again" booted_since "$node" "${before[$node]}"
  done
}

node_connected() { st_is ".nodes.$1.connected" true; }

# Connected in Coolify and answering system.ping through Flux right now.
node_reachable() { artisan ping "$1" >/dev/null 2>&1 && node_connected "$1"; }

nodes_reachable() { node_reachable a && node_reachable b; }

container_state() { node_ssh a "podman inspect -f '{{.State.Status}}' $CONTAINER_NAME 2>/dev/null || echo absent"; }

workload_running() { [[ "$(container_state)" == running ]]; }

dns_answer() {
  node_ssh "$1" "dig +short +time=2 +tries=1 @${NODE_WG[$1]} $WORKLOAD_DNS_NAME.default.coolify.internal" 2>/dev/null
}

dns_resolves() { [[ "$(dns_answer "$1")" == "$WORKLOAD_IP" ]]; }

dns_resolves_everywhere() { dns_resolves a && dns_resolves b; }

# Expiry of the workload endpoint row in Corrosion on a Node, as Unix seconds.
endpoint_expires_at() {
  node_ssh "$1" "corrosion query --config /etc/corrosion/config.toml \"SELECT expires_at FROM workload_endpoints WHERE workload_id = '$WORKLOAD_UUID' AND container_ip = '$WORKLOAD_IP'\"" 2>/dev/null | tail -1
}

cluster_active() { st_is '.cluster.network_status' active; }

no_active_workload_operations() { st_is '.workload.active_operations | length' 0; }

# Brings the environment back to normal: Coolify up, both VMs up and connected, workload running.
restore_baseline() {
  coolify_start || return 1
  vm_start a || return 1
  vm_start b || return 1
  try_until 240 nodes_reachable || {
    log 'Nodes did not reconnect within 4 minutes'
    return 1
  }
  if [[ "$(st '.workload.desired_state')" != running ]] || ! workload_running; then
    try_until 300 no_active_workload_operations || true
    artisan lifecycle start >/dev/null || artisan deploy >/dev/null || true
    try_until 180 workload_running || {
      log 'The workload is not running'
      return 1
    }
  fi
  try_until 300 cluster_active || log "cluster network is $(st '.cluster.network_status' 2>/dev/null || echo unknown)"
}

load_environment() {
  local status
  mkdir -p "$WORK"
  if [[ ! -s "$KEY" ]]; then
    docker cp "$APP:/var/www/html/storage/app/ssh/keys/ssh_key@ssh" "$KEY" >/dev/null
  fi
  chmod 600 "$KEY"
  status="$(status_json)" || {
    echo "Coolify ($APP) does not answer. Start the instance with ./scripts/dev start." >&2
    exit 1
  }
  for node in a b; do
    NODE_IP[$node]="$(jq -er ".nodes.$node.ip" <<<"$status")"
    NODE_WG[$node]="$(jq -er ".nodes.$node.wireguard_ip" <<<"$status")"
  done
  WORKLOAD_UUID="$(jq -er '.workload.uuid' <<<"$status")"
  WORKLOAD_DNS_NAME="$(jq -er '.workload.dns_name' <<<"$status")"
  CONTAINER_NAME="$(jq -er '.workload.container_name' <<<"$status")"
  WORKLOAD_IP="$(jq -er '.workload.container_ips[0]' <<<"$status")"
  [[ "$(jq -er '.workload.node_uuids[0]' <<<"$status")" == development-qemu-node-worker-a ]] ||
    { echo 'The end-to-end workload must run on worker A.' >&2; exit 1; }
}

cleanup_work() {
  rm -f "$KEY" "$WORK"/sentinel "$WORK"/*.log
  rmdir "$WORK" 2>/dev/null || true
  rm -rf "$RESTART_DIR"
}

# ---------------------------------------------------------------------------
# Mailpit (the instance SMTP settings point at it)

mailpit_url() { echo "http://127.0.0.1:$(grep -E '^FORWARD_MAILPIT_DASHBOARD_PORT=' "$ENV_FILE" | cut -d= -f2-)"; }

# mail_count <subject substring> <since RFC3339>
mail_count() {
  curl -fsS "$(mailpit_url)/api/v1/messages?limit=200" |
    jq --arg subject "$1" --arg since "$2" '[.messages[] | select((.Subject | contains($subject)) and .Created >= $since)] | length'
}

mail_arrived() { (($(mail_count "$1" "$2") > 0)); }

# ---------------------------------------------------------------------------
# Build and install

build_and_install() {
  local version="${NODE_E2E_SENTINEL_VERSION:-0.0.0-dev-e2e-$(date +%Y%m%d%H%M)}" id node
  mkdir -p "$WORK"
  log "building Sentinel $version from $SENTINEL_DIR"
  docker build -q -f "$SENTINEL_DIR/Dockerfile.host" --build-arg VERSION="$version" -t "sentinel-host:$INSTANCE-dev" "$SENTINEL_DIR" >/dev/null
  log 'building Flux'
  docker build -q -f "$SENTINEL_DIR/Dockerfile.flux" -t "flux:$INSTANCE-dev" "$SENTINEL_DIR" >/dev/null
  id="$(docker create "sentinel-host:$INSTANCE-dev" /sentinel)"
  docker cp "$id:/sentinel" "$WORK/sentinel" >/dev/null
  docker rm "$id" >/dev/null
  for node in a b; do
    log "installing Sentinel on worker $node"
    scp "${SSH_OPTS[@]}" -q "$WORK/sentinel" "root@${NODE_IP[$node]}:/usr/local/bin/sentinel.new"
    node_ssh "$node" 'chmod 755 /usr/local/bin/sentinel.new && mv /usr/local/bin/sentinel.new /usr/local/bin/sentinel && systemctl restart sentinel && systemctl try-restart coolify-discovery-dns'
  done
  rm -f "$WORK/sentinel"
  printf 'services:\n  flux:\n    image: flux:%s-dev\n' "$INSTANCE" >"$FLUX_OVERRIDE"
  log 'recreating Flux with the local image'
  local -a env_files=()
  [[ -f .env ]] && env_files+=(--env-file .env)
  docker compose -p "$PROJECT" -f docker-compose.dev-multi.yml -f docker-compose.node-dev.yml -f "$FLUX_OVERRIDE" \
    "${env_files[@]}" --env-file "$ENV_FILE" up -d --no-deps flux >/dev/null 2>&1
  try_until 180 nodes_reachable || { echo 'Nodes did not reconnect after the install.' >&2; exit 1; }
  for node in a b; do
    log "worker $node runs $(node_ssh "$node" 'sentinel --version')"
  done
}

# Answers the restart request of `dev:node-e2e ca-continue` during the CA switch.
serve_flux_restart() {
  local pid="$1"
  while kill -0 "$pid" 2>/dev/null; do
    if [[ -f "$RESTART_DIR/flux-restart.request" ]]; then
      log "restarting Flux for certificate $(cut -c1-23 "$RESTART_DIR/flux-restart.request")..."
      rm -f "$RESTART_DIR/flux-restart.request"
      docker restart "$FLUX" >/dev/null
      touch "$RESTART_DIR/flux-restart.done"
    fi
    sleep 1
  done
}

# ---------------------------------------------------------------------------
# Scenarios

scenario_preflight() {
  local container
  for container in "$APP" "$FLUX" "$PROJECT-postgres-1" "$PROJECT-redis-1"; do
    [[ "$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)" == running ]] || fail "container $container is not running"
  done
  coolify_running || fail 'Coolify is not healthy'
  [[ "$(vm_state a)" == running && "$(vm_state b)" == running ]] || fail 'both worker VMs must run'
  wait_until 120 'both Nodes connected and answering system.ping' nodes_reachable
  wait_until 300 'cluster network active' cluster_active
  [[ "$(st '.workload.desired_state')" == running ]] || fail 'workload desired state is not running'
  workload_running || fail "workload container is $(container_state)"
  wait_until 60 'DNS resolves on worker A' dns_resolves a
  wait_until 60 'DNS resolves on worker B' dns_resolves b
  note "Sentinel $(st '.nodes.a.sentinel_version')/$(st '.nodes.b.sentinel_version'), revision $(st '.cluster.desired_revision')"
}

scenario_coolify_down_dns() {
  local before_a before_b after_a after_b started
  before_a="$(endpoint_expires_at a)"
  before_b="$(endpoint_expires_at b)"
  [[ -n "$before_a" && -n "$before_b" ]] || fail 'workload endpoint row is missing in Corrosion'
  on_exit coolify_start
  coolify_stop
  started=$SECONDS
  while ((SECONDS - started < 150)); do
    sleep 30
    dns_resolves a || fail "DNS stopped resolving on worker A after $((SECONDS - started))s"
    dns_resolves b || fail "DNS stopped resolving on worker B after $((SECONDS - started))s"
  done
  after_a="$(endpoint_expires_at a)"
  after_b="$(endpoint_expires_at b)"
  ((after_a > before_a + 100 && after_b > before_b + 100)) ||
    fail "endpoint expiry did not move: A $before_a -> $after_a, B $before_b -> $after_b"
  (($(node_ssh a 'date +%s') < after_a)) || fail 'endpoint expired'
  note "expires_at A +$((after_a - before_a))s, B +$((after_b - before_b))s while Coolify was down 150s"
  coolify_start || fail 'Coolify did not start again'
  wait_until 180 'Nodes reconnect after Coolify start' nodes_reachable
}

# ---------------------------------------------------------------------------
# Ingress: requests go to the Node address with the domain in the Host header, so no DNS is needed.

INGRESS_DOMAIN='whoami.node-e2e.test'

# ingress_code <node> [host]: HTTP status of a request through the ingress of a Node (000 without an answer).
ingress_code() {
  curl -s -o /dev/null -m 10 -w '%{http_code}' -H "Host: ${2:-$INGRESS_DOMAIN}" "http://${NODE_IP[$1]}/" 2>/dev/null || true
}

# ingress_serves <node>: the whoami workload answers through the ingress of the Node.
ingress_serves() {
  curl -fsS -m 10 -H "Host: $INGRESS_DOMAIN" "http://${NODE_IP[$1]}/" 2>/dev/null | grep -q "^IP: $WORKLOAD_IP\$"
}

ingress_serves_everywhere() { ingress_serves a && ingress_serves b; }

ingress_no_upstream_everywhere() { [[ "$(ingress_code a)" == 502 && "$(ingress_code b)" == 502 ]]; }

caddy_active_everywhere() {
  node_ssh a 'systemctl is-active --quiet coolify-ingress' && node_ssh b 'systemctl is-active --quiet coolify-ingress'
}

ingress_active_everywhere() { st_is '[.nodes.a.ingress_state, .nodes.b.ingress_state] | join(",")' active,active; }

# restore_ingress <domains> <port> <a on|off> <b on|off>: the workload domains and the ingress setting of both Nodes.
restore_ingress() {
  local result
  result="$(artisan domains "$1" --http-port="$2")" || log "restoring the workload domains failed: $result"
  result="$(artisan ingress a "$3")" || log "restoring ingress on worker A failed: $result"
  result="$(artisan ingress b "$4")" || log "restoring ingress on worker B failed: $result"
}

scenario_ingress() {
  local status result started domains port ingress_a ingress_b
  status="$(status_json)"
  # The original state, restored also after a failure.
  domains="$(jq -r '.workload.domains | join(",")' <<<"$status")"
  port="$(jq -r '.workload.http_port // ""' <<<"$status")"
  ingress_a="$(jq -r 'if .nodes.a.is_ingress then "on" else "off" end' <<<"$status")"
  ingress_b="$(jq -r 'if .nodes.b.is_ingress then "on" else "off" end' <<<"$status")"
  on_exit "restore_ingress '$domains' '$port' $ingress_a $ingress_b"
  started=$SECONDS
  result="$(artisan ingress a on)" || fail "turning on ingress on worker A failed: $result"
  result="$(artisan ingress b on)" || fail "turning on ingress on worker B failed: $result"
  result="$(artisan domains "$INGRESS_DOMAIN" --http-port=80)" || fail "setting the domain failed: $result"
  wait_until 180 'Caddy active on both Nodes' caddy_active_everywhere
  wait_until 180 'the domain is served through both Nodes' ingress_serves_everywhere
  note "served through A (local) and B (over WireGuard) $((SECONDS - started))s after the domain was set"
  [[ "$(ingress_code a unknown.node-e2e.test)/$(ingress_code b unknown.node-e2e.test)" == 404/404 ]] || fail 'an unknown host is not answered with 404'
  wait_until 180 'Coolify shows ingress active on both Nodes' ingress_active_everywhere

  result="$(artisan lifecycle stop)" || fail "stop failed: $result"
  [[ "$(jq -r .status <<<"$result")" == succeeded ]] || fail "stop operation: $result"
  started=$SECONDS
  wait_until 60 'both Nodes answer 502 after the stop' ingress_no_upstream_everywhere
  note "502 on both Nodes $((SECONDS - started))s after the stop"
  result="$(artisan lifecycle start)" || fail "start failed: $result"
  [[ "$(jq -r .status <<<"$result")" == succeeded ]] || fail "start operation: $result"
  started=$SECONDS
  wait_until 60 'served through both Nodes after the start' ingress_serves_everywhere
  note "served again $((SECONDS - started))s after the start"

  on_exit coolify_start
  coolify_stop
  started=$SECONDS
  while ((SECONDS - started < 150)); do
    sleep 30
    ingress_serves a || fail "worker A stopped serving after $((SECONDS - started))s without Coolify (HTTP $(ingress_code a))"
    ingress_serves b || fail "worker B stopped serving after $((SECONDS - started))s without Coolify (HTTP $(ingress_code b))"
  done
  note 'served through both Nodes while Coolify was down 150s'
  coolify_start || fail 'Coolify did not start again'
  wait_until 180 'Nodes reconnect after Coolify start' nodes_reachable
}

# network_restored <node>: WireGuard interface, firewall table, and discovery services after boot.
network_restored() {
  node_ssh "$1" 'ip link show coolify0 >/dev/null && nft list table inet coolify_cluster >/dev/null && systemctl is-active --quiet corrosion coolify-discovery-dns'
}

network_and_dns_restored() { network_restored "$1" && dns_resolves "$1"; }

scenario_reboot_recovery() {
  local node started
  on_exit coolify_start
  coolify_stop
  started=$SECONDS
  vm_reboot a b
  for node in a b; do
    wait_until 180 "network, Corrosion, and DNS restored on worker $node" network_and_dns_restored "$node"
  done
  wait_until 180 'workload running after reboot' workload_running
  note "restored $((SECONDS - started))s after the reboot command, without Coolify"
  coolify_start || fail 'Coolify did not start again'
  wait_until 240 'Nodes reconnect after Coolify start' nodes_reachable
}

scenario_stop_marker() {
  local result started
  on_exit coolify_start
  result="$(artisan lifecycle stop)" || fail "stop failed: $result"
  [[ "$(jq -r .status <<<"$result")" == succeeded ]] || fail "stop operation: $result"
  [[ "$(container_state)" != running ]] || fail 'container still runs after stop'
  coolify_stop
  vm_reboot a
  wait_until 180 'network restored on worker A' network_restored a
  # Give the boot restore time to (wrongly) start the container.
  started=$SECONDS
  while ((SECONDS - started < 60)); do
    [[ "$(container_state)" != running ]] || fail 'stopped workload started after reboot'
    sleep 10
  done
  note "stopped workload stayed $(container_state) after reboot"
  coolify_start || fail 'Coolify did not start again'
  wait_until 240 'worker A reconnects' node_reachable a
  wait_until 120 'no active workload operation' no_active_workload_operations
  result="$(artisan lifecycle start)" || fail "start failed: $result"
  [[ "$(jq -r .status <<<"$result")" == succeeded ]] || fail "start operation: $result"
  wait_until 60 'workload running after start' workload_running
  coolify_stop
  vm_reboot a
  wait_until 180 'workload running after second reboot' workload_running
  wait_until 120 'DNS resolves on both Nodes after second reboot' dns_resolves_everywhere
  coolify_start || fail 'Coolify did not start again'
  wait_until 240 'worker A reconnects' node_reachable a
}

operation_status() { artisan operation "$1" | jq -r .status; }

scenario_flux_restart() {
  local result operation status started
  wait_until 420 'no active workload operation' no_active_workload_operations
  # Deployments need fresh Node resource data, which the scheduler collects every minute.
  started=$SECONDS
  until result="$(artisan deploy --pull=newer)"; do
    [[ "$result" == *'resource data'* && $((SECONDS - started)) -lt 180 ]] || fail "deploy failed: $result"
    sleep 10
  done
  operation="$(jq -r .uuid <<<"$result")"
  [[ "$(jq -r .created <<<"$result")" == true ]] || fail "deployment was not created: $result"
  started=$SECONDS
  status="$(artisan await-operation "$operation" | jq -r .status)"
  [[ "$status" != queued ]] || fail 'deployment stayed queued for 60s'
  # Without a stop timeout: Flux goes away mid-command like after a crash.
  log "restarting Flux while the deployment is $status"
  docker restart -t 0 "$FLUX" >/dev/null
  sleep 5
  note "Flux restarted while the operation was $status, 5s later it was $(operation_status "$operation")"
  # Stale operation recovery runs every five minutes and confirms an uncertain operation that converged.
  wait_until 420 'no active operation for the workload' no_active_workload_operations
  result="$(artisan operation "$operation")"
  note "operation ended $(jq -r .status <<<"$result")$(jq -r 'if .error then ": " + .error else "" end' <<<"$result") after $((SECONDS - started))s"
  wait_until 300 'workload running' workload_running
  wait_until 120 'Nodes reconnected to Flux' nodes_reachable
  wait_until 60 'DNS resolves on both Nodes' dns_resolves_everywhere
}

offline_network_degraded() {
  local status
  status="$(status_json)" || return 1
  jq -e '.nodes.a.network_state == "converged" and .nodes.a.network_applied_revision == .cluster.desired_revision
    and .nodes.b.network_status == "pending" and .cluster.network_status == "degraded"' <<<"$status" >/dev/null
}

unreachable_notified() {
  st_is '.nodes.b.unreachable_notified_at != null and .cluster.network_unhealthy_notified_at != null' true
}

recovered() {
  local status
  status="$(status_json)" || return 1
  jq -e '.cluster.network_status == "active" and .nodes.b.network_state == "converged"
    and .nodes.b.network_applied_revision == .cluster.desired_revision
    and .nodes.b.unreachable_notified_at == null and .cluster.network_unhealthy_notified_at == null' <<<"$status" >/dev/null
}

scenario_offline_node() {
  local since started result
  since="$(date -u +%Y-%m-%dT%H:%M:%S)"
  on_exit 'vm_start b'
  vm_shutdown b
  started=$SECONDS
  wait_until 180 'Coolify sees worker B unreachable' eval '! node_connected b'
  note "B disconnected after $((SECONDS - started))s"
  on_exit 'artisan firewall-remove >/dev/null'
  result="$(artisan firewall-add)" || fail "adding the firewall rule failed: $result"
  wait_until 240 'A converged, B pending, cluster degraded after a rule change' offline_network_degraded
  wait_until 360 'unreachable and cluster unhealthy notification state set' unreachable_notified
  note "notification state set $((SECONDS - started))s after shutdown"
  result="$(artisan firewall-remove)" || fail "removing the firewall rule failed: $result"
  wait_until 240 'A converged on the second revision while B is offline' offline_network_degraded
  if st_is '.notification_channels.server_unreachable | index("EmailChannel") != null' true; then
    wait_until 60 'unreachable mail in Mailpit' mail_arrived 'QEMU worker node B) is unreachable' "$since"
    wait_until 60 'cluster degraded mail in Mailpit' mail_arrived 'Development QEMU mesh) is degraded' "$since"
  else
    note 'unreachable mails skipped: email is disabled for server_unreachable'
  fi
  vm_start b || fail 'worker B did not boot'
  started=$SECONDS
  wait_until 180 'worker B reconnects' node_reachable b
  wait_until 300 'B converges, cluster active, notification state cleared' recovered
  note "B converged $((SECONDS - started))s after it reconnected"
  wait_until 60 'DNS resolves on both Nodes' dns_resolves_everywhere
  if st_is '.notification_channels.server_reachable | index("EmailChannel") != null' true; then
    wait_until 120 'recovery mails in Mailpit' mail_arrived 'QEMU worker node B) is reachable' "$since"
  else
    note 'recovery mails skipped: the team disabled email for server_reachable (the default)'
  fi
  note "mails: $(curl -fsS "$(mailpit_url)/api/v1/messages?limit=50" | jq -r --arg since "$since" '[.messages[] | select(.Created >= $since) | .Subject] | join(" | ")')"
}

rotation_status() { artisan ca-status; }

rotation_blocking_none() { [[ "$(rotation_status | jq -r .blocking)" == 0 ]]; }

rotation_idle() { [[ "$(rotation_status | jq -r '.rotation.in_progress // false')" == false ]]; }

pings_work() { artisan ping a >/dev/null && artisan ping b >/dev/null; }

# Finishes or cancels a rotation that a failed run left behind.
rotation_restore() {
  local step
  for _ in 1 2 3 4; do
    rotation_idle && return 0
    step="$(rotation_status | jq -r .next_step)"
    log "rotation restore: $step"
    if [[ "$step" == switch ]]; then
      artisan ca-cancel >/dev/null || return 1
      continue
    fi
    try_until 180 rotation_blocking_none || true
    rotation_continue || return 1
  done
}

# Runs `ca-continue` and restarts Flux when the CA switch asks for it.
rotation_continue() {
  local output="$WORK/ca-continue.json" pid status
  mkdir -p "$RESTART_DIR"
  rm -f "$RESTART_DIR"/flux-restart.*
  artisan ca-continue >"$output" 2>&1 &
  pid=$!
  serve_flux_restart "$pid"
  wait "$pid" && status=0 || status=$?
  if ((status != 0)); then
    log "ca-continue failed: $(cat "$output")"
  fi
  rm -f "$output"
  return "$status"
}

# node_trust_bundle <node>: "<certificate count> <bundle version> <sha256 of the first CA>" as Sentinel stores them.
node_trust_bundle() {
  node_ssh "$1" 'echo "$(grep -c "BEGIN CERTIFICATE" /etc/coolify/sentinel-flux-ca.pem) $(cat /etc/coolify/sentinel-flux-ca.version) $(openssl x509 -in /etc/coolify/sentinel-flux-ca.pem -noout -fingerprint -sha256 | cut -d= -f2 | tr -d : | tr A-F a-f)"'
}

# rotation_start: start a rotation and print the bundle version before it.
rotation_start() {
  local start_version result
  rotation_idle || fail 'a CA rotation is already in progress'
  start_version="$(rotation_status | jq -r .bundle_version)"
  on_exit rotation_restore
  result="$(artisan ca-start)" || fail "start failed: $result"
  [[ "$(jq -r .rotation.dual_bundle_version <<<"$result")" == $((start_version + 1)) ]] || fail "unexpected dual bundle: $result"
  ROTATION_START_VERSION=$start_version
}

# rotation_complete: every step after both Nodes acknowledged the dual bundle, with checks.
rotation_complete() {
  local start_version="$ROTATION_START_VERSION" result node
  rotation_continue || fail 'switch to the new CA failed'
  [[ "$(rotation_status | jq -r .rotation.status)" == switched ]] || fail 'rotation is not switched'
  wait_until 240 'Nodes reconnect through the new Flux certificate' nodes_reachable
  wait_until 60 'system.ping through the new certificate' pings_work
  rotation_continue || fail 'retirement start failed'
  wait_until 300 'both Nodes acknowledge the final bundle' rotation_blocking_none
  rotation_continue || fail 'retiring the old CA failed'
  result="$(rotation_status)"
  [[ "$(jq -r .rotation.status <<<"$result")" == completed ]] || fail "rotation did not complete: $(jq -c .rotation <<<"$result")"
  [[ "$(jq -r '[.nodes[] | select(.is_usable) | .version] | unique | join(",")' <<<"$result")" == $((start_version + 2)) ]] ||
    fail "Nodes did not all acknowledge bundle $((start_version + 2)): $(jq -c '[.nodes[] | {name, version}]' <<<"$result")"
  [[ "$(jq -r .rotation.switch_forced <<<"$result")/$(jq -r .rotation.completion_forced <<<"$result")" == false/false ]] || fail 'a step was forced'
  for node in a b; do
    [[ "$(node_trust_bundle "$node")" == "1 $((start_version + 2)) $(jq -r .rotation.to_fingerprint <<<"$result")" ]] ||
      fail "worker $node trusts $(node_trust_bundle "$node") instead of the new CA alone"
  done
  wait_until 120 'Nodes connected after the rotation' nodes_reachable
  wait_until 60 'system.ping on both Nodes' pings_work
}

scenario_ca_rotation() {
  local started=$SECONDS
  rotation_start
  wait_until 300 'both Nodes acknowledge the dual bundle' rotation_blocking_none
  note "dual bundle $((ROTATION_START_VERSION + 1)) acknowledged after $((SECONDS - started))s"
  rotation_complete
  note "completed in $((SECONDS - started))s, bundle $ROTATION_START_VERSION -> $((ROTATION_START_VERSION + 2))"
}

node_acknowledged() { [[ "$(rotation_status | jq -r --arg uuid "development-qemu-node-worker-$1" '.nodes[] | select(.uuid == $uuid) | .acknowledged')" == true ]]; }

scenario_ca_rotation_offline() {
  local started output
  on_exit 'vm_start b'
  vm_shutdown b
  wait_until 180 'Coolify sees worker B unreachable' eval '! node_connected b'
  rotation_start
  wait_until 300 'worker A acknowledges the dual bundle' node_acknowledged a
  [[ "$(rotation_status | jq -r .blocking)" == 1 ]] || fail 'offline worker B does not block the switch'
  output="$(artisan ca-continue)" && fail 'the switch ran while worker B had not acknowledged the dual bundle'
  [[ "$output" == *'QEMU worker node B'* ]] || fail "unexpected switch refusal: $output"
  vm_start b || fail 'worker B did not boot'
  started=$SECONDS
  wait_until 180 'worker B reconnects' node_reachable b
  wait_until 360 'worker B catches up with the dual bundle' rotation_blocking_none
  note "B acknowledged the dual bundle $((SECONDS - started))s after it booted"
  rotation_complete
  note "completed, bundle $ROTATION_START_VERSION -> $((ROTATION_START_VERSION + 2))"
}

leave_completed_on_b() {
  st_is '.nodes.b.network_pending_leave' false &&
    node_ssh b '! ip link show coolify0 >/dev/null 2>&1 && ! nft list table inet coolify_cluster >/dev/null 2>&1 && ! systemctl is-active --quiet corrosion'
}

scenario_offline_removal() {
  local result
  [[ "$(st '.nodes.b.cluster_id')" == "$(st '.cluster.id')" ]] || fail 'worker B is not in the cluster'
  on_exit 'vm_start b'
  on_exit readd_node_b
  vm_shutdown b
  wait_until 180 'Coolify sees worker B unreachable' eval '! node_connected b'
  result="$(artisan remove-node b)" || fail "removal failed: $result"
  [[ "$(jq -r .pending_leave <<<"$result")" == true ]] || fail "leave was not deferred: $result"
  wait_until 240 'cluster active with worker A only' cluster_active
  vm_start b || fail 'worker B did not boot'
  wait_until 180 'worker B reconnects' node_reachable b
  wait_until 300 'deferred leave ran and B network is torn down' leave_completed_on_b
  note 'deferred leave completed after B reconnected'
  readd_node_b || fail 'adding worker B back failed'
  wait_until 240 'cluster active with both Nodes' cluster_active
  wait_until 120 'DNS resolves on both Nodes' dns_resolves_everywhere
}

readd_node_b() {
  local result
  if [[ "$(st '.nodes.b.cluster_id')" == "$(st '.cluster.id')" ]]; then
    return 0
  fi
  try_until 240 node_reachable b || return 1
  try_until 300 eval "st_is '.nodes.b.network_pending_leave' false" || return 1
  result="$(artisan add-node b)" || {
    log "add-node failed: $result"
    return 1
  }
  NODE_WG[b]="$(st '.nodes.b.wireguard_ip')"
}

# ---------------------------------------------------------------------------

run_scenario() {
  local name="$1" status started reason notes
  REASON_FILE="$(mktemp -p "$WORK")"
  NOTES_FILE="$(mktemp -p "$WORK")"
  started=$SECONDS
  log "=== $name: ${DESCRIPTIONS[$name]}"
  set +e
  (
    set -e
    RESTORE_STEPS=()
    trap '[[ -s "$REASON_FILE" ]] || printf "unexpected failure at line %s: %s" "$LINENO" "$BASH_COMMAND" >"$REASON_FILE"' ERR
    trap run_restore_steps EXIT
    "scenario_${name//-/_}"
  )
  status=$?
  set -e
  reason="$(cat "$REASON_FILE")"
  notes="$(sed 's/; $//' "$NOTES_FILE")"
  rm -f "$REASON_FILE" "$NOTES_FILE"
  if ((status == 0)); then
    RESULTS+=("PASS  $name ($((SECONDS - started))s)${notes:+ - $notes}")
    return 0
  fi
  RESULTS+=("FAIL  $name ($((SECONDS - started))s) - ${reason:-exit $status}${notes:+ [$notes]}")
  return 1
}

main() {
  local build=false only='' name failed=0
  local -a selected=()
  while [[ $# -gt 0 ]]; do
    case "$1" in
      --list)
        for name in "${SCENARIOS[@]}"; do
          printf '%-20s %s\n' "$name" "${DESCRIPTIONS[$name]}"
        done
        exit 0
        ;;
      --only)
        only="${2:?--only needs a scenario list}"
        shift
        ;;
      --build) build=true ;;
      -h | --help)
        usage
        exit 0
        ;;
      *)
        usage >&2
        exit 2
        ;;
    esac
    shift
  done
  if [[ -n "$only" ]]; then
    IFS=, read -ra selected <<<"$only"
    for name in "${selected[@]}"; do
      [[ -v "DESCRIPTIONS[$name]" ]] || {
        echo "Unknown scenario '$name'. Use --list." >&2
        exit 2
      }
    done
  else
    selected=("${DEFAULT_SCENARIOS[@]}")
  fi

  trap cleanup_work EXIT
  load_environment
  if [[ "$build" == true ]]; then
    build_and_install
  fi

  RESULTS=()
  for name in "${selected[@]}"; do
    run_scenario "$name" || failed=1
  done
  printf '\n'
  printf '%s\n' "${RESULTS[@]}"
  return "$failed"
}

main "$@"