fix(ci): bound dependency downloads and recover through mirrors

.github/scripts/ci-gradle.init.gradle

@@ -0,0 +1,54 @@
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier
+import org.gradle.api.artifacts.repositories.MavenArtifactRepository
+
+def target = System.getenv('DBX_CI_GRADLE_ROOT')
+def backup = System.getenv('DBX_CI_GRADLE_SOURCE') == 'backup'
+def mirror = 'https://maven-central.storage-download.googleapis.com/maven2/'
+def applies = { File directory -> target != null && directory.canonicalPath == target }
+
+beforeSettings { settings ->
+    if (applies(settings.settingsDir)) {
+        if (System.getenv('DBX_CI_GRADLE_PHASE') == 'execute') {
+            gradle.startParameter.offline = true
+        }
+        if (backup) {
+            settings.pluginManagement.repositories.clear()
+            settings.pluginManagement.repositories.maven { url = uri(mirror) }
+        }
+    }
+}
+
+afterProject { project ->
+    if (applies(project.rootDir) && backup) {
+        project.repositories.withType(MavenArtifactRepository).each { repository ->
+            if (repository.url.toString() in ['https://repo.maven.apache.org/maven2/',
+                                              'https://repo.maven.apache.org/maven2']) {
+                if (repository.credentials.username || repository.credentials.password) {
+                    throw new GradleException('CI mirror switching does not support authenticated Central repositories')
+                }
+                repository.url = uri(mirror)
+            }
+        }
+    }
+}
+
+projectsEvaluated {
+    if (applies(rootProject.rootDir)) {
+        rootProject.tasks.register('dbxCiAcquireDependencies') {
+            doLast {
+                rootProject.allprojects.each { project ->
+                    project.configurations.findAll { configuration ->
+                        configuration.canBeResolved &&
+                            (System.getenv('DBX_CI_GRADLE_SCOPE') != 'jdbc' ||
+                             configuration.name in ['compileClasspath', 'runtimeClasspath', 'annotationProcessor', 'shadow'])
+                    }.each { configuration ->
+                        def artifacts = configuration.incoming.artifactView {
+                            componentFilter { identifier -> identifier instanceof ModuleComponentIdentifier }
+                        }.files.files
+                        println("Acquired ${project.path}:${configuration.name}: ${artifacts.size()} external artifacts")
+                    }
+                }
+            }
+        }
+    }
+}

.github/scripts/ci-gradle.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""Acquire Gradle dependencies with bounded recovery, then execute the original command once offline."""
+
+import argparse
+import os
+from pathlib import Path
+import re
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+from urllib.parse import urlsplit
+
+ROOT = Path(__file__).resolve().parents[2]
+INIT = Path(__file__).with_name("ci-gradle.init.gradle")
+starting_process = False
+pending_cancel = None
+
+
+class Cancelled(BaseException):
+    def __init__(self, signum):
+        self.signum = signum
+
+
+def cancel(signum, _frame):
+    global pending_cancel
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    if starting_process:
+        pending_cancel = signum
+    else:
+        raise Cancelled(signum)
+
+
+def dispose(process):
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+    return process.communicate()[0] or ""
+
+
+def run_process(command, cwd, env, budget, capture=True):
+    global starting_process, pending_cancel
+    process = None
+    timed_out = False
+    try:
+        starting_process = True
+        process = subprocess.Popen(command, cwd=cwd, env=env,
+                                   stdout=subprocess.PIPE if capture else None,
+                                   stderr=subprocess.STDOUT if capture else None,
+                                   text=True, start_new_session=True)
+        starting_process = False
+        if pending_cancel is not None:
+            signum, pending_cancel = pending_cancel, None
+            raise Cancelled(signum)
+        try:
+            output = process.communicate(timeout=budget)[0] or ""
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            output = dispose(process)
+    except BaseException:
+        if process is not None:
+            dispose(process)
+        raise
+    finally:
+        starting_process = False
+    if capture:
+        def redact(match):
+            url = urlsplit(match[0])
+            return f"{url.scheme}://{url.hostname}{url.path}"
+        print(re.sub(r"https?://[^\s'\"<>]+", redact, output), end="", flush=True)
+    return process.returncode, output, timed_out
+
+
+def public_maven_url(value):
+    url = urlsplit(value)
+    return (url.scheme == "https" and not url.username and not url.password and
+            not url.query and not url.fragment and url.port in (None, 443) and
+            ((url.hostname == "repo.maven.apache.org" and url.path.startswith("/maven2/")) or
+             (url.hostname == "plugins.gradle.org" and url.path.startswith("/m2/")) or
+             (url.hostname == "maven-central.storage-download.googleapis.com" and url.path.startswith("/maven2/"))))
+
+
+def acquisition_failure(output, timed_out=False):
+    if re.search(r"certificate|PKIX|SSLHandshake|checksum|signature|verification failed|"
+                 r"No space left|Permission denied|Could not compile|Compilation failed|"
+                 r"Could not find |Could not resolve.*conflict|401\b|404\b", output, re.I):
+        return None
+    section = output.split("* What went wrong:", 1)[-1].split("* Try:", 1)[0]
+    requests = re.findall(r"Could not (?:GET|HEAD) '([^']+)'", section)
+    if timed_out and "* What went wrong:" not in output and not requests:
+        return "transient"
+    if not requests or not all(public_maven_url(url) for url in requests):
+        return None
+    network = re.compile(r"timed out|Connection (?:reset|refused)|UnknownHostException|"
+                         r"Temporary failure|Name or service not known|Remote host terminated|"
+                         r"Received status code (?:403|429|500|502|503|504)\b", re.I)
+    context = re.compile(r"^(?:A problem occurred configuring |Could not resolve |"
+                         r"Could not get resource |Could not (?:GET|HEAD) '|"
+                         r"There (?:is|are) \d+ more failure)")
+    details = re.findall(r"^\s*>\s*(.+)$", section, re.M)
+    if not details or any(not context.search(line) and not network.search(line) for line in details):
+        return None
+    if any(re.search(r"Received status code (?!403\b|429\b|500\b|502\b|503\b|504\b)\d+", line)
+           for line in details):
+        return None
+    if re.search(r"Received status code 403\b", section):
+        return "forbidden"
+    return "transient" if timed_out or network.search(section) else None
+
+
+def wrapper_failure(output, timed_out):
+    if re.search(r"certificate|PKIX|checksum|signature|Permission denied|No space left|"
+                 r"HTTP response code: (?:401|403|404)", output, re.I):
+        return False
+    return timed_out or bool(re.search(r"timed out|UnknownHostException|Connection (?:reset|refused)|"
+                                      r"HTTP response code: (?:429|500|502|503|504)", output, re.I))
+
+
+def acquire_and_run(project, scope, command, runner=run_process, sleep=time.sleep):
+    env = {**os.environ, "LC_ALL": "C", "DBX_CI_GRADLE_ROOT": str(project.resolve()),
+           "DBX_CI_GRADLE_SCOPE": scope, "DBX_CI_GRADLE_SOURCE": "primary",
+           "DBX_CI_GRADLE_PHASE": "acquire"}
+    env["GRADLE_OPTS"] = env.get("GRADLE_OPTS", "") + " -Dorg.gradle.daemon=false -Duser.language=en -Duser.country=US"
+    home = Path(env.get("GRADLE_USER_HOME", str(Path.home() / ".gradle"))).expanduser().resolve()
+    env["GRADLE_USER_HOME"] = str(home)
+    init_directory = home / "init.d"
+    init_directory.mkdir(parents=True, exist_ok=True)
+    wrapper = [str(project / "gradlew"), "--no-daemon", "--console=plain"]
+    init_path = None
+    try:
+        with tempfile.NamedTemporaryFile(prefix="dbx-ci-acquire-", suffix=".gradle", dir=init_directory,
+                                         delete=False) as temporary:
+            init_path = Path(temporary.name)
+            temporary.write(INIT.read_bytes())
+        for attempt in range(3):
+            print(f"Gradle phase=wrapper attempt={attempt + 1} budget=180s", flush=True)
+            code, output, timed_out = runner([*wrapper, "--version"], project, env, 180)
+            if code == 0 and not timed_out:
+                break
+            if attempt == 2 or not wrapper_failure(output, timed_out):
+                return code or 1
+            sleep(2 ** (attempt + 1))
+        acquisition_count = 0
+        for source in ["primary", "backup"]:
+            env["DBX_CI_GRADLE_SOURCE"] = source
+            attempts = 2 if source == "primary" else 3 - acquisition_count
+            for attempt in range(attempts):
+                acquisition_count += 1
+                print(f"Gradle phase=acquire source={source} attempt={attempt + 1} budget=600s", flush=True)
+                code, output, timed_out = runner([*wrapper, "dbxCiAcquireDependencies"], project, env, 600)
+                if code == 0 and not timed_out:
+                    env["DBX_CI_GRADLE_PHASE"] = "execute"
+                    print(f"Gradle phase=execute source={source} offline=true attempts=1", flush=True)
+                    code, _output, _timed_out = runner(command, project, env, None, capture=False)
+                    return code
+                reason = acquisition_failure(output, timed_out)
+                if reason is None:
+                    return code or 1
+                if source == "primary" and (reason == "forbidden" or attempt + 1 == attempts):
+                    break
+                if attempt + 1 == attempts:
+                    return code or 1
+                sleep(2)
+            if source == "primary":
+                print("Gradle switching once to Google-hosted Maven Central mirror", flush=True)
+        return 1
+    finally:
+        if init_path is not None:
+            init_path.unlink()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("project", choices=["agents", "jdbc"])
+    parser.add_argument("command", nargs=argparse.REMAINDER)
+    args = parser.parse_args()
+    command = args.command[1:] if args.command[:1] == ["--"] else args.command
+    if not command:
+        parser.error("an original command is required after --")
+    project = ROOT / ("agents" if args.project == "agents" else "plugins/jdbc")
+    signal.signal(signal.SIGINT, cancel)
+    signal.signal(signal.SIGTERM, cancel)
+    try:
+        return acquire_and_run(project, args.project, command)
+    except Cancelled as error:
+        print("Gradle cancelled; no acquisition retry", file=sys.stderr)
+        return 128 + error.signum
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/ci-gradle.test.py

@@ -0,0 +1,222 @@
+import importlib.util
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+from unittest.mock import patch
+
+SPEC = importlib.util.spec_from_file_location("ci_gradle", Path(__file__).with_name("ci-gradle.py"))
+gradle = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(gradle)
+OK = (0, "", False)
+
+
+def failure(status=403, url="https://repo.maven.apache.org/maven2/example/a.pom"):
+    return (1, f"""FAILURE: Build failed with an exception.
+* What went wrong:
+A problem occurred configuring root project 'fixture'.
+> Could not resolve all artifacts for configuration 'classpath'.
+   > Could not resolve example:a:1.
+      > Could not get resource '{url}'.
+         > Could not GET '{url}'.
+            > Received status code {status} from server: Forbidden
+* Try:
+> Run with --stacktrace option to get the stack trace.
+""", False)
+
+
+class AcquisitionTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+        self.project = self.root / "project"
+        self.project.mkdir()
+        self.home = self.root / "gradle-home"
+        self.calls = []
+        self.sleeps = []
+
+    def execute(self, responses):
+        responses = iter(responses)
+
+        def run(command, cwd, env, budget, capture=True):
+            self.assertEqual(cwd, self.project)
+            self.assertEqual(len(list((self.home / "init.d").glob("dbx-ci-acquire-*.gradle"))), 1)
+            self.calls.append((command, dict(env), budget, capture))
+            result = next(responses)
+            if isinstance(result, BaseException):
+                raise result
+            return result
+
+        with patch.dict(os.environ, {"GRADLE_USER_HOME": str(self.home)}):
+            try:
+                return gradle.acquire_and_run(self.project, "agents", ["original", "test", "shadowJar", "--continue"],
+                                              runner=run, sleep=self.sleeps.append)
+            finally:
+                self.assertFalse(list((self.home / "init.d").glob("dbx-ci-acquire-*.gradle")))
+
+    def test_success_acquires_before_one_offline_original_command(self):
+        self.assertEqual(self.execute([OK, OK, OK]), 0)
+        self.assertEqual([call[2] for call in self.calls], [180, 600, None])
+        self.assertEqual(self.calls[1][0][-1], "dbxCiAcquireDependencies")
+        self.assertEqual(self.calls[2][0], ["original", "test", "shadowJar", "--continue"])
+        self.assertEqual(self.calls[2][1]["DBX_CI_GRADLE_PHASE"], "execute")
+        self.assertEqual(self.calls[2][1]["DBX_CI_GRADLE_SOURCE"], "primary")
+        self.assertFalse(self.calls[2][3])
+        self.assertEqual(self.sleeps, [])
+
+    def test_public_forbidden_switches_once_without_repeating_original(self):
+        self.assertEqual(self.execute([OK, failure(), OK, OK]), 0)
+        self.assertEqual([call[1]["DBX_CI_GRADLE_SOURCE"] for call in self.calls],
+                         ["primary", "primary", "backup", "backup"])
+        self.assertEqual(self.sleeps, [])
+
+    def test_transient_retries_once_before_backup(self):
+        self.assertEqual(self.execute([OK, failure(503), failure(503), OK, OK]), 0)
+        self.assertEqual(self.sleeps, [2])
+        self.assertEqual([call[1]["DBX_CI_GRADLE_SOURCE"] for call in self.calls[1:]],
+                         ["primary", "primary", "backup", "backup"])
+
+    def test_backup_failure_and_execution_failure_are_terminal(self):
+        for responses, count in [([OK, failure(), failure(), failure()], 4),
+                                 ([OK, OK, failure()], 3)]:
+            with self.subTest(responses=responses):
+                self.calls.clear()
+                self.assertEqual(self.execute(responses), 1)
+                self.assertEqual(len(self.calls), count)
+
+    def test_forbidden_preserves_remaining_attempt_for_backup(self):
+        backup_error = failure(503, "https://maven-central.storage-download.googleapis.com/maven2/example/a.pom")
+        self.assertEqual(self.execute([OK, failure(), backup_error, OK, OK]), 0)
+        self.assertEqual([call[1]["DBX_CI_GRADLE_SOURCE"] for call in self.calls[1:]],
+                         ["primary", "backup", "backup", "backup"])
+        self.assertEqual(self.sleeps, [2])
+
+    def test_timeout_recovers_only_acquisition(self):
+        timeout = (-9, "Downloading dependencies\n", True)
+        self.assertEqual(self.execute([OK, timeout, timeout, OK, OK]), 0)
+        self.assertEqual(self.sleeps, [2])
+
+    def test_wrapper_has_separate_bounded_retry_and_no_project_task(self):
+        self.assertEqual(self.execute([(1, "java.net.SocketTimeoutException: Read timed out", False), OK, OK, OK]), 0)
+        self.assertEqual(self.sleeps, [2])
+        self.assertTrue(all(call[0][-1] == "--version" for call in self.calls[:2]))
+
+    def test_deterministic_private_and_mixed_failures_do_not_switch(self):
+        errors = [failure(404), failure(401), failure(url="https://private.test/maven2/a.pom"),
+                  failure(url="https://user:password@repo.maven.apache.org/maven2/a.pom"),
+                  failure(url="https://repo.maven.apache.org/maven2/a.pom?token=secret"),
+                  (1, failure()[1] + "\n> No space left on device", False),
+                  (1, failure()[1].replace("* Try:", "> PKIX certificate failure\n* Try:"), False),
+                  (1, failure()[1].replace("* Try:", "> An unknown failure\n* Try:"), False)]
+        for error in errors:
+            with self.subTest(error=error):
+                self.calls.clear()
+                self.assertEqual(self.execute([OK, error]), 1)
+                self.assertEqual(len(self.calls), 2)
+
+    def test_unknown_wrapper_failure_does_not_enter_acquisition(self):
+        self.assertEqual(self.execute([(1, "checksum verification failed", False)]), 1)
+        self.assertEqual(len(self.calls), 1)
+
+    def test_cancellation_never_switches_and_preserves_other_init_scripts(self):
+        directory = self.home / "init.d"
+        directory.mkdir(parents=True)
+        other = directory / "existing.gradle"
+        other.write_text("// existing")
+        with self.assertRaises(gradle.Cancelled):
+            self.execute([OK, gradle.Cancelled(signal.SIGTERM)])
+        self.assertEqual(len(self.calls), 2)
+        self.assertEqual(other.read_text(), "// existing")
+
+    def test_original_exception_still_cleans_up(self):
+        with self.assertRaises(OSError):
+            self.execute([OK, OK, OSError("cannot start original")])
+        self.assertEqual(len(self.calls), 3)
+
+    @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups")
+    def test_spawn_cancellation_reaps_child_before_init_cleanup(self):
+        events = []
+        with patch.object(gradle.signal, "signal"), patch.object(gradle.os, "killpg", side_effect=lambda *_: events.append("kill")), \
+                patch.object(gradle.subprocess, "Popen") as popen:
+            child = popen.return_value
+            child.communicate.side_effect = lambda: (events.append("reap") or "", None)
+
+            def spawn(*_args, **_kwargs):
+                gradle.cancel(signal.SIGTERM, None)
+                return child
+
+            popen.side_effect = spawn
+            with self.assertRaises(gradle.Cancelled):
+                gradle.run_process(["gradle"], self.project, {}, 1)
+        self.assertEqual(events, ["kill", "reap"])
+
+    @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups and /proc")
+    def test_timeout_and_cancellation_dispose_owned_descendants(self):
+        for signum in [0, signal.SIGTERM, signal.SIGINT]:
+            with self.subTest(signum=signum):
+                case = self.root / str(signum)
+                case.mkdir()
+                pids = case / "pids"
+                harness = textwrap.dedent("""
+                    import importlib.util, os, signal, sys
+                    from pathlib import Path
+                    spec = importlib.util.spec_from_file_location('acquire', sys.argv[1])
+                    acquire = importlib.util.module_from_spec(spec); spec.loader.exec_module(acquire)
+                    signal.signal(signal.SIGINT, acquire.cancel)
+                    signal.signal(signal.SIGTERM, acquire.cancel)
+                    os.environ['GRADLE_USER_HOME'] = sys.argv[2]
+                    payload = '''import os, subprocess, sys, time
+                    child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])
+                    with open(sys.argv[1], 'a') as output: print(os.getpid(), child.pid, file=output)
+                    time.sleep(60)
+                    '''
+                    def run(command, cwd, env, budget, capture=True):
+                        if command[-1] == '--version': return 0, '', False
+                        if not capture: raise RuntimeError('execution reached without acquisition')
+                        return acquire.run_process([sys.executable, '-c', payload, sys.argv[3]], cwd, env, 1)
+                    try:
+                        code = acquire.acquire_and_run(Path(sys.argv[2]), 'agents', ['original'], runner=run, sleep=lambda _: None)
+                        sys.exit(code)
+                    except acquire.Cancelled as error:
+                        sys.exit(128 + error.signum)
+                """)
+                process = subprocess.Popen([sys.executable, "-c", harness, gradle.__file__,
+                                            str(case), str(pids)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+                try:
+                    deadline = time.monotonic() + 10
+                    while (not pids.exists() or not pids.read_text().strip()) and process.poll() is None and time.monotonic() < deadline:
+                        time.sleep(0.02)
+                    self.assertTrue(pids.exists() and pids.read_text().strip(), "fake acquisition did not start")
+                    if signum:
+                        process.send_signal(signum)
+                    output = process.communicate(timeout=10)[0]
+                    self.assertNotEqual(process.returncode, 0, output)
+                    self.assertNotIn("phase=execute", output)
+                    self.assertEqual(len(pids.read_text().splitlines()), 1 if signum else 3)
+                    self.assertFalse(list((case / "init.d").glob("dbx-ci-acquire-*.gradle")))
+                    for pid in pids.read_text().split():
+                        status = Path(f"/proc/{pid}/stat")
+                        deadline = time.monotonic() + 5
+                        while status.exists() and status.read_text().split()[2] != "Z" and time.monotonic() < deadline:
+                            time.sleep(0.02)
+                        self.assertTrue(not status.exists() or status.read_text().split()[2] == "Z", f"owned child {pid} survived")
+                finally:
+                    if pids.exists():
+                        for line in pids.read_text().splitlines():
+                            try:
+                                os.killpg(int(line.split()[0]), signal.SIGKILL)
+                            except ProcessLookupError:
+                                pass
+                    if process.poll() is None:
+                        process.kill()
+                    process.communicate(timeout=5)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/scripts/ci-install-mold.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+archive=$(mktemp)
+trap 'rm -f -- "$archive"' EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+curl --fail --location --silent --show-error \
+  --retry 2 --retry-connrefused --retry-delay 2 \
+  --connect-timeout 15 --max-time 120 --retry-max-time 360 \
+  https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
+  --output "$archive"
+sudo tar -C /usr/local --strip-components=1 -xzf "$archive"
+mold --version

.github/scripts/ci-install-mold.test.py

@@ -0,0 +1,117 @@
+import http.server
+import os
+from pathlib import Path
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import threading
+import unittest
+
+SCRIPT = Path(__file__).with_name("ci-install-mold.sh")
+
+
+@unittest.skipUnless(sys.platform == "linux", "requires the Linux CI shell and curl")
+class MoldDownloadTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.root = Path(self.directory.name)
+        self.bin = self.root / "bin"
+        self.bin.mkdir()
+        self.downloads = self.root / "downloads"
+        self.downloads.mkdir()
+        self.statuses = [200]
+        self.requests = 0
+        self.started = threading.Event()
+        self.release = threading.Event()
+        owner = self
+
+        class Handler(http.server.BaseHTTPRequestHandler):
+            def do_GET(self):
+                status = owner.statuses[min(owner.requests, len(owner.statuses) - 1)]
+                owner.requests += 1
+                self.send_response(200 if status == "partial" else status)
+                self.send_header("Content-Length", "1000" if status == "partial" else "7")
+                self.end_headers()
+                self.wfile.write(b"archive")
+                self.wfile.flush()
+                owner.started.set()
+                if status == "partial":
+                    owner.release.wait(10)
+
+            def log_message(self, *_args):
+                pass
+
+        server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+        self.addCleanup(server.server_close)
+        self.addCleanup(server.shutdown)
+        self.addCleanup(self.release.set)
+        threading.Thread(target=server.serve_forever, daemon=True).start()
+        self.env = {**os.environ, "PATH": str(self.bin) + os.pathsep + os.environ["PATH"],
+                    "REAL_CURL": shutil.which("curl"), "TEST_URL": f"http://127.0.0.1:{server.server_port}/mold",
+                    "TEST_ROOT": str(self.root), "TMPDIR": str(self.downloads)}
+        self.tool("curl", """import os, sys
+args = sys.argv[1:]
+assert '--retry-all-errors' not in args
+for option, value in [('--retry', '2'), ('--connect-timeout', '15'), ('--max-time', '120'), ('--retry-max-time', '360')]:
+    assert args[args.index(option) + 1] == value
+assert 'https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz' in args
+args = [os.environ['TEST_URL'] if arg.startswith('https://') else arg for arg in args]
+os.execv(os.environ['REAL_CURL'], [os.environ['REAL_CURL'], *args])
+""")
+        self.tool("sudo", """import os, sys
+from pathlib import Path
+assert sys.argv[1:6] == ['tar', '-C', '/usr/local', '--strip-components=1', '-xzf']
+assert Path(sys.argv[-1]).read_bytes() == b'archive'
+with (Path(os.environ['TEST_ROOT']) / 'installed').open('a') as output: output.write('once\\n')
+""")
+        self.tool("mold", "print('mold 2.42.1')\n")
+
+    def tool(self, name, body):
+        path = self.bin / name
+        path.write_text(f"#!{sys.executable}\n" + body)
+        path.chmod(0o755)
+
+    def run_script(self):
+        result = subprocess.run(["bash", str(SCRIPT)], env=self.env, capture_output=True, text=True, timeout=20)
+        self.assertEqual(list(self.downloads.iterdir()), [])
+        return result
+
+    def test_transient_status_recovers_before_one_install(self):
+        self.statuses = [503, 503, 200]
+        result = self.run_script()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.requests, 3)
+        self.assertEqual((self.root / "installed").read_text(), "once\n")
+
+    def test_not_found_and_exhausted_retries_never_install(self):
+        for statuses, requests in [([404], 1), ([503], 3)]:
+            with self.subTest(statuses=statuses):
+                self.statuses = statuses
+                self.requests = 0
+                result = self.run_script()
+                self.assertNotEqual(result.returncode, 0)
+                self.assertEqual(self.requests, requests)
+                self.assertFalse((self.root / "installed").exists())
+
+    def test_cancellation_cleans_partial_file_without_install(self):
+        self.statuses = ["partial"]
+        process = subprocess.Popen(["bash", str(SCRIPT)], env=self.env, stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE, text=True, start_new_session=True)
+        try:
+            self.assertTrue(self.started.wait(5), "curl did not start")
+            os.killpg(process.pid, signal.SIGTERM)
+            process.communicate(timeout=5)
+            self.assertNotEqual(process.returncode, 0)
+            self.assertFalse((self.root / "installed").exists())
+            self.assertEqual(list(self.downloads.iterdir()), [])
+        finally:
+            if process.poll() is None:
+                os.killpg(process.pid, signal.SIGKILL)
+                process.communicate(timeout=5)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/scripts/ci-install-system-deps.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""Download Ubuntu dependencies with at most two mirror switches, then install offline."""
+
+import argparse
+import os
+import re
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+SOURCES = Path("/etc/apt/sources.list.d/ubuntu.sources")
+PRIMARY = "https://archive.ubuntu.com/ubuntu/"
+BACKUP = "https://mirrors.edge.kernel.org/ubuntu/"
+LAST_RESORT = "http://azure.archive.ubuntu.com/ubuntu/"
+MIRRORS = [("primary", PRIMARY), ("backup", BACKUP), ("last-resort", LAST_RESORT)]
+PACKAGES = ["libwebkit2gtk-4.1-dev", "libappindicator3-dev", "librsvg2-dev",
+            "patchelf", "libssl-dev", "libsecret-1-dev"]
+PACKAGE_GROUPS = {"gtk": PACKAGES, "packaging": ["zstd"],
+                  "duckdb": ["libdbus-1-dev", "pkg-config"]}
+ACQUIRE_OPTIONS = ["-o", "Acquire::Retries=0", "-o", "Acquire::http::Timeout=30",
+                   "-o", "Acquire::https::Timeout=30", "-o", "APT::Update::Error-Mode=any"]
+starting_process = False
+pending_cancel = None
+
+
+def rewrite_ubuntu_uris(source, mirror):
+    known = {"mirror+file:/etc/apt/apt-mirrors.txt", PRIMARY, PRIMARY.rstrip("/"),
+             BACKUP, BACKUP.rstrip("/"), LAST_RESORT, LAST_RESORT.rstrip("/")}
+    found = 0
+    in_uris = False
+    result = []
+
+    def replace(match):
+        nonlocal found
+        if match.group() in known:
+            found += 1
+            return mirror
+        return match.group()
+
+    for line in source.splitlines(keepends=True):
+        if line.lstrip().startswith("#"):
+            result.append(line)
+            continue
+        field = re.match(r"^([A-Za-z][A-Za-z0-9-]*):", line)
+        if field:
+            in_uris = field[1].lower() == "uris"
+        elif not line.strip() or not line[0].isspace():
+            in_uris = False
+        if in_uris:
+            prefix = field.end() if field else 0
+            line = line[:prefix] + re.sub(r"\S+", replace, line[prefix:])
+        result.append(line)
+    if not found:
+        raise ValueError("Ubuntu sources do not contain the expected runner mirror URI")
+    return "".join(result)
+
+
+def is_retryable_acquisition_failure(output, timed_out):
+    if re.search(r"signature|NO_PUBKEY|GPG error|certificate|not signed|Release file|"
+                 r"Hash Sum mismatch|unexpected size|Unable to locate package|unmet dependencies|"
+                 r"held broken packages|Could not get lock|Unable to acquire|Waiting for cache lock|"
+                 r"dpkg was interrupted|Permission denied|No space left|404\s", output, re.I):
+        return False
+    reasons = []
+    for line in output.splitlines():
+        fetch = re.match(r"[EW]: Failed to fetch \S+\s+(.+)", line)
+        if fetch:
+            reasons.append(fetch[1])
+        elif line.startswith("E:") and not line.startswith((
+            "E: Unable to fetch some archives,", "E: Some index files failed to download."
+        )):
+            return False
+    network = re.compile(r"Temporary failure resolving|Could not resolve|Could not connect|"
+                         r"Connection (?:failed|timed out|reset)|Network is unreachable|"
+                         r"Error reading from server|Remote end closed|"
+                         r"(?:^|\s)(?:429|500|502|503|504)(?:\s|$)", re.I)
+    if any(not network.search(reason) for reason in reasons):
+        return False
+    return timed_out or bool(reasons)
+
+
+class Cancelled(BaseException):
+    def __init__(self, signum):
+        self.signum = signum
+
+
+def cancel(signum, _frame):
+    global pending_cancel
+    # Finish child disposal and source restoration even if cancellation repeats.
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    if starting_process:
+        pending_cancel = signum
+        return
+    raise Cancelled(signum)
+
+
+def dispose(process):
+    try:
+        os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+    return process.communicate()[0]
+
+
+def run_apt(arguments, budget):
+    global starting_process, pending_cancel
+    started = time.monotonic()
+    process = None
+    timed_out = False
+    try:
+        starting_process = True
+        process = subprocess.Popen(
+            ["apt-get", *ACQUIRE_OPTIONS, *arguments], stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT, text=True, env={**os.environ, "LC_ALL": "C"},
+            start_new_session=True,
+        )
+        starting_process = False
+        if pending_cancel is not None:
+            signum, pending_cancel = pending_cancel, None
+            raise Cancelled(signum)
+        try:
+            output = process.communicate(timeout=budget)[0]
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            output = dispose(process)
+    except BaseException:
+        if process is not None:
+            dispose(process)
+        raise
+    finally:
+        starting_process = False
+    print(output, end="", flush=True)
+    print(f"APT finished elapsed={time.monotonic() - started:.1f}s "
+          f"exit={process.returncode} timed_out={timed_out}", flush=True)
+    return process.returncode, output, timed_out
+
+
+def install(source_path=SOURCES, group="gtk"):
+    packages = PACKAGE_GROUPS[group]
+    original = source_path.read_bytes()
+    source = original.decode("utf-8")
+    candidates = [(label, rewrite_ubuntu_uris(source, mirror)) for label, mirror in MIRRORS]
+    try:
+        for index, (label, candidate) in enumerate(candidates):
+            source_path.write_bytes(candidate.encode("utf-8"))
+            for phase, arguments, budget in [
+                ("index", ["update"], 60),
+                ("download", ["--download-only", "install", "-y", *packages], 120),
+            ]:
+                print(f"APT source={label} phase={phase} budget={budget}s", flush=True)
+                code, output, timed_out = run_apt(arguments, budget)
+                if code != 0 or timed_out:
+                    if index + 1 < len(candidates) and is_retryable_acquisition_failure(output, timed_out):
+                        reason = "acquisition deadline" if timed_out else "transient acquisition error"
+                        print(f"APT switching to next source {MIRRORS[index + 1][1]}: {reason}", flush=True)
+                        break
+                    raise RuntimeError(f"APT {label} {phase} failed (exit={code}, timed_out={timed_out})")
+            else:
+                break
+        print("APT phase=install budget=1200s downloads=disabled", flush=True)
+        code, _output, timed_out = run_apt(["--no-download", "install", "-y", *packages], 1200)
+        if code != 0 or timed_out:
+            raise RuntimeError(f"APT offline install failed (exit={code}, timed_out={timed_out})")
+    finally:
+        source_path.write_bytes(original)
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGINT, cancel)
+    signal.signal(signal.SIGTERM, cancel)
+    try:
+        parser = argparse.ArgumentParser(description=__doc__)
+        parser.add_argument("--group", choices=PACKAGE_GROUPS, default="gtk")
+        install(group=parser.parse_args().group)
+    except Cancelled as error:
+        print("APT cancelled; no mirror retry", file=sys.stderr)
+        sys.exit(128 + error.signum)
+    except Exception as error:
+        print(f"::error::{error}", file=sys.stderr)
+        sys.exit(1)

.github/scripts/ci-install-system-deps.test.py

@@ -0,0 +1,282 @@
+import importlib.util
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+import textwrap
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+SCRIPT = Path(__file__).with_name("ci-install-system-deps.py")
+spec = importlib.util.spec_from_file_location("deps", SCRIPT)
+deps = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(deps)
+
+SOURCE = """Types: deb
+URIs: mirror+file:/etc/apt/apt-mirrors.txt
+Suites: noble noble-updates noble-backports
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+
+Types: deb
+URIs:
+ mirror+file:/etc/apt/apt-mirrors.txt
+Suites: noble-security
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+
+Types: deb
+URIs: https://security.ubuntu.com/ubuntu/
+Suites: noble-security
+Components: main
+
+Types: deb
+URIs: https://example.test/apt
+Suites: stable
+Components: main
+"""
+OK = (0, "", False)
+TIMEOUT = (-9, "", True)
+NETWORK = (100, "E: Failed to fetch http://azure.archive.ubuntu.com/a.deb Connection timed out\n"
+           "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n", False)
+
+
+class InstallerTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        self.source = Path(self.directory.name) / "ubuntu.sources"
+        self.source.write_bytes(SOURCE.encode())
+        self.calls = []
+
+    def execute(self, results, group="gtk"):
+        responses = iter(results)
+
+        def run(arguments, budget):
+            self.calls.append((arguments, budget, self.source.read_text()))
+            return next(responses)
+
+        with patch.object(deps, "run_apt", side_effect=run):
+            try:
+                deps.install(self.source, group)
+            finally:
+                self.assertEqual(self.source.read_bytes(), SOURCE.encode())
+
+    def test_success_downloads_before_one_offline_install(self):
+        self.execute([OK, OK, OK])
+        self.assertEqual([call[1] for call in self.calls], [60, 120, 1200])
+        self.assertEqual(self.calls[1][0], ["--download-only", "install", "-y", *deps.PACKAGES])
+        self.assertEqual(self.calls[2][0], ["--no-download", "install", "-y", *deps.PACKAGES])
+        self.assertTrue(all(deps.BACKUP not in call[2] for call in self.calls))
+
+    def test_fixed_package_groups_preserve_each_caller_list(self):
+        for group, packages in {"packaging": ["zstd"],
+                                "duckdb": ["libdbus-1-dev", "pkg-config"]}.items():
+            with self.subTest(group=group):
+                self.calls.clear()
+                self.execute([OK, OK, OK], group)
+                self.assertEqual(self.calls[1][0], ["--download-only", "install", "-y", *packages])
+                self.assertEqual(self.calls[2][0], ["--no-download", "install", "-y", *packages])
+
+    def test_official_primary_and_known_runner_sources(self):
+        self.assertEqual(deps.PRIMARY, "https://archive.ubuntu.com/ubuntu/")
+        for uri in [deps.PRIMARY, deps.BACKUP, deps.LAST_RESORT, "mirror+file:/etc/apt/apt-mirrors.txt"]:
+            with self.subTest(uri=uri):
+                source = SOURCE.replace("mirror+file:/etc/apt/apt-mirrors.txt", uri)
+                rewritten = deps.rewrite_ubuntu_uris(source, deps.PRIMARY)
+                self.assertIn("URIs: https://archive.ubuntu.com/ubuntu/", rewritten)
+                self.assertIn("URIs: https://security.ubuntu.com/ubuntu/", rewritten)
+                self.assertIn("Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg", rewritten)
+
+    def test_timeout_or_network_failure_switches_once_and_reuses_cache(self):
+        for failure in [TIMEOUT, NETWORK]:
+            with self.subTest(failure=failure):
+                self.calls.clear()
+                self.execute([OK, failure, OK, OK, OK])
+                self.assertEqual([call[1] for call in self.calls], [60, 120, 60, 120, 1200])
+                self.assertIn(deps.PRIMARY, self.calls[1][2])
+                self.assertNotIn(deps.PRIMARY, self.calls[2][2])
+                self.assertIn(deps.BACKUP, self.calls[2][2])
+                self.assertFalse(any("clean" in call[0] or "--fix-missing" in call[0] for call in self.calls))
+                self.assertTrue(all(not any("Dir::Cache" in arg for arg in call[0]) for call in self.calls))
+
+    def test_network_index_failure_refreshes_backup_before_downloading(self):
+        self.execute([NETWORK, OK, OK, OK])
+        self.assertEqual([call[0][0] for call in self.calls], ["update", "update", "--download-only", "--no-download"])
+
+    def test_all_three_sources_fail_without_installing_partial_downloads(self):
+        with self.assertRaises(RuntimeError):
+            self.execute([OK, TIMEOUT, OK, NETWORK, OK, NETWORK])
+        self.assertEqual(len(self.calls), 6)
+        self.assertFalse(any("--no-download" in call[0] for call in self.calls))
+
+    def test_second_switch_is_last_resort_and_never_loops(self):
+        self.execute([OK, TIMEOUT, OK, NETWORK, OK, OK, OK])
+        self.assertEqual(len(self.calls), 7)
+        self.assertIn(deps.PRIMARY, self.calls[1][2])
+        self.assertIn(deps.BACKUP, self.calls[3][2])
+        self.assertIn(deps.LAST_RESORT, self.calls[5][2])
+        self.assertEqual(self.calls[6][0][0], "--no-download")
+
+    def test_deterministic_and_unknown_errors_never_switch(self):
+        for output in ["E: Unable to locate package missing", "W: GPG error: invalid signatures",
+                       "E: Failed to fetch https://archive.ubuntu.com/a.deb Certificate verification failed",
+                       "E: Failed to fetch http://azure.archive.ubuntu.com/a.deb 404 Not Found",
+                       "E: Failed to fetch http://azure.archive.ubuntu.com/a.deb Hash Sum mismatch",
+                       "E: Could not get lock /var/lib/dpkg/lock", "E: No space left on device",
+                       "E: Unknown error", NETWORK[1] + "E: unmet dependencies"]:
+            for timed_out in [False, True]:
+                with self.subTest(output=output, timed_out=timed_out):
+                    self.calls.clear()
+                    with self.assertRaises(RuntimeError):
+                        self.execute([OK, (100, output, timed_out)])
+                    self.assertEqual(len(self.calls), 2)
+        self.assertFalse(deps.is_retryable_acquisition_failure("", False))
+        self.assertFalse(deps.is_retryable_acquisition_failure(
+            NETWORK[1] + "E: Failed to fetch https://example.test/a.deb Unknown error", False))
+
+    def test_http_transient_errors_are_allowed_but_not_unknown_fetch_errors(self):
+        for status in [429, 500, 502, 503, 504]:
+            self.assertTrue(deps.is_retryable_acquisition_failure(
+                f"E: Failed to fetch https://archive.ubuntu.com/a.deb {status} Server error", False))
+        self.assertFalse(deps.is_retryable_acquisition_failure(
+            "E: Failed to fetch https://archive.ubuntu.com/a.deb Something went wrong", False))
+
+    def test_install_failure_is_terminal_without_retry(self):
+        for failure in [TIMEOUT, NETWORK, (100, "E: dpkg failed", False)]:
+            self.calls.clear()
+            with self.assertRaises(RuntimeError):
+                self.execute([OK, OK, failure])
+            self.assertEqual(len(self.calls), 3)
+
+    def test_source_rewrite_preserves_all_non_uri_fields_and_security(self):
+        rewritten = deps.rewrite_ubuntu_uris(SOURCE, deps.BACKUP)
+        self.assertEqual(rewritten, SOURCE.replace("mirror+file:/etc/apt/apt-mirrors.txt", deps.BACKUP))
+        self.assertEqual(deps.rewrite_ubuntu_uris(SOURCE.replace("mirror+file:/etc/apt/apt-mirrors.txt", deps.PRIMARY), deps.BACKUP), rewritten)
+        self.source.write_text("Types: deb\nURIs: https://unknown.test/ubuntu\nSuites: noble\n")
+        original = self.source.read_bytes()
+        with self.assertRaises(ValueError), patch.object(deps, "run_apt") as run:
+            deps.install(self.source)
+        run.assert_not_called()
+        self.assertEqual(self.source.read_bytes(), original)
+
+    def test_run_apt_uses_an_owned_process_group_and_c_locale(self):
+        with patch.object(deps.subprocess, "Popen") as popen:
+            popen.return_value.communicate.return_value = ("downloaded\n", None)
+            popen.return_value.returncode = 0
+            self.assertEqual(deps.run_apt(["--download-only", "install", "-y", *deps.PACKAGES], 120), (0, "downloaded\n", False))
+        self.assertTrue(popen.call_args.kwargs["start_new_session"])
+        self.assertEqual(popen.call_args.kwargs["env"]["LC_ALL"], "C")
+        self.assertIn("Acquire::Retries=0", popen.call_args.args[0])
+
+    @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups")
+    def test_cancellation_during_spawn_reaps_before_restoring_sources(self):
+        events = []
+        write = Path.write_bytes
+
+        def record_write(path, content):
+            if content == SOURCE.encode():
+                events.append("restore")
+            return write(path, content)
+
+        with patch.object(deps.subprocess, "Popen") as popen, \
+                patch.object(deps.signal, "signal"), \
+                patch.object(deps.os, "killpg", side_effect=lambda *_: events.append("kill")), \
+                patch.object(Path, "write_bytes", autospec=True, side_effect=record_write):
+            process = popen.return_value
+            process.communicate.side_effect = lambda: (events.append("reap") or "", None)
+
+            def spawn(*_args, **_kwargs):
+                events.append("spawn")
+                deps.cancel(signal.SIGTERM, None)
+                return process
+
+            popen.side_effect = spawn
+            with self.assertRaises(deps.Cancelled):
+                deps.install(self.source)
+            popen.assert_called_once()
+        self.assertEqual(events, ["spawn", "kill", "reap", "restore"])
+        self.assertEqual(self.source.read_bytes(), SOURCE.encode())
+
+    def run_child_case(self, signum=0):
+        pid_file = Path(self.directory.name) / f"child-{signum}.pid"
+        payload = textwrap.dedent("""
+            import os, subprocess, sys, time
+            child = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])
+            open(sys.argv[1], 'w').write(f'{os.getpid()} {child.pid}')
+            time.sleep(60)
+        """)
+        harness = textwrap.dedent("""
+            import importlib.util, signal, subprocess, sys
+            from pathlib import Path
+            spec = importlib.util.spec_from_file_location('deps', sys.argv[1])
+            deps = importlib.util.module_from_spec(spec); spec.loader.exec_module(deps)
+            deps.signal.signal(signal.SIGINT, deps.cancel)
+            deps.signal.signal(signal.SIGTERM, deps.cancel)
+            original_popen = subprocess.Popen
+            deps.subprocess.Popen = lambda *a, **kw: original_popen([sys.executable, '-c', sys.argv[3], sys.argv[4]], **kw)
+            original_run = deps.run_apt
+            deps.run_apt = lambda args, budget: original_run(args, 3)
+            try:
+                deps.install(Path(sys.argv[2]))
+            except deps.Cancelled as error:
+                sys.exit(128 + error.signum)
+            except RuntimeError:
+                sys.exit(1)
+        """)
+        process = subprocess.Popen([sys.executable, "-c", harness, str(SCRIPT), str(self.source), payload, str(pid_file)],
+                                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+        def cleanup():
+            if pid_file.exists() and pid_file.read_text():
+                try:
+                    os.killpg(int(pid_file.read_text().split()[0]), signal.SIGKILL)
+                except ProcessLookupError:
+                    pass
+            if process.poll() is None:
+                process.kill()
+            process.communicate(timeout=5)
+
+        self.addCleanup(cleanup)
+        deadline = time.monotonic() + 10
+        while not pid_file.exists() and process.poll() is None and time.monotonic() < deadline:
+            time.sleep(0.02)
+        self.assertTrue(pid_file.exists(), "fake apt did not start")
+        if signum:
+            process.send_signal(signum)
+        output = process.communicate(timeout=15)[0]
+        self.assertEqual(process.returncode, 128 + signum if signum else 1, output)
+        self.assertEqual(self.source.read_bytes(), SOURCE.encode())
+        self.assertNotIn("phase=install", output)
+        child = Path(f"/proc/{pid_file.read_text().split()[1]}/stat")
+
+        def child_running():
+            try:
+                return child.read_text().split()[2] != "Z"
+            except FileNotFoundError:
+                return False
+
+        deadline = time.monotonic() + 5
+        while child_running() and time.monotonic() < deadline:
+            time.sleep(0.02)
+        self.assertFalse(child_running(), "descendant still running")
+        return output
+
+    @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups and /proc")
+    def test_timeout_disposes_descendants_before_backup(self):
+        output = self.run_child_case()
+        self.assertIn("timed_out=True", output)
+        self.assertEqual(output.count("switching to next source"), 2)
+
+    @unittest.skipUnless(sys.platform == "linux", "requires Linux process groups and /proc")
+    def test_cancellation_disposes_descendants_without_backup_and_restores_sources(self):
+        for signum in [signal.SIGINT, signal.SIGTERM]:
+            with self.subTest(signum=signum):
+                output = self.run_child_case(signum)
+                self.assertNotIn("switching to next source", output)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/scripts/ci-plan.test.mjs

@@ -232,7 +232,21 @@ test("code generation and test-only dependencies participate in impact analysis"
   assert.ok(plan(["crates/dbx-formats/src/lib.rs"], { metadata: fixture }).affected_packages.includes("dbx-types"));
 });
 
-for (const file of ["Cargo.toml", "Cargo.lock", ".cargo/config.toml", "rust-toolchain.toml", "vendor/wry/src/lib.rs", "crates/dbx-sql/Cargo.toml", ".github/workflows/ci.yml", ".github/scripts/ci-plan.mjs", ".github/scripts/ci-gate.test.mjs"]) {
+for (const file of [
+  "Cargo.toml",
+  "Cargo.lock",
+  ".cargo/config.toml",
+  "rust-toolchain.toml",
+  "vendor/wry/src/lib.rs",
+  "crates/dbx-sql/Cargo.toml",
+  ".github/workflows/ci.yml",
+  ".github/scripts/ci-plan.mjs",
+  ".github/scripts/ci-gate.test.mjs",
+  ".github/scripts/ci-gradle.py",
+  ".github/scripts/ci-gradle.init.gradle",
+  ".github/scripts/ci-install-system-deps.py",
+  ".github/scripts/ci-install-mold.sh",
+]) {
   test(`shared input ${file} retains full workspace and Agent coverage`, () => {
     const result = plan([file]);
     assert.deepEqual(groups(result), ["workspace"]);

.github/scripts/ci-workflow.test.mjs

@@ -279,15 +279,36 @@ test("Rust build timings remain available after failures without waiting for can
   assert.ok(job("agent-rust").includes('--manifest-path "drivers/$DRIVER/Cargo.toml" --locked --no-fail-fast --timings'));
 });
 
-test("Linux Rust dependency installs bound apt mirror stalls", () => {
-  for (const name of ["packages", "rust-fmt-clippy", "rust-test", "agent-rust"]) {
-    const content = job(name);
-    const deadlines = [...content.matchAll(/sudo timeout --kill-after=30s (\d+)m apt-get/g)].map((match) => Number(match[1]));
-    assert.deepEqual(deadlines, [10, name === "agent-rust" ? 10 : 20]);
-    for (const option of ["Acquire::Retries=3", "Acquire::http::Timeout=30", "Acquire::https::Timeout=30"]) {
-      assert.ok(content.includes(option));
-    }
+test("Linux dependency installs share bounded acquisition before offline installation", () => {
+  for (const name of ["packages", "rust-fmt-clippy", "rust-test"]) {
+    assert.ok(job(name).includes("sudo python3 .github/scripts/ci-install-system-deps.py"));
+    assert.ok(!job(name).includes("apt-get"));
   }
+  for (const name of ["agent-checks", "agent-java"]) {
+    assert.ok(job(name).includes("sudo python3 ../.github/scripts/ci-install-system-deps.py --group packaging"));
+  }
+  assert.ok(job("agent-rust").includes("sudo python3 ../.github/scripts/ci-install-system-deps.py --group duckdb"));
+  assert.ok(job("github-scripts").includes("python3 .github/scripts/ci-install-system-deps.test.py"));
+});
+
+test("Java CI acquires before original commands without retrying packaging or tests", () => {
+  assert.ok(job("agent-java").includes("python3 ../.github/scripts/ci-gradle.py agents -- ./gradlew test shadowJar --continue"));
+  for (const name of ["jdbc", "offline-jdbc-release"]) {
+    assert.ok(job(name).includes("python3 .github/scripts/ci-gradle.py jdbc -- ./package.sh"));
+  }
+  assert.ok(job("github-scripts").includes("python3 .github/scripts/ci-gradle.test.py"));
+});
+
+test("mold downloads are isolated from extraction in CI and keepalive", () => {
+  for (const name of ["packages", "rust-test"]) {
+    assert.ok(job(name).includes("run: bash .github/scripts/ci-install-mold.sh"));
+  }
+  assert.ok(job("agent-rust").includes("run: bash ../.github/scripts/ci-install-mold.sh"));
+  const keepalive = readFileSync(new URL("../workflows/cache-keepalive.yml", import.meta.url), "utf8");
+  assert.ok(keepalive.includes("run: bash .github/scripts/ci-install-mold.sh"));
+  assert.ok(keepalive.includes("sudo python3 .github/scripts/ci-install-system-deps.py"));
+  assert.ok(job("github-scripts").includes("python3 .github/scripts/ci-install-mold.test.py"));
+  assert.ok(job("github-scripts").includes("./.github/scripts/prepare-webview2-win7-runtime.test.ps1"));
 });
 
 for (const name of ["rust-test", "agent-integration"]) {
@@ -297,7 +318,14 @@ for (const name of ["rust-test", "agent-integration"]) {
       const block = step.slice(step.indexOf("          timeout 30s")).replace(/^          /gm, "");
       const directory = mkdtempSync(path.join(tmpdir(), "dbx-docker-ready-"));
       try {
-        const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", `
+        const result = spawnSync(
+          "bash",
+          [
+            "-e",
+            "-o",
+            "pipefail",
+            "-c",
+            `
           sudo() { "$@"; }
           systemctl() { printf 'restart\\n' >> "$STATE/restarts"; }
           timeout() { shift; "$@"; }
@@ -314,7 +342,10 @@ for (const name of ["rust-test", "agent-integration"]) {
           }
           ${block}
           printf 'verified\\n'
-        `], { env: { ...process.env, STATE: directory.replaceAll("\\", "/"), SCENARIO: scenario }, encoding: "utf8", timeout: 15000 });
+        `,
+          ],
+          { env: { ...process.env, STATE: directory.replaceAll("\\", "/"), SCENARIO: scenario }, encoding: "utf8", timeout: 15000 },
+        );
         assert.ifError(result.error);
         assert.equal(readFileSync(path.join(directory, "restarts"), "utf8"), "restart\n");
         const count = Number(readFileSync(path.join(directory, "count"), "utf8"));
@@ -389,7 +420,8 @@ test("Windows compatibility jobs cache Rust compilation without wrapping C or C+
   const win7 = job("windows-win7-bundle");
   assert.ok(win7.includes("needs.changes.outputs.windows_win7_bundle == 'true'"));
   assert.doesNotMatch(win7, /x86_64-pc-windows-msvc|Setup Rust for standard Windows/);
-  for (const setting of ["RUSTC_WRAPPER: sccache", "SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}", "SCCACHE_GHA_VERSION: win7-webview2-1.0.902.49-v1", 'SCCACHE_IDLE_TIMEOUT: "0"', 'CARGO_PROFILE_RELEASE_LTO: "off"', 'CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "8"']) assert.ok(win7.includes(setting));
+  for (const setting of ["RUSTC_WRAPPER: sccache", "SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}", "SCCACHE_GHA_VERSION: win7-webview2-1.0.902.49-v1", 'SCCACHE_IDLE_TIMEOUT: "0"', 'CARGO_PROFILE_RELEASE_LTO: "off"', 'CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "8"'])
+    assert.ok(win7.includes(setting));
   assert.ok(win7.includes("fc920bf0ec8de6ee65d409111f7ec508035751ba"));
   assert.ok(win7.includes('version: "v0.16.0"'));
   assert.ok(win7.includes("sccache --show-stats"));
@@ -432,7 +464,10 @@ test("Win7 TLS artifacts come from main and retain a build fallback", () => {
   const win7 = job("windows-win7-bundle");
   assert.ok(win7.includes("actions: read"));
   assert.doesNotMatch(win7, /uses: actions\/cache(?:\/restore|\/save)?@v4/);
-  for (const [name, id] of [["AWS-LC", "aws-lc-artifact"], ["OpenSSL", "openssl-artifact"]]) {
+  for (const [name, id] of [
+    ["AWS-LC", "aws-lc-artifact"],
+    ["OpenSSL", "openssl-artifact"],
+  ]) {
     assert.ok(win7.includes(`if: steps.${id}.outputs.run_id != ''`));
     assert.ok(win7.includes(`if: steps.${id}.outputs.run_id == ''`));
     assert.ok(win7.includes(`if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.${id}.outputs.run_id == ''`));

.github/scripts/prepare-webview2-win7-runtime.ps1

@@ -7,6 +7,64 @@ param(
 Set-StrictMode -Version Latest
 $ErrorActionPreference = "Stop"
 
+function Get-WebView2Archive {
+  param([string]$Uri, [string]$OutFile)
+
+  for ($attempt = 0; $attempt -lt 3; $attempt++) {
+    try {
+      Invoke-WebRequest -Uri $Uri -OutFile $OutFile -TimeoutSec 120
+      return
+    } catch {
+      $errorObject = $_.Exception
+      $retryable = $false
+      if ($errorObject.PSObject.Properties["Response"] -and $null -ne $errorObject.Response) {
+        $retryable = [int]$errorObject.Response.StatusCode -in @(408, 429, 500, 502, 503, 504)
+      } elseif ($errorObject -is [System.Net.Http.HttpRequestException] -and $null -ne $errorObject.StatusCode) {
+        $retryable = [int]$errorObject.StatusCode -in @(408, 429, 500, 502, 503, 504)
+      } elseif ($errorObject -is [System.Net.Http.HttpRequestException]) {
+        $networkCause = $false
+        $unsafeCause = $false
+        for ($cause = $errorObject.InnerException; $null -ne $cause; $cause = $cause.InnerException) {
+          if ($cause -is [System.Security.Authentication.AuthenticationException] -or
+              $cause -is [System.OperationCanceledException]) {
+            $unsafeCause = $true
+          } elseif ($cause -is [System.Net.Sockets.SocketException]) {
+            $networkCause = $networkCause -or $cause.SocketErrorCode -in @(
+              [System.Net.Sockets.SocketError]::ConnectionRefused,
+              [System.Net.Sockets.SocketError]::ConnectionReset,
+              [System.Net.Sockets.SocketError]::ConnectionAborted,
+              [System.Net.Sockets.SocketError]::TimedOut,
+              [System.Net.Sockets.SocketError]::HostNotFound,
+              [System.Net.Sockets.SocketError]::TryAgain,
+              [System.Net.Sockets.SocketError]::NetworkUnreachable,
+              [System.Net.Sockets.SocketError]::HostUnreachable,
+              [System.Net.Sockets.SocketError]::NetworkDown
+            )
+          }
+        }
+        $retryable = $networkCause -and !$unsafeCause
+      } elseif ($errorObject -is [System.Net.WebException]) {
+        $retryable = $errorObject.Status -in @(
+          [System.Net.WebExceptionStatus]::Timeout,
+          [System.Net.WebExceptionStatus]::ConnectFailure,
+          [System.Net.WebExceptionStatus]::NameResolutionFailure,
+          [System.Net.WebExceptionStatus]::ConnectionClosed,
+          [System.Net.WebExceptionStatus]::ReceiveFailure,
+          [System.Net.WebExceptionStatus]::SendFailure
+        )
+      } elseif ($errorObject -is [System.Threading.Tasks.TaskCanceledException]) {
+        $retryable = $true
+      }
+      if (!$retryable -or $attempt -eq 2) { throw }
+      Write-Host "WebView2 archive acquisition failed temporarily; retry $($attempt + 2)/3."
+      if (Test-Path -LiteralPath $OutFile) {
+        Remove-Item -LiteralPath $OutFile -Force
+      }
+      Start-Sleep -Seconds ([math]::Pow(2, $attempt + 1))
+    }
+  }
+}
+
 $runtimeVersion = "109.0.1518.78"
 $runtimeFolderName = "Microsoft.WebView2.FixedVersionRuntime.$runtimeVersion.x64"
 $archiveName = "$runtimeFolderName.cab"
@@ -32,7 +90,7 @@ if (Test-Path $archivePath) {
 
 if (!(Test-Path $archivePath)) {
   Write-Host "Downloading WebView2 fixed runtime $runtimeVersion for Windows 7..."
-  Invoke-WebRequest -Uri $runtimeUrl -OutFile $archivePath
+  Get-WebView2Archive -Uri $runtimeUrl -OutFile $archivePath
 }
 
 $actualHash = (Get-FileHash -LiteralPath $archivePath -Algorithm SHA256).Hash.ToLowerInvariant()

.github/scripts/prepare-webview2-win7-runtime.test.ps1

@@ -0,0 +1,91 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+$source = Join-Path $PSScriptRoot "prepare-webview2-win7-runtime.ps1"
+$tokens = $null
+$errors = $null
+$ast = [System.Management.Automation.Language.Parser]::ParseFile($source, [ref]$tokens, [ref]$errors)
+if ($errors.Count) { throw $errors[0] }
+$definition = $ast.Find({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq "Get-WebView2Archive" }, $true)
+. ([scriptblock]::Create($definition.Extent.Text))
+
+$directory = Join-Path ([IO.Path]::GetTempPath()) "dbx-ci-webview-$([guid]::NewGuid())"
+New-Item -ItemType Directory -Path $directory | Out-Null
+$outFile = Join-Path $directory "archive.cab"
+
+function Invoke-WebRequest {
+  param($Uri, $OutFile, $TimeoutSec)
+  if ($TimeoutSec -ne 120) { throw "Download timeout changed" }
+  if ($global:dbxWebviewTestcalls -gt 0 -and (Test-Path -LiteralPath $OutFile)) { throw "Partial download was reused" }
+  $result = $global:dbxWebviewTestresponses[$global:dbxWebviewTestcalls]
+  $global:dbxWebviewTestcalls++
+  [IO.File]::WriteAllText($OutFile, "partial")
+  switch ($result) {
+    "ok" { [IO.File]::WriteAllText($OutFile, "complete"); return }
+    "timeout" { throw [Net.WebException]::new("timeout", [Net.WebExceptionStatus]::Timeout) }
+    "refused" { throw [Net.Http.HttpRequestException]::new("connection refused", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::ConnectionRefused)) }
+    "dns" { throw [Net.Http.HttpRequestException]::new("name lookup failed", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::HostNotFound)) }
+    "socket-denied" { throw [Net.Http.HttpRequestException]::new("access denied", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::AccessDenied)) }
+    "cancel-socket" { throw [Net.Http.HttpRequestException]::new("cancelled", [OperationCanceledException]::new("cancelled", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::ConnectionReset))) }
+    "tls-socket" { throw [Net.Http.HttpRequestException]::new("TLS failed", [Security.Authentication.AuthenticationException]::new("certificate failure", [Net.Sockets.SocketException]::new([int][Net.Sockets.SocketError]::ConnectionReset))) }
+    "tls" { throw [Net.WebException]::new("certificate failure", [Net.WebExceptionStatus]::TrustFailure) }
+    "cancel" { throw [OperationCanceledException]::new("cancelled") }
+    "unknown" { throw "Unknown failure" }
+    default { throw [Net.Http.HttpRequestException]::new("HTTP failure", $null, [Net.HttpStatusCode][int]$result) }
+  }
+}
+function Start-Sleep { param($Seconds) $global:dbxWebviewTestdelays += $Seconds }
+
+try {
+  foreach ($case in @(
+    @{ Responses = @("ok"); Count = 1; Delays = @(); Success = $true },
+    @{ Responses = @("503", "ok"); Count = 2; Delays = @(2); Success = $true },
+    @{ Responses = @("timeout", "429", "ok"); Count = 3; Delays = @(2, 4); Success = $true },
+    @{ Responses = @("refused", "ok"); Count = 2; Delays = @(2); Success = $true },
+    @{ Responses = @("dns", "ok"); Count = 2; Delays = @(2); Success = $true },
+    @{ Responses = @("socket-denied"); Count = 1; Delays = @(); Success = $false },
+    @{ Responses = @("cancel-socket"); Count = 1; Delays = @(); Success = $false },
+    @{ Responses = @("tls-socket"); Count = 1; Delays = @(); Success = $false },
+    @{ Responses = @("503", "503", "503"); Count = 3; Delays = @(2, 4); Success = $false },
+    @{ Responses = @("404"); Count = 1; Delays = @(); Success = $false },
+    @{ Responses = @("403"); Count = 1; Delays = @(); Success = $false },
+    @{ Responses = @("tls"); Count = 1; Delays = @(); Success = $false },
+    @{ Responses = @("cancel"); Count = 1; Delays = @(); Success = $false },
+    @{ Responses = @("unknown"); Count = 1; Delays = @(); Success = $false }
+  )) {
+    Remove-Item -LiteralPath $outFile -Force -ErrorAction SilentlyContinue
+    $global:dbxWebviewTestresponses = $case.Responses
+    $global:dbxWebviewTestcalls = 0
+    $global:dbxWebviewTestdelays = @()
+    $success = $true
+    try { Get-WebView2Archive -Uri "https://example.test/archive.cab" -OutFile $outFile } catch { $success = $false }
+    if ($success -ne $case.Success -or $global:dbxWebviewTestcalls -ne $case.Count -or
+        ($global:dbxWebviewTestdelays -join ",") -ne ($case.Delays -join ",")) {
+      throw "Unexpected acquisition result for $($case.Responses -join ','): success=$success calls=$global:dbxWebviewTestcalls delays=$global:dbxWebviewTestdelays"
+    }
+  }
+
+  # Exercise the actual entry point: downloaded corrupt bytes must not be retried or expanded.
+  Remove-Item -LiteralPath $outFile -Force
+  $global:dbxWebviewTestresponses = @("ok")
+  $global:dbxWebviewTestcalls = 0
+  $global:dbxWebviewTestdelays = @()
+  $failed = $false
+  try { & $source -RuntimeDirectory (Join-Path $directory "runtime") -DownloadDirectory $directory } catch {
+    $failed = $_.Exception.Message -like "*SHA-256 mismatch*"
+  }
+  if (!$failed -or $global:dbxWebviewTestcalls -ne 1) { throw "Corrupt archive did not fail once before extraction" }
+
+  function Get-FileHash { param($LiteralPath, $Algorithm) [pscustomobject]@{ Hash = "7622281cf83de1a35e3a471f432f7a897d65f0a7d3975df08512b7b253dd45c7" } }
+  function Get-AuthenticodeSignature { param($LiteralPath) [pscustomobject]@{ Status = [System.Management.Automation.SignatureStatus]::NotSigned; SignerCertificate = $null } }
+  $failed = $false
+  $global:dbxWebviewTestcalls = 0
+  try { & $source -RuntimeDirectory (Join-Path $directory "runtime") -DownloadDirectory $directory } catch {
+    $failed = $_.Exception.Message -like "*valid Microsoft signature*"
+  }
+  if (!$failed -or $global:dbxWebviewTestcalls -ne 0) { throw "Invalid signature did not fail before extraction without re-download" }
+  Write-Host "WebView2 acquisition and integrity tests passed."
+} finally {
+  $resolved = [IO.Path]::GetFullPath($directory)
+  if ((Split-Path -Parent $resolved) -ne ([IO.Path]::GetTempPath()).TrimEnd('\', '/')) { throw "Unexpected test cleanup path" }
+  Remove-Item -LiteralPath $resolved -Recurse -Force
+}

.github/workflows/cache-keepalive.yml

@@ -41,25 +41,13 @@ jobs:
       - uses: actions/checkout@v7
 
       - name: Install system dependencies
-        run: |
-          sudo timeout --kill-after=30s 10m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            update
-          sudo timeout --kill-after=30s 10m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+        run: sudo python3 .github/scripts/ci-install-system-deps.py
 
       # Pin mold because runner image updates can change bundled C++ links.
       # See the ci.yml rust-test note for the pinned upstream build.
       - name: Install mold linker
-        run: |
-          curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
-            | sudo tar -C /usr/local --strip-components=1 -xz
-          mold --version
+        timeout-minutes: 8
+        run: bash .github/scripts/ci-install-mold.sh
 
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@1.97.1

.github/workflows/ci.yml

@@ -174,6 +174,19 @@ jobs:
       - name: Docker mirror config tests
         run: python3 .github/scripts/configure-docker-mirror.test.py
 
+      - name: APT acquisition tests
+        run: python3 .github/scripts/ci-install-system-deps.test.py
+
+      - name: Gradle acquisition tests
+        run: python3 .github/scripts/ci-gradle.test.py
+
+      - name: Mold acquisition tests
+        run: python3 .github/scripts/ci-install-mold.test.py
+
+      - name: WebView2 acquisition tests
+        shell: pwsh
+        run: ./.github/scripts/prepare-webview2-win7-runtime.test.ps1
+
       - name: PowerShell MCP installer tests
         shell: pwsh
         run: ./.github/scripts/install-mcp-powershell.test.ps1
@@ -207,25 +220,13 @@ jobs:
           cache: pnpm
 
       - name: Install system dependencies
-        run: |
-          sudo timeout --kill-after=30s 10m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            update
-          sudo timeout --kill-after=30s 20m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+        run: sudo python3 .github/scripts/ci-install-system-deps.py
 
       # Pin mold because runner image updates can change bundled DuckDB C++ links.
       # The pinned build prevents duckdb_create_config runtime failures.
       - name: Install mold linker
-        run: |
-          curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
-            | sudo tar -C /usr/local --strip-components=1 -xz
-          mold --version
+        timeout-minutes: 8
+        run: bash .github/scripts/ci-install-mold.sh
 
       - name: Install frontend dependencies
         run: pnpm install --frozen-lockfile
@@ -906,17 +907,7 @@ jobs:
       - uses: actions/checkout@v7
 
       - name: Install system dependencies
-        run: |
-          sudo timeout --kill-after=30s 10m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            update
-          sudo timeout --kill-after=30s 20m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+        run: sudo python3 .github/scripts/ci-install-system-deps.py
 
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@1.97.1
@@ -1026,26 +1017,14 @@ jobs:
           done
 
       - name: Install system dependencies
-        run: |
-          sudo timeout --kill-after=30s 10m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            update
-          sudo timeout --kill-after=30s 20m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            install -y libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev libsecret-1-dev
+        run: sudo python3 .github/scripts/ci-install-system-deps.py
         if: matrix.group != 'foundation'
 
       # Pin mold because runner image updates can change bundled C++ links.
       # See the packages job note for the pinned upstream build.
       - name: Install mold linker
-        run: |
-          curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
-            | sudo tar -C /usr/local --strip-components=1 -xz
-          mold --version
+        timeout-minutes: 8
+        run: bash .github/scripts/ci-install-mold.sh
 
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@1.97.1
@@ -1174,7 +1153,7 @@ jobs:
           node .github/scripts/check-jdbc-plugin-version.mjs "$BASE_SHA" HEAD
 
       - name: JDBC plugin package check
-        run: ./plugins/jdbc/package.sh
+        run: python3 .github/scripts/ci-gradle.py jdbc -- ./package.sh
 
       - name: JDBC response recovery tests
         run: bash ./plugins/jdbc/gradlew -p plugins/jdbc --no-daemon test --tests app.dbx.jdbc.LargeResponseRecoveryTest --tests app.dbx.jdbc.StatementCancelTest --tests app.dbx.jdbc.DeferredLobValuesTest
@@ -1203,7 +1182,7 @@ jobs:
         env:
           DBX_OFFLINE_JDBC_MAVEN_CACHE: ~/.cache/dbx-offline-jdbc-maven
         run: |
-          ./plugins/jdbc/package.sh
+          python3 .github/scripts/ci-gradle.py jdbc -- ./package.sh
           JDBC_VERSION="$(sed -nE "s/^version[[:space:]]*=[[:space:]]*'([^']+)'.*/\1/p" plugins/jdbc/build.gradle | head -n 1)"
           node agents/scripts/build_offline_jdbc_payload.mjs \
             release \
@@ -1330,8 +1309,10 @@ jobs:
               - '.github/workflows/ci.yml'
             jdbc:
               - 'plugins/jdbc/**'
+              - '.github/scripts/ci-gradle*'
             offline_jdbc:
               - 'plugins/jdbc/**'
+              - '.github/scripts/ci-gradle*'
               - 'agents/scripts/build_offline_jdbc_payload.mjs'
               - 'agents/scripts/build_offline_zip.sh'
               - 'agents/scripts/verify_offline_jdbc_release.mjs'
@@ -1464,9 +1445,7 @@ jobs:
         with:
           node-version: 22.23.3
       - name: Install packaging tools
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y zstd
+        run: sudo python3 ../.github/scripts/ci-install-system-deps.py --group packaging
       - name: Agent script tests
         run: |
           python3 -m unittest discover -s scripts -p '*_test.py'
@@ -1494,11 +1473,9 @@ jobs:
         with:
           cache-provider: basic
       - name: Install packaging tools
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y zstd
+        run: sudo python3 ../.github/scripts/ci-install-system-deps.py --group packaging
       - name: Java agent tests and packages
-        run: ./gradlew test shadowJar --continue
+        run: python3 ../.github/scripts/ci-gradle.py agents -- ./gradlew test shadowJar --continue
       - name: Agent jar validation
         run: python3 scripts/validate_agent_jars.py
   agent-rust:
@@ -1516,26 +1493,14 @@ jobs:
       - uses: actions/checkout@v7
       - name: Install system dependencies
         if: matrix.driver == 'duckdb'
-        run: |
-          sudo timeout --kill-after=30s 10m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            update
-          sudo timeout --kill-after=30s 10m apt-get \
-            -o Acquire::Retries=3 \
-            -o Acquire::http::Timeout=30 \
-            -o Acquire::https::Timeout=30 \
-            install -y libdbus-1-dev pkg-config
+        run: sudo python3 ../.github/scripts/ci-install-system-deps.py --group duckdb
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@1.97.1
       # Pin mold because runner image updates can change bundled DuckDB C++ links.
       # See the packages job note for the pinned upstream build.
       - name: Install mold linker
-        run: |
-          curl -fsSL https://github.com/rui314/mold/releases/download/v2.42.1/mold-2.42.1-x86_64-linux.tar.gz \
-            | sudo tar -C /usr/local --strip-components=1 -xz
-          mold --version
+        timeout-minutes: 8
+        run: bash ../.github/scripts/ci-install-mold.sh
       - name: Setup sccache
         uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
         with: