fix(ci): repair cross image builds (#6617)

* fix(ci): repair cross image builds

* fix(ci): expose Lambda artifact to cross builds

* fix(ci): add curl headers to GNU cross images

* fix(ci): expose ARM64 Zlib to CMake

* fix(ci): preserve Cargo CMake dependency prefixes

* fix(ci): link resolver for ARM64 releases

* fix(ci): preserve ARM64 release Rust flags

* fix(ci): route ARM64 CMake builds through GCC 10

* fix(ci): propagate ARM64 release Rust flags

.cargo/config.toml

@@ -13,5 +13,13 @@ rustflags = ["-C", "target-cpu=x86-64-v3", "--cfg", "tokio_unstable"]
 # Targeting neoverse-n1 covers AWS Graviton2+ and GCP T2A (Ampere Altra).
 # Key gains over generic ARMv8-A: LSE atomics (better concurrency), dot
 # product instructions, and ARMv8.2 crypto extensions.
-rustflags = ["-C", "target-cpu=neoverse-n1", "--cfg", "tokio_unstable"]
-
+# sasl2-sys omits libresolv when HOST != TARGET, but its vendored Kerberos
+# archive still references resolver symbols when cross-compiling for Linux.
+rustflags = [
+  "-C",
+  "target-cpu=neoverse-n1",
+  "-C",
+  "link-arg=-lresolv",
+  "--cfg",
+  "tokio_unstable",
+]

.github/actions/cross-build-binary/action.yml

@@ -36,6 +36,11 @@ runs:
         echo "QW_COMMIT_HASH=$(git rev-parse HEAD)" >> $GITHUB_ENV
         echo "QW_COMMIT_TAGS=$(git tag --points-at HEAD | tr '\n' ',')" >> $GITHUB_ENV
       shell: bash
+    - name: Configure ARM64 Linux Rust flags
+      if: inputs.target == 'aarch64-unknown-linux-gnu'
+      run: |
+        echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS=-C target-cpu=neoverse-n1 -C link-arg=-lresolv --cfg tokio_unstable" >> "$GITHUB_ENV"
+      shell: bash
     - name: Build Quickwit
       run: cross build --release --features release-feature-vendored-set --target ${{ inputs.target }} --bin quickwit
       shell: bash

.github/workflows/publish_release_packages.yml

@@ -107,12 +107,12 @@ jobs:
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: ${{ needs.build-lambda.outputs.artifact-name }}
-          # Must be within the workspace so cross can mount it into the build container.
-          path: ./lambda-artifact
+          # `cross` mounts the Rust workspace (`quickwit/`) at `/project` in the container.
+          path: ./quickwit/lambda-artifact
       - name: Set LAMBDA_ZIP_PATH
         run: |
-          ZIP=$(ls lambda-artifact/*.zip)
-          echo "LAMBDA_ZIP_PATH=$(pwd)/$ZIP" >> $GITHUB_ENV
+          ZIP=$(ls quickwit/lambda-artifact/*.zip)
+          echo "LAMBDA_ZIP_PATH=/project/lambda-artifact/$(basename "$ZIP")" >> "$GITHUB_ENV"
       - uses: ./.github/actions/cross-build-binary
         with:
           target: ${{ matrix.target }}

Makefile

@@ -71,7 +71,8 @@ IMAGE_TAGS = x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu x86_64-unknown-l
 
 .PHONY: cross-images
 cross-images:
-	@for tag in ${IMAGE_TAGS}; do \
+	@set -e; \
+	for tag in ${IMAGE_TAGS}; do \
 		docker build --tag quickwit/cross:$$tag --file ./build/cross-images/$$tag.dockerfile ./build/cross-images; \
 		docker push quickwit/cross:$$tag; \
 	done

build/cross-images/aarch64-unknown-linux-gnu.dockerfile

@@ -12,14 +12,39 @@ ARG PBC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v21.5
 # `rdkafka/gssapi-vendored` feature when there is a release including:
 # https://github.com/MaterializeInc/rust-sasl/pull/48
 
+# librdkafka 2.12.1 includes curl/curl.h even when OIDC support is disabled.
 RUN dpkg --add-architecture arm64 && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
         binutils-aarch64-linux-gnu \
+        g++-10-aarch64-linux-gnu \
+        gcc-10-aarch64-linux-gnu \
+        libcurl4-openssl-dev:arm64 \
         libsasl2-dev:arm64 \
         unzip && \
     rm -rf /var/lib/apt/lists/*
 
+# cmake-rs passes vendored dependency prefixes in the CMAKE_PREFIX_PATH
+# environment variable. Materialize that colon-separated value before the
+# cross-rs toolchain constructs its target-only search roots.
+#
+# The toolchain also selects the unversioned GCC binaries explicitly, which
+# overrides the target-specific CC/CXX variables below. Rewrite only the
+# compiler entries: unlike GCC, the target binutils are not version-suffixed.
+RUN sed -i \
+    -e '/set(CMAKE_FIND_ROOT_PATH /i\    string(REPLACE ":" ";" CMAKE_PREFIX_PATH "$ENV{CMAKE_PREFIX_PATH}")' \
+    -e '/set(CMAKE_C_COMPILER /c\set(CMAKE_C_COMPILER "${prefix}gcc-10")' \
+    -e '/set(CMAKE_ASM_COMPILER /c\set(CMAKE_ASM_COMPILER "${prefix}gcc-10")' \
+    -e '/set(CMAKE_CXX_COMPILER /c\set(CMAKE_CXX_COMPILER "${prefix}g++-10")' \
+    /opt/toolchain.cmake
+
+# GCC 9.4 is affected by https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189.
+# aws-lc cannot execute its compiler probe while cross-compiling, so select the
+# verified GCC 10.5 toolchain explicitly for both compilation and linking.
+ENV CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc-10 \
+    CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++-10 \
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc-10
+
 RUN curl -fLO $PBC_URL && \
     unzip protoc-21.5-linux-x86_64.zip -d ./protobuf && \
     mv ./protobuf/bin/protoc /usr/bin/ && \

build/cross-images/aarch64-unknown-linux-musl.dockerfile

@@ -11,6 +11,7 @@ FROM rustembedded/cross:aarch64-unknown-linux-musl@sha256:22627e0ba533781062127b
 # ALSO UPDATE hooks/build!
 ARG OPENSSL_VERSION=1.1.1i
 ARG ZLIB_VERSION=1.2.11
+ARG ZLIB_SHA256=629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff
 
 RUN echo "Building OpenSSL" && \
     cd /tmp && \
@@ -26,7 +27,9 @@ RUN echo "Building OpenSSL" && \
 
 RUN echo "Building zlib" && \
     cd /tmp && \
-    curl -fLO "https://zlib.net/fossils/zlib-$ZLIB_VERSION.tar.gz" && \
+    curl -fL --retry 3 -o "zlib-$ZLIB_VERSION.tar.gz" \
+        "https://github.com/madler/zlib/archive/refs/tags/v$ZLIB_VERSION.tar.gz" && \
+    echo "$ZLIB_SHA256  zlib-$ZLIB_VERSION.tar.gz" | sha256sum --check - && \
     tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \
     AR=aarch64-linux-musl-ar CC=aarch64-linux-musl-gcc ./configure --static --prefix=/usr/local/aarch64-linux-musl && \
     make && make install && \

build/cross-images/x86_64-unknown-linux-gnu.dockerfile

@@ -7,12 +7,22 @@ FROM ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main@sha256:2431cbfcf2499f8a00570
 
 ARG PBC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v21.5/protoc-21.5-linux-x86_64.zip"
 
+# librdkafka 2.12.1 includes curl/curl.h even when OIDC support is disabled.
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+        g++-10 \
+        gcc-10 \
+        libcurl4-openssl-dev \
         libsasl2-dev \
         unzip && \
     rm -rf /var/lib/apt/lists/*
 
+# GCC 9.4 is affected by https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95189,
+# which aws-lc correctly rejects because it can miscompile memcmp at -O3.
+ENV CC_x86_64_unknown_linux_gnu=gcc-10 \
+    CXX_x86_64_unknown_linux_gnu=g++-10 \
+    CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc-10
+
 RUN curl -fLO $PBC_URL && \
     unzip protoc-21.5-linux-x86_64.zip -d ./protobuf && \
     mv ./protobuf/bin/protoc /usr/bin/ && \

quickwit/Cross.toml

@@ -1,5 +1,6 @@
 [build.env]
 passthrough = [
+    "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUSTFLAGS",
     "QW_COMMIT_DATE",
     "QW_COMMIT_HASH",
     "QW_COMMIT_TAGS",