diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index face057ee48..2de3a4bfa74 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -300,6 +300,80 @@ steps: - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images' + + # ---- AWS EFA variants: docker/Dockerfile.efa on the images built above ---- + - label: "Build release image - x86_64 - CUDA 13.0 - EFA" + depends_on: + - build-release-image-x86 + id: build-release-image-x86-efa + agents: + queue: cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh efa) \ + --build-arg BASE_IMAGE=public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ + --progress plain \ + -f docker/Dockerfile.efa docker/ + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-efa" + # re-tag to default image tag and push, just in case arm64 build fails + - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-efa public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-efa" + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-efa" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-efa" release-images' + + - label: "Build release image - aarch64 - CUDA 13.0 - EFA" + depends_on: + - build-release-image-arm64 + id: build-release-image-arm64-efa + agents: + queue: arm64_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh efa) \ + --build-arg BASE_IMAGE=public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) \ + --progress plain \ + -f docker/Dockerfile.efa docker/ + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-efa" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-efa" release-images' + + - label: "Build release image - x86_64 - Rubin CUDA 13.4 - EFA" + depends_on: + - build-release-image-x86-rubin-cuda-13-4 + id: build-release-image-x86-rubin-cuda-13-4-efa + if: build.env("NIGHTLY") == "1" + agents: + queue: cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134-efa) \ + --build-arg BASE_IMAGE=public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134 \ + --progress plain \ + -f docker/Dockerfile.efa docker/ + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134-efa" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134-efa" release-images' + + - label: "Build release image - aarch64 - Rubin CUDA 13.4 - EFA" + depends_on: + - build-release-image-arm64-rubin-cuda-13-4 + id: build-release-image-arm64-rubin-cuda-13-4-efa + if: build.env("NIGHTLY") == "1" + agents: + queue: arm64_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - | + DOCKER_BUILDKIT=1 docker build \ + $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134-efa) \ + --build-arg BASE_IMAGE=public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134 \ + --progress plain \ + -f docker/Dockerfile.efa docker/ + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134-efa" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134-efa" release-images' - label: ":docker: Build release image - x86_64 - XPU" depends_on: ~ id: build-xpu-release-image @@ -394,6 +468,35 @@ steps: - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: Rubin CUDA 13.4" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" release-manifests' + + + # ---- AWS EFA variant manifests ---- + - label: "Create multi-arch manifest - CUDA 13.0 - EFA" + depends_on: + - build-release-image-x86-efa + - build-release-image-arm64-efa + id: create-multi-arch-manifest-efa + agents: + queue: small_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-efa public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-efa public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-efa --amend" + - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-efa" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0 - EFA" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-efa" release-manifests' + + - label: "Create multi-arch manifest - Rubin CUDA 13.4 - EFA" + depends_on: + - build-release-image-x86-rubin-cuda-13-4-efa + - build-release-image-arm64-rubin-cuda-13-4-efa + id: create-multi-arch-manifest-rubin-cuda-13-4-efa + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" + - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134-efa public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu134-efa public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu134-efa --amend" + - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134-efa" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: Rubin CUDA 13.4 - EFA" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134-efa" release-manifests' - label: "Create manifest - XPU" depends_on: - build-xpu-release-image @@ -761,6 +864,10 @@ steps: allow_failure: true - step: build-cpu-release-image-arm64 allow_failure: true + # Wait for the EFA images, but don't let an EFA failure block publishing + # the others; the cuda-13-0-efa publish job fails on a partial build. + - step: create-multi-arch-manifest-efa + allow_failure: true - label: "Publish {{matrix}} release images to DockerHub" depends_on: @@ -772,6 +879,8 @@ steps: - "bash .buildkite/scripts/publish-release-images.sh {{matrix}}" matrix: - "cuda-13-0" + # AWS EFA variant; skipped if no EFA image was built, fails on a partial build. + - "cuda-13-0-efa" - "rocm" - "rocm72" - "xpu" diff --git a/.buildkite/scripts/publish-release-images.sh b/.buildkite/scripts/publish-release-images.sh index eb49a45e8a7..bdd24be4ee4 100755 --- a/.buildkite/scripts/publish-release-images.sh +++ b/.buildkite/scripts/publish-release-images.sh @@ -10,9 +10,9 @@ set -euo pipefail TARGET="${1:-all}" case "${TARGET}" in - cuda-13-0 | rocm | rocm72 | xpu | cpu | all) ;; + cuda-13-0 | cuda-13-0-efa | rocm | rocm72 | xpu | cpu | all) ;; *) - echo "Usage: $0 {cuda-13-0|rocm|rocm72|xpu|cpu|all}" + echo "Usage: $0 {cuda-13-0|cuda-13-0-efa|rocm|rocm72|xpu|cpu|all}" exit 2 ;; esac @@ -21,6 +21,63 @@ target_enabled() { [ "${TARGET}" = "all" ] || [ "${TARGET}" = "$1" ] } +# Promote one image family from ECR staging to DockerHub: per-arch tags plus the +# latest and versioned multi-arch manifests. $1 is the tag suffix shared by both +# registries: "" for the default CUDA family, else e.g. "-efa". +publish_image_family() { + local suffix="$1" arch + local staging="public.ecr.aws/q9t5s3a7/vllm-release-repo" + local latest="vllm/vllm-openai:latest${suffix}" + local versioned="vllm/vllm-openai:v${RELEASE_VERSION}${suffix}" + + for arch in x86_64 aarch64; do + docker pull "${staging}:${COMMIT}-${arch}${suffix}" + done + + for arch in x86_64 aarch64; do + docker tag "${staging}:${COMMIT}-${arch}${suffix}" "vllm/vllm-openai:latest-${arch}${suffix}" + docker tag "${staging}:${COMMIT}-${arch}${suffix}" "vllm/vllm-openai:v${RELEASE_VERSION}-${arch}${suffix}" + docker push "vllm/vllm-openai:latest-${arch}${suffix}" + docker push "vllm/vllm-openai:v${RELEASE_VERSION}-${arch}${suffix}" + done + + docker manifest rm "${latest}" || true + docker manifest rm "${versioned}" || true + docker manifest create "${latest}" \ + "vllm/vllm-openai:latest-x86_64${suffix}" \ + "vllm/vllm-openai:latest-aarch64${suffix}" + docker manifest create "${versioned}" \ + "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64${suffix}" \ + "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64${suffix}" + docker manifest push "${latest}" + docker manifest push "${versioned}" +} + +# Publish an opt-in family all-or-nothing, like the CPU images below: skip when +# neither arch was built, fail on a partial build so it is not silenced. +publish_optional_family() { + local suffix="$1" arch missing=() present=() + local staging="public.ecr.aws/q9t5s3a7/vllm-release-repo" + + for arch in x86_64 aarch64; do + if docker manifest inspect "${staging}:${COMMIT}-${arch}${suffix}" >/dev/null 2>&1; then + present+=("${arch}") + else + missing+=("${arch}") + fi + done + + if [ "${#present[@]}" -eq 0 ]; then + echo "WARNING: no ${suffix} images found in ECR for ${COMMIT}, skipping the ${suffix} publish" + elif [ "${#missing[@]}" -eq 0 ]; then + publish_image_family "${suffix}" + else + echo "ERROR: partial ${suffix} build (present: ${present[*]}; missing: ${missing[*]})." >&2 + echo " Refusing to publish to avoid per-arch and multi-arch tags disagreeing." >&2 + exit 1 + fi +} + RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//') if [ -z "${RELEASE_VERSION}" ]; then echo "ERROR: release-version metadata not set" @@ -41,25 +98,7 @@ aws ecr-public get-login-password --region us-east-1 | \ # ---- CUDA (default: 13.0) ---- if target_enabled cuda-13-0; then - docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64" - docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64" - - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64" vllm/vllm-openai:latest-x86_64 - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64" - docker push vllm/vllm-openai:latest-x86_64 - docker push "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64" - - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64" vllm/vllm-openai:latest-aarch64 - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64" - docker push vllm/vllm-openai:latest-aarch64 - docker push "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64" - - docker manifest rm vllm/vllm-openai:latest || true - docker manifest rm "vllm/vllm-openai:v${RELEASE_VERSION}" || true - docker manifest create vllm/vllm-openai:latest vllm/vllm-openai:latest-x86_64 vllm/vllm-openai:latest-aarch64 - docker manifest create "vllm/vllm-openai:v${RELEASE_VERSION}" "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64" "vllm/vllm-openai:v${RELEASE_VERSION}-aarch64" - docker manifest push vllm/vllm-openai:latest - docker manifest push "vllm/vllm-openai:v${RELEASE_VERSION}" + publish_image_family "" ZSTD_DIGEST=$(docker buildx imagetools inspect \ "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64" \ @@ -69,6 +108,14 @@ if target_enabled cuda-13-0; then "vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-zstd" fi +# ---- AWS EFA variant (docker/Dockerfile.efa) ---- +# No cu134 entry: Rubin CUDA 13.4 is nightly-only and is not promoted to +# DockerHub for the base variant either. + +if target_enabled cuda-13-0-efa; then + publish_optional_family "-efa" +fi + # ---- ROCm ---- # Each ROCm stack is identified by its base Dockerfile (see rocm/stack.sh). The # default stack also owns the plain :latest / :v tags. diff --git a/docker/Dockerfile b/docker/Dockerfile index c359a93c378..f3cac42652e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,7 +31,8 @@ ARG TRITON_INSTALL_FROM_SOURCE_REVISION= # DeepEPv2 requires NCCL >= 2.30.4 (GIN backend). # This version is used for CUDA 12+ builds to keep the DeepEP build and # runtime NCCL versions compatible. -ARG NCCL_VERSION=2.30.7 +# DeepEPv2 GDA over EFA requires NCCL 2.31.x or later. +ARG NCCL_VERSION=2.31.2 # By parameterizing the base images, we allow third-party to use their own # base images. One use case is hermetic builds with base images stored in diff --git a/docker/Dockerfile.efa b/docker/Dockerfile.efa new file mode 100644 index 00000000000..00c22248c13 --- /dev/null +++ b/docker/Dockerfile.efa @@ -0,0 +1,156 @@ +# AWS EFA variant of a vllm-openai image. Adds the EFA userspace stack +# (libfabric, aws-ofi-nccl, hwloc), the EFA Mooncake wheel (x86_64 only), +# and DeepEP rebuilt from the amazon-contributing fork for NCCL GIN over EFA. +# +# The base must be built with INSTALL_KV_CONNECTORS=true (NIXL and Mooncake): +# docker build -f docker/Dockerfile.efa \ +# --build-arg BASE_IMAGE=vllm/vllm-openai:latest -t vllm:openai-efa docker/ +# +# The host still provides the EFA kernel module and memlock limits. +ARG BASE_IMAGE=vllm/vllm-openai:latest +FROM ${BASE_IMAGE} + +ARG TARGETARCH +ARG EFA_INSTALLER_VERSION=1.51.0 +ARG EFA_INSTALLER_SHA256=83993044fa4df3dc2f8b5c2f7051d61b9689441be023a2c92ea51d3f53f2465c + +# Without the KV connectors there is no NIXL plugin or Mooncake wheel to use EFA. +RUN set -eu; \ + if ! python3 -c "import importlib.metadata as m; m.version('nixl')" \ + 2>/dev/null; then \ + echo "Dockerfile.efa needs a base image built with \ +--build-arg INSTALL_KV_CONNECTORS=true" >&2; \ + exit 1; \ + fi + +# Install the AWS EFA userspace stack. +RUN set -eux; \ + EFA_INSTALLER_ARCHIVE="/tmp/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz"; \ + curl -fsSL --retry 3 --retry-all-errors \ + "https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz" \ + -o "${EFA_INSTALLER_ARCHIVE}"; \ + echo "${EFA_INSTALLER_SHA256} ${EFA_INSTALLER_ARCHIVE}" | sha256sum -c -; \ + tar -xzf "${EFA_INSTALLER_ARCHIVE}" -C /tmp; \ + # The installer apt-installs dependencies; the base image purged the lists. + apt-get update -y; \ + # Minimal install: libfabric, aws-ofi-nccl and hwloc only. --build-ngc + # implies --skip-kmod --skip-limit-conf --no-verify. + (cd /tmp/aws-efa-installer && \ + ./efa_installer.sh --skip-mpi --skip-rdma-core --build-ngc); \ + test -x /opt/amazon/efa/bin/fi_info; \ + /opt/amazon/efa/bin/fi_info --version; \ + # `-l` lists providers without EFA hardware, so this works in a build. + /opt/amazon/efa/bin/fi_info -l | grep -q '^efa:'; \ + EFA_NIXL_PLUGIN=$(find /usr/local/lib -name libplugin_LIBFABRIC.so \ + -print -quit); \ + test -n "${EFA_NIXL_PLUGIN}"; \ + ldd "${EFA_NIXL_PLUGIN}" | grep -q 'libfabric\.so\.1 => /opt/amazon/efa/lib/'; \ + # Plugins that NCCL_NET_PLUGIN=ofi and NCCL_TUNER_PLUGIN=ofi resolve to. + ldconfig -p | grep -q '/opt/amazon/ofi-nccl/lib/libnccl-net-ofi\.so$'; \ + ldconfig -p | grep -q '/opt/amazon/ofi-nccl/lib/libnccl-tuner-ofi\.so$'; \ + apt-get clean; \ + rm -rf "${EFA_INSTALLER_ARCHIVE}" /tmp/aws-efa-installer \ + /var/lib/apt/lists/* + +# Route NCCL over EFA. Override with `-e NCCL_NET_PLUGIN= -e NCCL_TUNER_PLUGIN=` +# to run this image on a non-EFA host. +ENV NCCL_NET_PLUGIN=ofi \ + NCCL_TUNER_PLUGIN=ofi + +# Swap in the EFA Mooncake wheel at the base image's Mooncake version. Both +# wheels own the `mooncake` module; the EFA wheels are published for x86_64 only. +RUN --mount=type=cache,target=/opt/uv/cache \ + set -eu; \ + pkg_version() { python3 -c "import importlib.metadata as m, sys; \ +print(m.version(sys.argv[1]))" "$1" 2>/dev/null; }; \ + if MOONCAKE_VERSION=$(pkg_version mooncake-transfer-engine-cuda13); then \ + MOONCAKE_PKG=mooncake-transfer-engine-cuda13; \ + MOONCAKE_EFA_PKG=mooncake-transfer-engine-efa-cuda13; \ + elif MOONCAKE_VERSION=$(pkg_version mooncake-transfer-engine); then \ + MOONCAKE_PKG=mooncake-transfer-engine; \ + MOONCAKE_EFA_PKG=mooncake-transfer-engine-efa; \ + else \ + echo "No Mooncake wheel in the base image; build it with \ +--build-arg INSTALL_KV_CONNECTORS=true" >&2; \ + exit 1; \ + fi; \ + if [ "${TARGETARCH}" != "amd64" ]; then \ + echo "TARGETARCH=${TARGETARCH}: ${MOONCAKE_EFA_PKG} is x86_64-only, \ +keeping ${MOONCAKE_PKG} (Mooncake EFA protocol unavailable)"; \ + else \ + echo "Swapping ${MOONCAKE_PKG}==${MOONCAKE_VERSION} for \ +${MOONCAKE_EFA_PKG}==${MOONCAKE_VERSION}"; \ + uv pip uninstall --system "${MOONCAKE_PKG}"; \ + uv pip install --system --no-deps \ + "${MOONCAKE_EFA_PKG}==${MOONCAKE_VERSION}"; \ + test "$(pkg_version "${MOONCAKE_EFA_PKG}")" = "${MOONCAKE_VERSION}"; \ + # `! cmd` is exempt from `set -e`, so check explicitly. + if pkg_version "${MOONCAKE_PKG}" >/dev/null; then \ + echo "${MOONCAKE_PKG} is still installed" >&2; \ + exit 1; \ + fi; \ + fi + +# Replace the base image's upstream DeepEP with the amazon-contributing fork, +# whose deepep_v2 backend carries EP traffic over EFA through NCCL GIN. +ARG DEEPEP_REPO="https://github.com/amazon-contributing/DeepEP.git" +ARG DEEPEP_REF="874779c9ccd2294b56304bd6cc5f138f1f71d097" +ARG DEEPEP_CUDA_ARCH_LIST="9.0a;10.0a;10.3a" + +# Headers and git for DeepEP's nvcc build. `reinstall` also installs packages +# that are absent. +RUN set -eux; \ + CUDA_PKG_VERSION=$(nvcc --version \ + | grep -oP 'release \K[0-9]+\.[0-9]+' | tr '.' '-'); \ + DEBIAN_FRONTEND=noninteractive apt-get update -y; \ + DEBIAN_FRONTEND=noninteractive apt-get reinstall -y --no-install-recommends \ + git \ + libcusparse-dev-${CUDA_PKG_VERSION} \ + libcusolver-dev-${CUDA_PKG_VERSION} \ + cuda-nvrtc-dev-${CUDA_PKG_VERSION}; \ + rm -rf /var/lib/apt/lists/* + +RUN --mount=type=cache,target=/opt/uv/cache \ + set -eux; \ + git clone "${DEEPEP_REPO}" /tmp/deepep; \ + cd /tmp/deepep; \ + git checkout "${DEEPEP_REF}"; \ + git submodule update --init --recursive; \ + uv pip uninstall --system deep_ep 2>/dev/null || true; \ + NCCL_ROOT="$(python3 -c "import importlib.util, os; \ +s = importlib.util.find_spec('nvidia.nccl'); \ +print(os.path.dirname(s.origin or s.submodule_search_locations[0]))")/nccl"; \ + export EP_NCCL_ROOT_DIR="${NCCL_ROOT}"; \ + export LD_LIBRARY_PATH="${NCCL_ROOT}/lib:${LD_LIBRARY_PATH:-}"; \ + # The CUDA driver stubs let DeepEP link libcuda in a builder with no driver. + export LIBRARY_PATH="${NCCL_ROOT}/lib:/usr/local/cuda/lib64/stubs:\ +/usr/local/cuda/targets/x86_64-linux/lib/stubs:${LIBRARY_PATH:-}"; \ + export DISABLE_AGGRESSIVE_PTX_INSTRS=1; \ + export TORCH_CUDA_ARCH_LIST="${DEEPEP_CUDA_ARCH_LIST}"; \ + uv pip install --system --no-deps -vv --no-build-isolation .; \ + cd /; \ + rm -rf /tmp/deepep; \ + # vLLM only logs when the deepep_v2 backend is unavailable, so fail here. + python3 -c "import deep_ep; assert hasattr(deep_ep, 'ElasticBuffer'), \ +'DeepEP built without the ElasticBuffer (v2) API'" + +# Provenance. Build args from docker-build-metadata-args.sh override the values +# inherited from the base image; without them the base's values are kept. +ARG BASE_IMAGE +ARG VLLM_BUILD_COMMIT +ARG VLLM_BUILD_PIPELINE +ARG VLLM_BUILD_URL +ARG VLLM_IMAGE_TAG +ENV VLLM_BUILD_COMMIT=${VLLM_BUILD_COMMIT} \ + VLLM_BUILD_PIPELINE=${VLLM_BUILD_PIPELINE} \ + VLLM_BUILD_URL=${VLLM_BUILD_URL} \ + VLLM_IMAGE_TAG=${VLLM_IMAGE_TAG} +LABEL org.opencontainers.image.revision="${VLLM_BUILD_COMMIT}" \ + org.opencontainers.image.version="${VLLM_IMAGE_TAG}" \ + org.opencontainers.image.url="${VLLM_BUILD_URL}" \ + ai.vllm.build.commit="${VLLM_BUILD_COMMIT}" \ + ai.vllm.build.pipeline="${VLLM_BUILD_PIPELINE}" \ + ai.vllm.build.url="${VLLM_BUILD_URL}" \ + ai.vllm.image.tag="${VLLM_IMAGE_TAG}" \ + ai.vllm.efa.base-image="${BASE_IMAGE}" \ + ai.vllm.efa.installer-version="${EFA_INSTALLER_VERSION}" diff --git a/docker/versions.json b/docker/versions.json index 75c21afafc6..03a9babdc0d 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -14,7 +14,7 @@ "default": "false" }, "NCCL_VERSION": { - "default": "2.30.7" + "default": "2.31.2" }, "BUILD_BASE_IMAGE": { "default": "pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359" diff --git a/docs/contributing/dockerfile/dockerfile.md b/docs/contributing/dockerfile/dockerfile.md index ddd75129798..b5c8b8001a2 100644 --- a/docs/contributing/dockerfile/dockerfile.md +++ b/docs/contributing/dockerfile/dockerfile.md @@ -34,6 +34,25 @@ EP extension for the installed PyTorch, with `tools/build_nixl_with_torch215_for `vllm-openai-base` then replaces the NIXL packages installed from the KV-connector requirements with these wheels. +The AWS EFA images are built separately by `docker/Dockerfile.efa` on top of a +`vllm-openai` image built with `INSTALL_KV_CONNECTORS=true`. It adds a minimal +AWS EFA userspace install (libfabric, aws-ofi-nccl and hwloc), the EFA Mooncake +wheel (x86_64 only), and DeepEP built for NCCL GIN over EFA. Kernel modules and +host limits remain host-managed. + +```bash +docker build -f docker/Dockerfile.efa \ + --build-arg BASE_IMAGE=vllm/vllm-openai:latest -t vllm:openai-efa docker/ +``` + +The image does not change vLLM's Mooncake protocol default (`rdma`). To use +EFA, set it per connector: + +- `MooncakeConnector`: `"mooncake_protocol": "efa"` in `kv_connector_extra_config`. +- `MooncakeStoreConnector`: `"protocol": "efa"` in the JSON file at `MOONCAKE_CONFIG_PATH`. + +The EFA Mooncake wheel is x86_64-only, so use `tcp` on arm64. + >
> ![](../../assets/contributing/dockerfile-stages-dependency.png){ align="center" alt="query" width="100%" } >