mirrors/vllm
0
0 Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2026-10-28 09:45:26 +08:00
Code Activity

[CI] Disable jobs with unpinned runtime installs (#63609)

Browse Source 
Signed-off-by: khluu <khluu000@gmail.com>
This commit is contained in:
Kevin H. Luu authored and GitHub committed 2026-10-25 12:40:35 -07:00
1 parent 9d48c8ab95
commit f8c12c212a
16 files changed
+2354 -2016
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/hardware_tests/ascend_npu.yaml
+101 -100
View File
@@ -1,102 +1,103 @@
group: Hardware
depends_on: ~
steps:
- label: "Ascend NPU Test"
key: ascend-npu-test
soft_fail: true
no_plugin: true
device: ascend_npu
timeout_in_minutes: 30
source_file_dependencies:
- vllm/
- .buildkite/scripts/hardware_ci/run-npu-test.sh
- "!vllm/platforms/cuda.py"
- "!vllm/platforms/xpu.py"
- "!vllm/platforms/tpu.py"
- "!vllm/platforms/cpu.py"
- "!vllm/platforms/zen_cpu.py"
- "!vllm/cute_utils/"
- "!vllm/tilelang_utils/"
- "!vllm/device_allocator/"
- "!vllm/vllm_flash_attn/"
- "!vllm/kernels/"
- "!vllm/model_executor/kernels/"
- "!vllm/engine/"
- "!vllm/entrypoints/"
- "!vllm/ray/"
- "!vllm/v1/attention/backends/flash_attn.py"
- "!vllm/v1/attention/backends/flash_attn_diffkv.py"
- "!vllm/v1/attention/backends/flashinfer.py"
- "!vllm/v1/attention/backends/triton_attn.py"
- "!vllm/v1/attention/backends/triton_attn_diffkv.py"
- "!vllm/v1/attention/backends/rocm_attn.py"
- "!vllm/v1/attention/backends/rocm_aiter_fa.py"
- "!vllm/v1/attention/backends/rocm_aiter_unified_attn.py"
- "!vllm/v1/attention/backends/flex_attention.py"
- "!vllm/v1/attention/backends/hpc_attn.py"
- "!vllm/v1/attention/backends/mamba1_attn.py"
- "!vllm/v1/attention/backends/mamba2_attn.py"
- "!vllm/v1/attention/backends/mamba_attn.py"
- "!vllm/v1/attention/backends/short_conv_attn.py"
- "!vllm/v1/attention/backends/turboquant_attn.py"
- "!vllm/v1/attention/backends/cpu_attn.py"
- "!vllm/v1/attention/backends/fa_utils.py"
- "!vllm/distributed/device_communicators/pynccl.py"
- "!vllm/distributed/device_communicators/pynccl_wrapper.py"
- "!vllm/distributed/device_communicators/pynccl_allocator.py"
- "!vllm/distributed/device_communicators/xpu_communicator.py"
- "!vllm/distributed/device_communicators/cpu_communicator.py"
- "!vllm/distributed/device_communicators/cuda_wrapper.py"
- "!vllm/distributed/device_communicators/flashinfer_all_reduce.py"
- "!vllm/distributed/device_communicators/aiter_custom_all_reduce.py"
- "!vllm/distributed/device_communicators/custom_all_reduce.py"
- "!vllm/distributed/device_communicators/quick_all_reduce.py"
- "!vllm/distributed/device_communicators/all2all.py"
- "!vllm/distributed/device_communicators/all_reduce_utils.py"
- "!vllm/distributed/device_communicators/shm_object_storage.py"
- "!vllm/distributed/device_communicators/symm_mem.py"
- "!vllm/distributed/device_communicators/mnnvl_compat.py"
- "!vllm/third_party/flashmla/"
- "!vllm/third_party/pynvml.py"
- "!vllm/_custom_ops.py"
- "!vllm/_xpu_ops.py"
- "!vllm/_aiter_ops"
- "!vllm/collect_env.py"
- "!vllm/connections.py"
- "!vllm/env_override.py"
- "!vllm/logits_process.py"
- "!vllm/logprobs.py"
- "!vllm/model_inspection.py"
- "!vllm/outputs.py"
- "!vllm/scalar_type.py"
- "!vllm/version.py"
- "!vllm/assets/"
- "!vllm/benchmarks/"
- "!vllm/inputs/"
- "!vllm/ir/"
- "!vllm/parser/"
- "!vllm/plugins/"
- "!vllm/reasoning/"
- "!vllm/renderers/"
- "!vllm/tokenizers/"
- "!vllm/tool_parsers/"
- "!vllm/tracing/"
- "!vllm/usage/"
- "!tests/"
- "!benchmarks/"
- "!docs/"
- "!examples/"
- "!vllm/model_executor/layers/quantization/auto_awq.py"
- "!vllm/model_executor/layers/quantization/auto_gptq.py"
- "!vllm/model_executor/layers/quantization/inc/"
- "!vllm/model_executor/layers/quantization/awq_triton.py"
- "!vllm/model_executor/layers/quantization/experts_int8.py"
- "!vllm/model_executor/layers/quantization/fbgemm_fp8.py"
- "!vllm/model_executor/layers/quantization/fp_quant.py"
- "!vllm/model_executor/layers/quantization/humming.py"
- "!vllm/model_executor/layers/quantization/moe_wna16.py"
- "!vllm/model_executor/layers/quantization/qutlass_utils.py"
- "!vllm/model_executor/layers/quantization/quark/"
- "!vllm/model_executor/layers/quantization/turboquant/"
commands:
- bash .buildkite/scripts/hardware_ci/run-npu-test.sh
steps: []
# Disabled until the Ascend packages installation is locked.
# - label: "Ascend NPU Test"
# key: ascend-npu-test
# soft_fail: true
# no_plugin: true
# device: ascend_npu
# timeout_in_minutes: 30
# source_file_dependencies:
# - vllm/
# - .buildkite/scripts/hardware_ci/run-npu-test.sh
# - "!vllm/platforms/cuda.py"
# - "!vllm/platforms/xpu.py"
# - "!vllm/platforms/tpu.py"
# - "!vllm/platforms/cpu.py"
# - "!vllm/platforms/zen_cpu.py"
# - "!vllm/cute_utils/"
# - "!vllm/tilelang_utils/"
# - "!vllm/device_allocator/"
# - "!vllm/vllm_flash_attn/"
# - "!vllm/kernels/"
# - "!vllm/model_executor/kernels/"
# - "!vllm/engine/"
# - "!vllm/entrypoints/"
# - "!vllm/ray/"
# - "!vllm/v1/attention/backends/flash_attn.py"
# - "!vllm/v1/attention/backends/flash_attn_diffkv.py"
# - "!vllm/v1/attention/backends/flashinfer.py"
# - "!vllm/v1/attention/backends/triton_attn.py"
# - "!vllm/v1/attention/backends/triton_attn_diffkv.py"
# - "!vllm/v1/attention/backends/rocm_attn.py"
# - "!vllm/v1/attention/backends/rocm_aiter_fa.py"
# - "!vllm/v1/attention/backends/rocm_aiter_unified_attn.py"
# - "!vllm/v1/attention/backends/flex_attention.py"
# - "!vllm/v1/attention/backends/hpc_attn.py"
# - "!vllm/v1/attention/backends/mamba1_attn.py"
# - "!vllm/v1/attention/backends/mamba2_attn.py"
# - "!vllm/v1/attention/backends/mamba_attn.py"
# - "!vllm/v1/attention/backends/short_conv_attn.py"
# - "!vllm/v1/attention/backends/turboquant_attn.py"
# - "!vllm/v1/attention/backends/cpu_attn.py"
# - "!vllm/v1/attention/backends/fa_utils.py"
# - "!vllm/distributed/device_communicators/pynccl.py"
# - "!vllm/distributed/device_communicators/pynccl_wrapper.py"
# - "!vllm/distributed/device_communicators/pynccl_allocator.py"
# - "!vllm/distributed/device_communicators/xpu_communicator.py"
# - "!vllm/distributed/device_communicators/cpu_communicator.py"
# - "!vllm/distributed/device_communicators/cuda_wrapper.py"
# - "!vllm/distributed/device_communicators/flashinfer_all_reduce.py"
# - "!vllm/distributed/device_communicators/aiter_custom_all_reduce.py"
# - "!vllm/distributed/device_communicators/custom_all_reduce.py"
# - "!vllm/distributed/device_communicators/quick_all_reduce.py"
# - "!vllm/distributed/device_communicators/all2all.py"
# - "!vllm/distributed/device_communicators/all_reduce_utils.py"
# - "!vllm/distributed/device_communicators/shm_object_storage.py"
# - "!vllm/distributed/device_communicators/symm_mem.py"
# - "!vllm/distributed/device_communicators/mnnvl_compat.py"
# - "!vllm/third_party/flashmla/"
# - "!vllm/third_party/pynvml.py"
# - "!vllm/_custom_ops.py"
# - "!vllm/_xpu_ops.py"
# - "!vllm/_aiter_ops"
# - "!vllm/collect_env.py"
# - "!vllm/connections.py"
# - "!vllm/env_override.py"
# - "!vllm/logits_process.py"
# - "!vllm/logprobs.py"
# - "!vllm/model_inspection.py"
# - "!vllm/outputs.py"
# - "!vllm/scalar_type.py"
# - "!vllm/version.py"
# - "!vllm/assets/"
# - "!vllm/benchmarks/"
# - "!vllm/inputs/"
# - "!vllm/ir/"
# - "!vllm/parser/"
# - "!vllm/plugins/"
# - "!vllm/reasoning/"
# - "!vllm/renderers/"
# - "!vllm/tokenizers/"
# - "!vllm/tool_parsers/"
# - "!vllm/tracing/"
# - "!vllm/usage/"
# - "!tests/"
# - "!benchmarks/"
# - "!docs/"
# - "!examples/"
# - "!vllm/model_executor/layers/quantization/auto_awq.py"
# - "!vllm/model_executor/layers/quantization/auto_gptq.py"
# - "!vllm/model_executor/layers/quantization/inc/"
# - "!vllm/model_executor/layers/quantization/awq_triton.py"
# - "!vllm/model_executor/layers/quantization/experts_int8.py"
# - "!vllm/model_executor/layers/quantization/fbgemm_fp8.py"
# - "!vllm/model_executor/layers/quantization/fp_quant.py"
# - "!vllm/model_executor/layers/quantization/humming.py"
# - "!vllm/model_executor/layers/quantization/moe_wna16.py"
# - "!vllm/model_executor/layers/quantization/qutlass_utils.py"
# - "!vllm/model_executor/layers/quantization/quark/"
# - "!vllm/model_executor/layers/quantization/turboquant/"
# commands:
# - bash .buildkite/scripts/hardware_ci/run-npu-test.sh
.buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
+23 -22
View File
@@ -84,28 +84,29 @@ steps:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh w8a8-fp8-linear'
- label: ":intel: (BMG) Server test"
key: xpu-server-test
depends_on:
- image-build-xpu
timeout_in_minutes: 45
optional: true
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
- .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server'
# Disabled until the av installation is locked.
# - label: ":intel: (BMG) Server test"
# key: xpu-server-test
# depends_on:
# - image-build-xpu
# timeout_in_minutes: 45
# optional: true
# device: intel_gpu
# agent_tags:
# label: production
# gpu: 1+
# mem: 16+
# no_plugin: true
# env:
# REGISTRY: "public.ecr.aws/q9t5s3a7"
# REPO: "vllm-ci-test-repo"
# source_file_dependencies:
# - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
# - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
# commands:
# - >-
# bash .buildkite/scripts/hardware_ci/run-intel-test.sh
# 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server'
- label: ":intel: (BMG) Quantization test"
key: xpu-quantization-test
depends_on:
.buildkite/intel_jobs/misc_intel.yaml
+66 -64
View File
@@ -30,71 +30,73 @@ steps:
pytest -v -s v1/determinism/test_xpu_batch_invariant_ut.py &&
bash ../.buildkite/scripts/hardware_ci/run-xpu-batch-invariance.sh'
- label: ":intel: (BMG) V1 Core + KV + Metrics"
key: v1-core-kv-metrics-intel
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/v1/core
- tests/v1/executor
- tests/v1/kv_offload
- tests/v1/worker
- tests/v1/kv_connector/unit
- tests/v1/ec_connector/unit
- tests/v1/metrics
- tests/entrypoints/openai/correctness/test_lmeval.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'pip install -r requirements/kv_connectors.txt &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
cd tests &&
pytest -v -s v1/executor &&
VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit
--ignore=v1/ec_connector/unit/test_ec_cpu_connector.py'
# Disabled until the KV connectors installation is locked.
# - label: ":intel: (BMG) V1 Core + KV + Metrics"
# key: v1-core-kv-metrics-intel
# timeout_in_minutes: 45
# device: intel_gpu
# agent_tags:
# label: production
# gpu: 1+
# mem: 16+
# no_plugin: true
# working_dir: "."
# env:
# REGISTRY: "public.ecr.aws/q9t5s3a7"
# REPO: "vllm-ci-test-repo"
# VLLM_TEST_DEVICE: "xpu"
# source_file_dependencies:
# - vllm/
# - tests/v1/core
# - tests/v1/executor
# - tests/v1/kv_offload
# - tests/v1/worker
# - tests/v1/kv_connector/unit
# - tests/v1/ec_connector/unit
# - tests/v1/metrics
# - tests/entrypoints/openai/correctness/test_lmeval.py
# commands:
# - >-
# bash .buildkite/scripts/hardware_ci/run-intel-test.sh
# 'pip install -r requirements/kv_connectors.txt &&
# export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
# cd tests &&
# pytest -v -s v1/executor &&
# VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit
# --ignore=v1/ec_connector/unit/test_ec_cpu_connector.py'
- label: ":intel: (BMG) EC CPU Connector (XPU)"
key: ec-cpu-connector-xpu
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/v1/core
- tests/v1/executor
- tests/v1/kv_offload
- tests/v1/worker
- tests/v1/kv_connector/unit
- tests/v1/ec_connector/unit
- tests/v1/metrics
- tests/entrypoints/openai/correctness/test_lmeval.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'pip install -r requirements/kv_connectors.txt &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
cd tests &&
pytest -v -s v1/ec_connector/unit/test_ec_cpu_connector.py'
# Disabled until the KV connectors installation is locked.
# - label: ":intel: (BMG) EC CPU Connector (XPU)"
# key: ec-cpu-connector-xpu
# timeout_in_minutes: 45
# device: intel_gpu
# agent_tags:
# label: production
# gpu: 1+
# mem: 24+
# no_plugin: true
# working_dir: "."
# env:
# REGISTRY: "public.ecr.aws/q9t5s3a7"
# REPO: "vllm-ci-test-repo"
# VLLM_TEST_DEVICE: "xpu"
# source_file_dependencies:
# - vllm/
# - tests/v1/core
# - tests/v1/executor
# - tests/v1/kv_offload
# - tests/v1/worker
# - tests/v1/kv_connector/unit
# - tests/v1/ec_connector/unit
# - tests/v1/metrics
# - tests/entrypoints/openai/correctness/test_lmeval.py
# commands:
# - >-
# bash .buildkite/scripts/hardware_ci/run-intel-test.sh
# 'pip install -r requirements/kv_connectors.txt &&
# export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
# cd tests &&
# pytest -v -s v1/ec_connector/unit/test_ec_cpu_connector.py'
- label: ":intel: (BMG) V1 Sample + Logits"
key: v1-sample-logits-intel
.buildkite/intel_jobs/model_executor_intel.yaml
+57 -55
View File
@@ -1,59 +1,61 @@
group: Model Executor Intel
depends_on:
- image-build-xpu
steps:
- label: ":intel: (BMG) Model Executor"
key: model-executor-intel
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/engine/arg_utils.py
- vllm/config/model.py
- vllm/model_executor
- tests/model_executor
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'apt-get update && apt-get install -y curl libsodium23 &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
export PYTHONFAULTHANDLER=1 &&
cd tests &&
pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --ignore="model_executor/model_loader/tensorizer_loader/test_tensorizer.py" --ignore="model_executor/model_loader/test_reload.py" --ignore="model_executor/model_loader/test_weight_cache.py" --deselect="tests/model_executor/test_enabled_custom_ops.py::test_topk_gating_enabled_by_cdna"'
steps: []
# Disabled until the apt packages installation is locked.
# - label: ":intel: (BMG) Model Executor"
# key: model-executor-intel
# timeout_in_minutes: 45
# device: intel_gpu
# agent_tags:
# label: production
# gpu: 1+
# mem: 16+
# no_plugin: true
# working_dir: "."
# env:
# REGISTRY: "public.ecr.aws/q9t5s3a7"
# REPO: "vllm-ci-test-repo"
# VLLM_TEST_DEVICE: "xpu"
# source_file_dependencies:
# - vllm/engine/arg_utils.py
# - vllm/config/model.py
# - vllm/model_executor
# - tests/model_executor
# commands:
# - >-
# bash .buildkite/scripts/hardware_ci/run-intel-test.sh
# 'apt-get update && apt-get install -y curl libsodium23 &&
# export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
# export PYTHONFAULTHANDLER=1 &&
# cd tests &&
# pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --ignore="model_executor/model_loader/tensorizer_loader/test_tensorizer.py" --ignore="model_executor/model_loader/test_reload.py" --ignore="model_executor/model_loader/test_weight_cache.py" --deselect="tests/model_executor/test_enabled_custom_ops.py::test_topk_gating_enabled_by_cdna"'
- label: ":intel: (BMG) Tensorizer Model Loader Tests (XPU)"
key: tensorizer-model-loader-tests-xpu
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/engine/arg_utils.py
- vllm/config/model.py
- vllm/model_executor
- tests/model_executor
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'apt-get update && apt-get install -y curl libsodium23 &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
export PYTHONFAULTHANDLER=1 &&
cd tests &&
pytest -v -s model_executor/model_loader/tensorizer_loader/test_tensorizer.py -m "not slow_test"'
# Disabled until the apt packages installation is locked.
# - label: ":intel: (BMG) Tensorizer Model Loader Tests (XPU)"
# key: tensorizer-model-loader-tests-xpu
# timeout_in_minutes: 45
# device: intel_gpu
# agent_tags:
# label: production
# gpu: 1+
# mem: 24+
# no_plugin: true
# working_dir: "."
# env:
# REGISTRY: "public.ecr.aws/q9t5s3a7"
# REPO: "vllm-ci-test-repo"
# VLLM_TEST_DEVICE: "xpu"
# source_file_dependencies:
# - vllm/engine/arg_utils.py
# - vllm/config/model.py
# - vllm/model_executor
# - tests/model_executor
# commands:
# - >-
# bash .buildkite/scripts/hardware_ci/run-intel-test.sh
# 'apt-get update && apt-get install -y curl libsodium23 &&
# export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
# export PYTHONFAULTHANDLER=1 &&
# cd tests &&
# pytest -v -s model_executor/model_loader/tensorizer_loader/test_tensorizer.py -m "not slow_test"'
.buildkite/intel_jobs/test-intel.yaml
+59 -57
View File
@@ -76,63 +76,65 @@ steps:
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096
'
- label: ":intel: (BMG) V1 test"
key: xpu-v1-test
depends_on:
- image-build-xpu
timeout_in_minutes: 70
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- vllm/
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests && pip install dpctl &&
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py --ignore=v1/engine/test_core_engine_actor_manager.py &&
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
pytest -v -s v1/structured_output &&
pytest -v -s v1/test_serial_utils.py &&
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py &&
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head &&
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]'
parallelism: 2
- label: ":intel: (BMG) Qwen3-30B + V1 KV Connector"
key: xpu-qwen3-30b-v1-kv-connector
depends_on:
- image-build-xpu
timeout_in_minutes: 90
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 24+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- vllm/
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pip install dpctl &&
pytest -v -s v1/kv_connector/unit/test_multi_connector.py &&
pytest -v -s v1/kv_connector/unit/test_example_connector.py &&
cd .. &&
VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192'
# Disabled until the dpctl installation is locked.
# - label: ":intel: (BMG) V1 test"
# key: xpu-v1-test
# depends_on:
# - image-build-xpu
# timeout_in_minutes: 70
# device: intel_gpu
# agent_tags:
# label: production
# gpu: 1+
# mem: 16+
# no_plugin: true
# env:
# REGISTRY: "public.ecr.aws/q9t5s3a7"
# REPO: "vllm-ci-test-repo"
# source_file_dependencies:
# - vllm/
# - .buildkite/intel_jobs/test-intel.yaml
# commands:
# - >-
# bash .buildkite/scripts/hardware_ci/run-intel-test.sh
# 'cd tests && pip install dpctl &&
# pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
# pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py --ignore=v1/engine/test_core_engine_actor_manager.py &&
# pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
# pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
# pytest -v -s v1/structured_output &&
# pytest -v -s v1/test_serial_utils.py &&
# pytest -v -s v1/e2e/general/test_correctness_sliding_window.py &&
# pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head &&
# pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]'
# parallelism: 2
# Disabled until the dpctl installation is locked.
# - label: ":intel: (BMG) Qwen3-30B + V1 KV Connector"
# key: xpu-qwen3-30b-v1-kv-connector
# depends_on:
# - image-build-xpu
# timeout_in_minutes: 90
# device: intel_gpu
# agent_tags:
# label: production
# gpu: 2+
# mem: 24+
# no_plugin: true
# env:
# REGISTRY: "public.ecr.aws/q9t5s3a7"
# REPO: "vllm-ci-test-repo"
# source_file_dependencies:
# - vllm/
# - .buildkite/intel_jobs/test-intel.yaml
# commands:
# - >-
# bash .buildkite/scripts/hardware_ci/run-intel-test.sh
# 'cd tests &&
# pip install dpctl &&
# pytest -v -s v1/kv_connector/unit/test_multi_connector.py &&
# pytest -v -s v1/kv_connector/unit/test_example_connector.py &&
# cd .. &&
# VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192'
- label: ":intel: (BMG) Server test"
key: xpu-server-test
depends_on:
.buildkite/test-amd.yaml
+619 -591
View File
File diff suppressed because it is too large. Load diff
.buildkite/test_areas/disaggregated.yaml
+491 -477
View File
@@ -1,209 +1,215 @@
group: Disaggregated
depends_on:
- image-build
steps:
- label: ":nvidia: (L4) [Nightly] Distributed NixlConnector PD accuracy"
key: distributed-nixlconnector-pd-accuracy-4-gpus
# Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 55
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- GPU_MEMORY_UTILIZATION=0.6 SLEEP_WAKE=1 bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
mirror:
amd:
# Run on PRs even when the L4 step is optional (nightly only).
optional: false
label: ":amd: (MI355) Distributed NixlConnector PD accuracy"
dind: false
device: mi355_4
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
steps: []
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) [Nightly] Distributed NixlConnector PD accuracy"
# key: distributed-nixlconnector-pd-accuracy-4-gpus
# # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 55
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# - GPU_MEMORY_UTILIZATION=0.6 SLEEP_WAKE=1 bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
# mirror:
# amd:
# # Run on PRs even when the L4 step is optional (nightly only).
# optional: false
# label: ":amd: (MI355) Distributed NixlConnector PD accuracy"
# dind: false
# device: mi355_4
# timeout_in_minutes: 60
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: ":nvidia: (L4) [Nightly] Distributed FlashInfer NixlConnector PD accuracy"
key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
# Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 55
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
# Run on PRs even when the L4 step is optional (nightly only).
optional: false
label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Preserve all six model/TP pairs with native AITER attention.
- pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) [Nightly] Distributed FlashInfer NixlConnector PD accuracy"
# key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
# # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 55
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# mirror:
# amd:
# # Run on PRs even when the L4 step is optional (nightly only).
# optional: false
# label: ":amd: (MI355) Distributed AITER NixlConnector PD accuracy"
# dind: false
# device: mi355_4
# num_devices: 4
# timeout_in_minutes: 60
# working_dir: "/vllm-workspace/tests"
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - tests/v1/kv_connector/rocm_pd_accuracy_utils.py
# - tests/utils.py
# - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
# - vllm/platforms/rocm.py
# - requirements/common.txt
# - requirements/kv_connectors_rocm.txt
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# # Preserve all six model/TP pairs with native AITER attention.
# - pytest -v -s v1/kv_connector/nixl_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) [Nightly] Push NixlConnector PP prefill PD accuracy"
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
# Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/nixl_push_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
mirror:
amd:
# Run on PRs even when the L4 step is optional (nightly only).
optional: false
label: ":amd: (MI300) Push NixlConnector PP prefill PD accuracy"
dind: false
device: mi300_4
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/nixl_push_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) [Nightly] Push NixlConnector PP prefill PD accuracy"
# key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
# # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 30
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - tests/v1/kv_connector/nixl_push_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
# mirror:
# amd:
# # Run on PRs even when the L4 step is optional (nightly only).
# optional: false
# label: ":amd: (MI300) Push NixlConnector PP prefill PD accuracy"
# dind: false
# device: mi300_4
# timeout_in_minutes: 55
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - tests/v1/kv_connector/nixl_push_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
- label: ":nvidia: (L4) [Nightly] DP EP Distributed NixlConnector PD accuracy"
key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
# Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
# Run on PRs even when the L4 step is optional (nightly only).
optional: false
label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy"
dind: false
device: mi355_4
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) [Nightly] DP EP Distributed NixlConnector PD accuracy"
# key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
# # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 30
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# mirror:
# amd:
# # Run on PRs even when the L4 step is optional (nightly only).
# optional: false
# label: ":amd: (MI355) DP EP Distributed NixlConnector PD accuracy"
# dind: false
# device: mi355_4
# timeout_in_minutes: 40
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: ":nvidia: (L4) [Nightly] CrossLayer KV layout Distributed NixlConnector PD accuracy"
key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
# Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 55
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
# Run on PRs even when the L4 step is optional (nightly only).
optional: false
label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy"
dind: false
device: mi300_4
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) [Nightly] CrossLayer KV layout Distributed NixlConnector PD accuracy"
# key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
# # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 55
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# mirror:
# amd:
# # Run on PRs even when the L4 step is optional (nightly only).
# optional: false
# label: ":amd: (MI300) CrossLayer KV layout Distributed NixlConnector PD accuracy"
# dind: false
# device: mi300_4
# timeout_in_minutes: 60
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: ":nvidia: (L4) [Nightly] Hybrid SSM NixlConnector PD accuracy"
key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
# Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
# Run on PRs even when the L4 step is optional (nightly only).
optional: false
label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy"
dind: false
device: mi300_4
timeout_in_minutes: 80
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) [Nightly] Hybrid SSM NixlConnector PD accuracy"
# key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
# # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 60
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# mirror:
# amd:
# # Run on PRs even when the L4 step is optional (nightly only).
# optional: false
# label: ":amd: (MI300) Hybrid SSM NixlConnector PD accuracy"
# dind: false
# device: mi300_4
# timeout_in_minutes: 80
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
# Per-PR core of the nightly-only 4-GPU sweeps above: hetero-TP (both
# directions), MoE DP-EP decode, push-mode PP prefill, sleep/wake, MLA,
@@ -211,290 +217,298 @@ steps:
# KV sharing on a 2-GPU pod. TP>1 configs share GPUs between prefill and
# decode (gpu-memory-utilization 0.35 per engine); TP1/TP1 architecture
# configs keep one dedicated GPU per side.
- label: ":nvidia: (L4) NixlConnector PD accuracy core"
key: nixlconnector-pd-accuracy-core-2-gpus
timeout_in_minutes: 90
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
# No AMD mirror: the MI300/MI355 mirrors above are optional: false and
# already cover these paths on PRs.
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/nixl_push_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/config_sweep_accuracy_core_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) NixlConnector PD accuracy core"
# key: nixlconnector-pd-accuracy-core-2-gpus
# timeout_in_minutes: 90
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 2
# # No AMD mirror: the MI300/MI355 mirrors above are optional: false and
# # already cover these paths on PRs.
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - tests/v1/kv_connector/nixl_push_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_core_test.sh
- label: ":nvidia: (L4) NixlConnector PD edge case"
key: nixlconnector-pd-edge-cases-2-gpus
timeout_in_minutes: 40
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- tests/v1/kv_connector/nixl_integration/
env:
PREFILL_GPU_ID: "0"
DECODE_GPU_ID: "1"
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
mirror:
amd:
label: ":amd: (MI300) NixlConnector PD edge case"
dind: false
device: mi300_2
timeout_in_minutes: 65
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- PREFILL_GPU_ID=0 DECODE_GPU_ID=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) NixlConnector PD edge case"
# key: nixlconnector-pd-edge-cases-2-gpus
# timeout_in_minutes: 40
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 2
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/v1/core/sched/
# - tests/v1/kv_connector/nixl_integration/
# env:
# PREFILL_GPU_ID: "0"
# DECODE_GPU_ID: "1"
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
# mirror:
# amd:
# label: ":amd: (MI300) NixlConnector PD edge case"
# dind: false
# device: mi300_2
# timeout_in_minutes: 65
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/v1/core/sched/
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - PREFILL_GPU_ID=0 DECODE_GPU_ID=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
- label: ":nvidia: (L4) Hybrid SSM NixlConnector PD prefix cache"
key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
timeout_in_minutes: 25
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- vllm/v1/core/kv_cache_coordinator.py
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
mirror:
amd:
label: ":amd: (MI355) Hybrid SSM NixlConnector PD prefix cache"
dind: false
device: mi355_2
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- vllm/v1/core/kv_cache_coordinator.py
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) Hybrid SSM NixlConnector PD prefix cache"
# key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
# timeout_in_minutes: 25
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 2
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/v1/core/sched/
# - vllm/v1/core/kv_cache_coordinator.py
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
# mirror:
# amd:
# label: ":amd: (MI355) Hybrid SSM NixlConnector PD prefix cache"
# dind: false
# device: mi355_2
# timeout_in_minutes: 25
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/v1/core/sched/
# - vllm/v1/core/kv_cache_coordinator.py
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
- label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD accuracy"
key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
timeout_in_minutes: 40
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy"
dind: false
device: mi355_2
timeout_in_minutes: 40
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
- requirements/kv_connectors_rocm.txt
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD accuracy"
# key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
# timeout_in_minutes: 40
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 2
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
# mirror:
# amd:
# label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD accuracy"
# dind: false
# device: mi355_2
# timeout_in_minutes: 40
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
# - tests/v1/kv_connector/nixl_integration/
# - requirements/kv_connectors_rocm.txt
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh
- label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance"
key: nixlconnector-pd-spec-decode-acceptance-2-gpus
timeout_in_minutes: 45
device: a100
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/worker/kv_connector_model_runner_mixin.py
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
mirror:
amd:
label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance"
dind: false
device: mi355_2
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/worker/kv_connector_model_runner_mixin.py
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (A100) NixlConnector PD + Spec Decode acceptance"
# key: nixlconnector-pd-spec-decode-acceptance-2-gpus
# timeout_in_minutes: 45
# device: a100
# working_dir: "/vllm-workspace/tests"
# num_devices: 2
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/v1/worker/kv_connector_model_runner_mixin.py
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
# mirror:
# amd:
# label: ":amd: (MI355) NixlConnector PD + Spec Decode acceptance"
# dind: false
# device: mi355_2
# timeout_in_minutes: 45
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/v1/worker/kv_connector_model_runner_mixin.py
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
- label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD edge cases"
key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus
timeout_in_minutes: 25
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
mirror:
amd:
label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD edge cases"
dind: false
device: mi355_2
timeout_in_minutes: 25
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) MultiConnector (Nixl+Offloading) PD edge cases"
# key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus
# timeout_in_minutes: 25
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 2
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
# - tests/v1/kv_connector/nixl_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
# mirror:
# amd:
# label: ":amd: (MI355) MultiConnector (Nixl+Offloading) PD edge cases"
# dind: false
# device: mi355_2
# timeout_in_minutes: 25
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/offloading/
# - tests/v1/kv_connector/nixl_integration/
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
# P TP 4 - D DPEP 4 test case for DSv4-Flash
- label: ":nvidia: (H200) [Nightly] DSv4-Flash Disaggregated DP EP"
key: dsv4-flash-disaggregated
timeout_in_minutes: 60
device: h200
optional: true
working_dir: "/vllm-workspace/tests"
num_devices: 8
env:
ENABLE_HMA_FLAG: "1"
DP_EP: "1"
GPU_MEMORY_UTILIZATION: "0.85"
PREFILLER_TP_SIZE: "4"
DECODER_TP_SIZE: "4"
PREFILL_BLOCK_SIZE: "256"
DECODE_BLOCK_SIZE: "256"
MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash"
VLLM_ENGINE_READY_TIMEOUT_S: "1800"
VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8"
# The V4 indexer only supports block-outermost layouts; let it pick.
PREFILLER_KV_LAYOUT: ""
DECODER_KV_LAYOUT: ""
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP"
dind: false
device: mi355_8
timeout_in_minutes: 85
depends_on:
- image-build-amd
source_file_dependencies:
- requirements/kv_connectors_rocm.txt
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/models/deepseek_v4/
- vllm/v1/attention/
- tests/v1/kv_connector/nixl_integration/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# A mirror does not inherit the parent env, so it is repeated here in
# the parent's order. The layouts must stay empty: unset, the script
# defaults to HND, which the V4 indexer rejects.
- >-
VLLM_ROCM_USE_AITER=1
ENABLE_HMA_FLAG=1
DP_EP=1
GPU_MEMORY_UTILIZATION=0.85
PREFILLER_TP_SIZE=4
DECODER_TP_SIZE=4
PREFILL_BLOCK_SIZE=256
DECODE_BLOCK_SIZE=256
MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash
VLLM_ENGINE_READY_TIMEOUT_S=1800
VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8
PREFILLER_KV_LAYOUT=
DECODER_KV_LAYOUT=
bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (H200) [Nightly] DSv4-Flash Disaggregated DP EP"
# key: dsv4-flash-disaggregated
# timeout_in_minutes: 60
# device: h200
# optional: true
# working_dir: "/vllm-workspace/tests"
# num_devices: 8
# env:
# ENABLE_HMA_FLAG: "1"
# DP_EP: "1"
# GPU_MEMORY_UTILIZATION: "0.85"
# PREFILLER_TP_SIZE: "4"
# DECODER_TP_SIZE: "4"
# PREFILL_BLOCK_SIZE: "256"
# DECODE_BLOCK_SIZE: "256"
# MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash"
# VLLM_ENGINE_READY_TIMEOUT_S: "1800"
# VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8"
# # The V4 indexer only supports block-outermost layouts; let it pick.
# PREFILLER_KV_LAYOUT: ""
# DECODER_KV_LAYOUT: ""
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
# mirror:
# amd:
# label: ":amd: (MI355) DSv4-Flash Disaggregated DP EP"
# dind: false
# device: mi355_8
# timeout_in_minutes: 85
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - requirements/kv_connectors_rocm.txt
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/model_executor/layers/fused_moe/
# - vllm/model_executor/layers/quantization/
# - vllm/models/deepseek_v4/
# - vllm/v1/attention/
# - tests/v1/kv_connector/nixl_integration/
# - vllm/_aiter_ops.py
# - vllm/platforms/rocm.py
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# # A mirror does not inherit the parent env, so it is repeated here in
# # the parent's order. The layouts must stay empty: unset, the script
# # defaults to HND, which the V4 indexer rejects.
# - >-
# VLLM_ROCM_USE_AITER=1
# ENABLE_HMA_FLAG=1
# DP_EP=1
# GPU_MEMORY_UTILIZATION=0.85
# PREFILLER_TP_SIZE=4
# DECODER_TP_SIZE=4
# PREFILL_BLOCK_SIZE=256
# DECODE_BLOCK_SIZE=256
# MODEL_NAMES=deepseek-ai/DeepSeek-V4-Flash
# VLLM_ENGINE_READY_TIMEOUT_S=1800
# VLLM_SERVE_EXTRA_ARGS=--trust-remote-code,--kv-cache-dtype,fp8
# PREFILLER_KV_LAYOUT=
# DECODER_KV_LAYOUT=
# bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
- label: ":nvidia: (H200) [Nightly] Kimi-Linear-48B-A3B Disaggregated DP EP"
key: kimi-linear-disaggregated
timeout_in_minutes: 60
device: h200
optional: true
working_dir: "/vllm-workspace/tests"
num_devices: 4
env:
ENABLE_HMA_FLAG: "1"
DP_EP: "1"
GPU_MEMORY_UTILIZATION: "0.9"
VLLM_SSM_CONV_STATE_LAYOUT: "DS"
PREFILLER_TP_SIZE: "2"
DECODER_TP_SIZE: "2"
MODEL_NAMES: "moonshotai/Kimi-Linear-48B-A3B-Instruct"
# Keep TP=2 prefill and TP=1 decode above both hybrid page-size minima.
PREFILL_BLOCK_SIZE: "2048"
DECODE_BLOCK_SIZE: "2048"
VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--max-model-len 32768"
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Kimi-Linear-48B-A3B Disaggregated DP EP"
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 105
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/platforms/rocm.py
- vllm/models/kimi_k3/
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- requirements/kv_connectors_rocm.txt
env:
VLLM_WORKER_MULTIPROC_METHOD: "spawn"
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (H200) [Nightly] Kimi-Linear-48B-A3B Disaggregated DP EP"
# key: kimi-linear-disaggregated
# timeout_in_minutes: 60
# device: h200
# optional: true
# working_dir: "/vllm-workspace/tests"
# num_devices: 4
# env:
# ENABLE_HMA_FLAG: "1"
# DP_EP: "1"
# GPU_MEMORY_UTILIZATION: "0.9"
# VLLM_SSM_CONV_STATE_LAYOUT: "DS"
# PREFILLER_TP_SIZE: "2"
# DECODER_TP_SIZE: "2"
# MODEL_NAMES: "moonshotai/Kimi-Linear-48B-A3B-Instruct"
# # Keep TP=2 prefill and TP=1 decode above both hybrid page-size minima.
# PREFILL_BLOCK_SIZE: "2048"
# DECODE_BLOCK_SIZE: "2048"
# VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--max-model-len 32768"
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
# mirror:
# amd:
# label: ":amd: (MI355) Kimi-Linear-48B-A3B Disaggregated DP EP"
# dind: false
# device: mi355_4
# num_devices: 4
# timeout_in_minutes: 105
# working_dir: /vllm-workspace/tests
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/platforms/rocm.py
# - vllm/models/kimi_k3/
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - tests/v1/kv_connector/nixl_integration/
# - requirements/kv_connectors_rocm.txt
# env:
# VLLM_WORKER_MULTIPROC_METHOD: "spawn"
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# - bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
.buildkite/test_areas/disaggregated_mooncake.yaml
+45 -44
View File
@@ -2,50 +2,51 @@ group: Disaggregated Mooncake
depends_on:
- image-build
steps:
- label: ":nvidia: (B200) [Nightly] Distributed MooncakeConnector PD accuracy"
key: distributed-mooncakeconnector-pd-accuracy-4-gpus
# Nightly only: 4-GPU B200 pods wait hours behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 10
working_dir: "/vllm-workspace/tests"
# Runs on b200-k8s because it is the only NVIDIA pool that can carry mooncake
# transfers: its pods are the only ones granted IPC_LOCK, which RDMA needs to
# pin memory, and the only ones using host networking. Elsewhere topology
# discovery finds no usable HCA and the tcp fallback then advertises the
# docker bridge address, so every transfer fails to connect.
device: b200-k8s
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
mirror:
amd:
label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
optional: true
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
- tests/v1/kv_connector/mooncake_integration/
- tests/v1/kv_connector/rocm_pd_accuracy_utils.py
- tests/utils.py
- examples/disaggregated/mooncake_connector/
- vllm/v1/attention/backends/rocm_aiter_unified_attn.py
- vllm/platforms/rocm.py
- requirements/common.txt
- requirements/kv_connectors_rocm.txt
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# Same full accuracy/TP sweep, using same-host HIP GPU transport.
- pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (B200) [Nightly] Distributed MooncakeConnector PD accuracy"
# key: distributed-mooncakeconnector-pd-accuracy-4-gpus
# # Nightly only: 4-GPU B200 pods wait hours behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 10
# working_dir: "/vllm-workspace/tests"
# # Runs on b200-k8s because it is the only NVIDIA pool that can carry mooncake
# # transfers: its pods are the only ones granted IPC_LOCK, which RDMA needs to
# # pin memory, and the only ones using host networking. Elsewhere topology
# # discovery finds no usable HCA and the tcp fallback then advertises the
# # docker bridge address, so every transfer fails to connect.
# device: b200-k8s
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
# - tests/v1/kv_connector/mooncake_integration/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - bash v1/kv_connector/mooncake_integration/config_sweep_accuracy_test.sh
# mirror:
# amd:
# label: ":amd: (MI355) Distributed MooncakeConnector HIP PD accuracy"
# optional: true
# dind: false
# device: mi355_4
# num_devices: 4
# timeout_in_minutes: 30
# working_dir: "/vllm-workspace/tests"
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/
# - tests/v1/kv_connector/mooncake_integration/
# - tests/v1/kv_connector/rocm_pd_accuracy_utils.py
# - tests/utils.py
# - examples/disaggregated/mooncake_connector/
# - vllm/v1/attention/backends/rocm_aiter_unified_attn.py
# - vllm/platforms/rocm.py
# - requirements/common.txt
# - requirements/kv_connectors_rocm.txt
# commands:
# - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
# # Same full accuracy/TP sweep, using same-host HIP GPU transport.
# - pytest -v -s v1/kv_connector/mooncake_integration/test_accuracy_rocm.py
- label: ":nvidia: (L4) Mooncake EC TCP E2E"
key: mooncake-ec-tcp-e2e-2-gpus
.buildkite/test_areas/distributed.yaml
+83 -82
View File
@@ -223,88 +223,89 @@ steps:
- HIP_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
- pytest -v -s v1/worker/test_worker_memory_snapshot.py
- label: ":nvidia: (L4) [Nightly] Distributed Torchrun + Examples"
key: distributed-torchrun-examples-4-gpus
# Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 45
working_dir: "/vllm-workspace"
device: l4
num_devices: 4
source_file_dependencies:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- tests/distributed/test_torchrun_example.py
- tests/distributed/test_torchrun_example_moe.py
- examples/rl/
- examples/features/data_parallel/data_parallel_offline.py
commands:
# https://github.com/NVIDIA/nccl/issues/1838
- export NCCL_CUMEM_HOST_ENABLE=0
# test with torchrun tp=2 and external_dp=2
- torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
# test with torchrun tp=2 and pp=2
- PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
# test with torchrun tp=4 and dp=1
- TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# test with torchrun tp=2, pp=2 and dp=1
- PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# test with torchrun tp=1 and dp=4 with ep
- DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# test with torchrun tp=2 and dp=2 with ep
- TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# test with internal dp
- python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
# rlhf examples (each launches its own `vllm serve` and tears it down)
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
# sharded RDT: 2 FSDP2 trainer ranks -> 2 vLLM DP ranks with EP. Its data
# plane is NIXL, which ships only in the kv-connector set, and the script
# also drops the nixl-cu wheel variant that does not match this image. Kept
# last so the commands above run in an unmodified environment. RDT_MODEL swaps
# the script's reference MoE for a tiny one: no weights are downloaded either
# way, but every trainer rank builds the model before sharding it, and the
# default does not fit an L4.
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
mirror:
amd:
# Run on PRs even when the L4 step is optional (nightly only).
optional: false
label: ":amd: (MI355) Distributed Torchrun + Examples"
dind: false
device: mi355_4
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- tests/distributed/test_torchrun_example.py
- tests/distributed/test_torchrun_example_moe.py
- examples/rl/
- examples/features/data_parallel/data_parallel_offline.py
- .buildkite/scripts/install-kv-connectors.sh
- requirements/kv_connectors_rocm.txt
- requirements/test/rocm.in
- requirements/test/rocm.txt
- vllm/platforms/rocm.py
commands:
# Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies.
- uv pip install --system --no-deps 'ray==2.56.1'
- export NCCL_CUMEM_HOST_ENABLE=0
- torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
- PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
- TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
- TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# Loading weights from the shared HF cache can exceed the default 300s --timeout.
- python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager --timeout 900
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (L4) [Nightly] Distributed Torchrun + Examples"
# key: distributed-torchrun-examples-4-gpus
# # Nightly only: 4-GPU L4 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 45
# working_dir: "/vllm-workspace"
# device: l4
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/
# - "!vllm/distributed/kv_transfer/"
# - tests/distributed/test_torchrun_example.py
# - tests/distributed/test_torchrun_example_moe.py
# - examples/rl/
# - examples/features/data_parallel/data_parallel_offline.py
# commands:
# # https://github.com/NVIDIA/nccl/issues/1838
# - export NCCL_CUMEM_HOST_ENABLE=0
# # test with torchrun tp=2 and external_dp=2
# - torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
# # test with torchrun tp=2 and pp=2
# - PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
# # test with torchrun tp=4 and dp=1
# - TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# # test with torchrun tp=2, pp=2 and dp=1
# - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# # test with torchrun tp=1 and dp=4 with ep
# - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# # test with torchrun tp=2 and dp=2 with ep
# - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# # test with internal dp
# - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
# # rlhf examples (each launches its own `vllm serve` and tears it down)
# - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
# - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
# # sharded RDT: 2 FSDP2 trainer ranks -> 2 vLLM DP ranks with EP. Its data
# # plane is NIXL, which ships only in the kv-connector set, and the script
# # also drops the nixl-cu wheel variant that does not match this image. Kept
# # last so the commands above run in an unmodified environment. RDT_MODEL swaps
# # the script's reference MoE for a tiny one: no weights are downloaded either
# # way, but every trainer rank builds the model before sharding it, and the
# # default does not fit an L4.
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
# mirror:
# amd:
# # Run on PRs even when the L4 step is optional (nightly only).
# optional: false
# label: ":amd: (MI355) Distributed Torchrun + Examples"
# dind: false
# device: mi355_4
# timeout_in_minutes: 55
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/
# - "!vllm/distributed/kv_transfer/"
# - tests/distributed/test_torchrun_example.py
# - tests/distributed/test_torchrun_example_moe.py
# - examples/rl/
# - examples/features/data_parallel/data_parallel_offline.py
# - .buildkite/scripts/install-kv-connectors.sh
# - requirements/kv_connectors_rocm.txt
# - requirements/test/rocm.in
# - requirements/test/rocm.txt
# - vllm/platforms/rocm.py
# commands:
# # Sharded RDT needs Ray >= 2.56; keep the ROCm image's dependencies.
# - uv pip install --system --no-deps 'ray==2.56.1'
# - export NCCL_CUMEM_HOST_ENABLE=0
# - torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
# - PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
# - TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
# # Loading weights from the shared HF cache can exceed the default 300s --timeout.
# - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager --timeout 900
# - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
# - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
- label: ":nvidia: (L4) [Nightly] Distributed DP Extended"
key: distributed-dp-tests-4-gpus
.buildkite/test_areas/expert_parallelism.yaml
+19 -18
View File
@@ -60,21 +60,22 @@ steps:
- tests/distributed/test_eplb_spec_decode.py
- vllm/platforms/rocm.py
- label: ":nvidia: (H100) [Nightly] Elastic EP Scaling"
key: elastic-ep-scaling-test
# Nightly only: 4-GPU H100 pods wait behind smaller jobs on per-PR runs.
optional: true
timeout_in_minutes: 40
device: h100
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/v1/executor/
- vllm/compilation/
- tests/distributed/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- pytest -v -s distributed/test_elastic_ep.py
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (H100) [Nightly] Elastic EP Scaling"
# key: elastic-ep-scaling-test
# # Nightly only: 4-GPU H100 pods wait behind smaller jobs on per-PR runs.
# optional: true
# timeout_in_minutes: 40
# device: h100
# working_dir: "/vllm-workspace/tests"
# num_devices: 4
# source_file_dependencies:
# - vllm/distributed/
# - "!vllm/distributed/kv_transfer/"
# - vllm/engine/
# - vllm/v1/executor/
# - vllm/compilation/
# - tests/distributed/
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - pytest -v -s distributed/test_elastic_ep.py
.buildkite/test_areas/fault_tolerance.yaml
+82 -41
View File
@@ -2,13 +2,85 @@ group: Fault Tolerance
depends_on:
- image-build
steps:
- label: ":nvidia: (H100) Fault Tolerance E2E"
key: fault-tolerance-e2e-2xh100
timeout_in_minutes: 35
device: h100
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (H100) Fault Tolerance E2E"
# key: fault-tolerance-e2e-2xh100
# timeout_in_minutes: 35
# device: h100
# num_devices: 2
# working_dir: "/vllm-workspace/tests"
# source_file_dependencies:
# - vllm/v1/fault_tolerance/
# - vllm/v1/worker/sentinel/
# - vllm/entrypoints/serve/fault_tolerance/
# - vllm/distributed/elastic_ep/
# - vllm/distributed/device_communicators/
# - vllm/distributed/kv_transfer/
# - vllm/v1/engine/
# - vllm/v1/executor/
# - vllm/v1/worker/
# - tests/v1/fault_tolerance/
# - tests/v1/distributed/test_external_lb_dp.py
# commands:
# # Base image has no nixl; install it or has_nixl_ep() skips the tests.
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# # https://github.com/NVIDIA/nccl/issues/1838
# - export NCCL_CUMEM_HOST_ENABLE=0
# - pytest -v -s v1/fault_tolerance/test_engine_core_sentinel.py
# - pytest -v -s v1/fault_tolerance/test_fault_tolerance_e2e.py
# mirror:
# amd:
# label: ":amd: (MI355) MoRI EP Healthy Baseline"
# dind: false
# device: mi355_2
# num_devices: 2
# timeout_in_minutes: 10
# working_dir: "/vllm-workspace/tests"
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - csrc/moe/
# - csrc/rocm/
# - tests/kernels/moe/
# - vllm/model_executor/layers/fused_moe/
# - vllm/model_executor/layers/quantization/
# - vllm/distributed/
# - vllm/config/
# - vllm/forward_context.py
# - vllm/v1/worker/workspace.py
# - vllm/utils/import_utils.py
# - vllm/utils/math_utils.py
# - vllm/utils/torch_utils.py
# - vllm/platforms/
# - vllm/_aiter_ops.py
# # TODO: Add fault injection, detection and recovery once MoRI supports them.
# # This mirror currently checks healthy EP execution only.
# commands:
# - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs kernels/moe/test_moe_layer.py::test_moe_layer_mori_graph
# - VLLM_TEST_ENABLE_MORI_MOE_LAYER=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs 'kernels/moe/test_moe_layer.py::test_moe_layer[False-mori_high_throughput-2-1-True]' --subtests='[32-512-512-8-2-bfloat16-None-False-False-False-False-mori_high_throughput-2-2-1]'
# Keep the AMD job: its commands do not run this installation.
- label: ':amd: (MI355) MoRI EP Healthy Baseline'
key: amd-fault-tolerance-e2e-2xh100
timeout_in_minutes: 10
device: mi355_2
num_devices: 2
working_dir: "/vllm-workspace/tests"
working_dir: /vllm-workspace/tests
source_file_dependencies:
- csrc/moe/
- csrc/rocm/
- tests/kernels/moe/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/distributed/
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/utils/import_utils.py
- vllm/utils/math_utils.py
- vllm/utils/torch_utils.py
- vllm/platforms/
- vllm/_aiter_ops.py
- vllm/v1/fault_tolerance/
- vllm/v1/worker/sentinel/
- vllm/entrypoints/serve/fault_tolerance/
@@ -21,39 +93,8 @@ steps:
- tests/v1/fault_tolerance/
- tests/v1/distributed/test_external_lb_dp.py
commands:
# Base image has no nixl; install it or has_nixl_ep() skips the tests.
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# https://github.com/NVIDIA/nccl/issues/1838
- export NCCL_CUMEM_HOST_ENABLE=0
- pytest -v -s v1/fault_tolerance/test_engine_core_sentinel.py
- pytest -v -s v1/fault_tolerance/test_fault_tolerance_e2e.py
mirror:
amd:
label: ":amd: (MI355) MoRI EP Healthy Baseline"
dind: false
device: mi355_2
num_devices: 2
timeout_in_minutes: 10
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/moe/
- csrc/rocm/
- tests/kernels/moe/
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/
- vllm/distributed/
- vllm/config/
- vllm/forward_context.py
- vllm/v1/worker/workspace.py
- vllm/utils/import_utils.py
- vllm/utils/math_utils.py
- vllm/utils/torch_utils.py
- vllm/platforms/
- vllm/_aiter_ops.py
# TODO: Add fault injection, detection and recovery once MoRI supports them.
# This mirror currently checks healthy EP execution only.
commands:
- VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs kernels/moe/test_moe_layer.py::test_moe_layer_mori_graph
- VLLM_TEST_ENABLE_MORI_MOE_LAYER=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs 'kernels/moe/test_moe_layer.py::test_moe_layer[False-mori_high_throughput-2-1-True]' --subtests='[32-512-512-8-2-bfloat16-None-False-False-False-False-mori_high_throughput-2-2-1]'
- VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs kernels/moe/test_moe_layer.py::test_moe_layer_mori_graph
- VLLM_TEST_ENABLE_MORI_MOE_LAYER=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -rs 'kernels/moe/test_moe_layer.py::test_moe_layer[False-mori_high_throughput-2-1-True]' --subtests='[32-512-512-8-2-bfloat16-None-False-False-False-False-mori_high_throughput-2-2-1]'
dind: false
depends_on:
- image-build-amd
.buildkite/test_areas/misc.yaml
+425 -334
View File
@@ -144,51 +144,53 @@ steps:
depends_on:
- image-build-amd
- label: ":nvidia: (H200 MIG 35GB) V1 KV Connectors"
device: h200_35gb
key: v1-kv-connectors
automatic_shard: true
timeout_in_minutes: 45
source_file_dependencies: *v1-core-kv-metrics-deps
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
- pytest -v -s -m 'not cpu_test' v1/ec_connector/unit
mirror:
amd:
label: ":amd: (MI355 DPX) V1 KV Connectors"
automatic_shard: true
dind: false
device: mi355_dpx
timeout_in_minutes: 60
depends_on:
- image-build-amd
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (H200 MIG 35GB) V1 KV Connectors"
# device: h200_35gb
# key: v1-kv-connectors
# automatic_shard: true
# timeout_in_minutes: 45
# source_file_dependencies: *v1-core-kv-metrics-deps
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
# - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit
# mirror:
# amd:
# label: ":amd: (MI355 DPX) V1 KV Connectors"
# automatic_shard: true
# dind: false
# device: mi355_dpx
# timeout_in_minutes: 60
# depends_on:
# - image-build-amd
# The shared KV Connectors steps above exclude vllm/distributed/kv_transfer/ from
# their triggers, and CUDA skips MoRIIO's mori/aiter-gated tests. Run the whole
# MoRIIO suite on ROCm whenever the connector or its tests change.
- label: ":amd: (MI355 DPX) MoRIIO Connector Unit Tests"
key: amd-moriio-connector-unit-mi355
dind: false
device: mi355_dpx
num_devices: 1
timeout_in_minutes: 30
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/moriio/
- vllm/distributed/kv_transfer/kv_connector/v1/base.py
- vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
- examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
- tests/v1/kv_connector/unit/test_moriio_
- tests/v1/kv_connector/unit/utils.py
- requirements/kv_connectors_rocm.txt
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s -rs v1/kv_connector/unit/test_moriio_*.py
# Disabled until the KV connectors installation is locked.
# - label: ":amd: (MI355 DPX) MoRIIO Connector Unit Tests"
# key: amd-moriio-connector-unit-mi355
# dind: false
# device: mi355_dpx
# num_devices: 1
# timeout_in_minutes: 30
# working_dir: /vllm-workspace/tests
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
# - vllm/distributed/kv_transfer/kv_connector/v1/base.py
# - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
# - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
# - tests/v1/kv_connector/unit/test_moriio_
# - tests/v1/kv_connector/unit/utils.py
# - requirements/kv_connectors_rocm.txt
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pytest -v -s -rs v1/kv_connector/unit/test_moriio_*.py
# UMBP is built on mori, so its tests skip on CUDA. Run them on MI355 whenever
# the connector or its tests change.
@@ -213,55 +215,57 @@ steps:
# Single-node MoRIIO P/D on dummy weights, WRITE and READ mode: prefill and
# decode on separate GPUs over xGMI, compared exactly against a direct serve.
- label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight E2E Shard %N"
key: amd-moriio-write-dummy-e2e-mi355
dind: false
device: mi355_2
num_devices: 2
# ~3.5-5 min per case; pytest-shard hashes test ids, so with 28 cases the
# largest of 6 shards holds 7 (~32 min on MI355). Every case picks free ports
# for its servers and proxy and its own compile-cache dir.
parallelism: 6
timeout_in_minutes: 45
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/moriio/
- vllm/distributed/kv_transfer/kv_connector/v1/base.py
- vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
- vllm/model_executor/model_loader/dummy_loader.py
- examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
- tests/v1/kv_connector/moriio_integration/
- requirements/kv_connectors_rocm.txt
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k "not heterogeneous_tp" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
# Disabled until the KV connectors installation is locked.
# - label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight E2E Shard %N"
# key: amd-moriio-write-dummy-e2e-mi355
# dind: false
# device: mi355_2
# num_devices: 2
# # ~3.5-5 min per case; pytest-shard hashes test ids, so with 28 cases the
# # largest of 6 shards holds 7 (~32 min on MI355). Every case picks free ports
# # for its servers and proxy and its own compile-cache dir.
# parallelism: 6
# timeout_in_minutes: 45
# working_dir: /vllm-workspace/tests
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
# - vllm/distributed/kv_transfer/kv_connector/v1/base.py
# - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
# - vllm/model_executor/model_loader/dummy_loader.py
# - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
# - tests/v1/kv_connector/moriio_integration/
# - requirements/kv_connectors_rocm.txt
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k "not heterogeneous_tp" --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
# The same P/D between different TP sizes (WRITE TP2 -> TP1, READ of MLA
# TP1 <-> TP2), which needs 3 GPUs: 8 cases, ~36 min on MI355.
- label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight Heterogeneous TP"
key: amd-moriio-write-dummy-hetero-tp-mi355
dind: false
device: mi355_4
num_devices: 4
timeout_in_minutes: 45
working_dir: /vllm-workspace/tests
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/moriio/
- vllm/distributed/kv_transfer/kv_connector/v1/base.py
- vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
- vllm/model_executor/model_loader/dummy_loader.py
- examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
- tests/v1/kv_connector/moriio_integration/
- requirements/kv_connectors_rocm.txt
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k heterogeneous_tp
# Disabled until the KV connectors installation is locked.
# - label: ":amd: (MI355) MoRIIO WRITE P/D Dummy-Weight Heterogeneous TP"
# key: amd-moriio-write-dummy-hetero-tp-mi355
# dind: false
# device: mi355_4
# num_devices: 4
# timeout_in_minutes: 45
# working_dir: /vllm-workspace/tests
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/distributed/kv_transfer/kv_connector/v1/moriio/
# - vllm/distributed/kv_transfer/kv_connector/v1/base.py
# - vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py
# - vllm/model_executor/model_loader/dummy_loader.py
# - examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py
# - tests/v1/kv_connector/moriio_integration/
# - requirements/kv_connectors_rocm.txt
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pytest -v -s -rs v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py -k heterogeneous_tp
- label: ":nvidia: (H200 MIG 35GB) V1 Metrics + LM Eval"
device: h200_35gb
@@ -409,37 +413,38 @@ steps:
- tests/v1/kv_connector/extract_hidden_states_integration/
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 18GB) Regression"
key: regression
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/config/
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/inputs/
- vllm/model_executor/
- vllm/multimodal/
- vllm/platforms/
- vllm/sampling_params.py
- vllm/transformers_utils/
- vllm/utils/
- vllm/v1/
- tests/test_regression
commands:
- pip install 'modelscope<1.38'
- pytest -v -s test_regression.py
working_dir: "/vllm-workspace/tests" # optional
mirror:
amd:
label: ":amd: (MI355 DPX) Regression"
dind: false
device: mi355_dpx
timeout_in_minutes: 25
depends_on:
- image-build-amd
working_dir: /vllm-workspace/tests
# Disabled until the modelscope installation is locked.
# - label: ":nvidia: (H200 MIG 18GB) Regression"
# key: regression
# timeout_in_minutes: 30
# device: h200_18gb
# source_file_dependencies:
# - vllm/config/
# - vllm/distributed/
# - "!vllm/distributed/kv_transfer/"
# - vllm/engine/
# - vllm/inputs/
# - vllm/model_executor/
# - vllm/multimodal/
# - vllm/platforms/
# - vllm/sampling_params.py
# - vllm/transformers_utils/
# - vllm/utils/
# - vllm/v1/
# - tests/test_regression
# commands:
# - pip install 'modelscope<1.38'
# - pytest -v -s test_regression.py
# working_dir: "/vllm-workspace/tests" # optional
# mirror:
# amd:
# label: ":amd: (MI355 DPX) Regression"
# dind: false
# device: mi355_dpx
# timeout_in_minutes: 25
# depends_on:
# - image-build-amd
# working_dir: /vllm-workspace/tests
- label: ":nvidia: (H200 MIG 35GB) Examples"
device: h200_35gb
@@ -487,43 +492,44 @@ steps:
depends_on:
- image-build-amd
- label: ":nvidia: (L4) Metrics, Tracing"
key: metrics-tracing-2-gpus
timeout_in_minutes: 25
device: l4
num_devices: 2
source_file_dependencies:
- vllm/config/
- vllm/distributed/
- "!vllm/distributed/kv_transfer/"
- vllm/engine/
- vllm/inputs/
- vllm/model_executor/
- vllm/multimodal/
- vllm/platforms/
- vllm/sampling_params.py
- vllm/tracing/
- vllm/transformers_utils/
- vllm/utils/
- vllm/v1/
- tests/v1/tracing
- tests/tracing/
commands:
- "pip install \
'opentelemetry-sdk>=1.26.0' \
'opentelemetry-api>=1.26.0' \
'opentelemetry-exporter-otlp>=1.26.0' \
'opentelemetry-semantic-conventions-ai>=0.4.1'"
- pytest -v -s v1/tracing
- pytest -v -s tracing
mirror:
amd:
label: ":amd: (MI300) Metrics, Tracing"
dind: false
device: mi300_2
timeout_in_minutes: 30
depends_on:
- image-build-amd
# Disabled until the tracing packages installation is locked.
# - label: ":nvidia: (L4) Metrics, Tracing"
# key: metrics-tracing-2-gpus
# timeout_in_minutes: 25
# device: l4
# num_devices: 2
# source_file_dependencies:
# - vllm/config/
# - vllm/distributed/
# - "!vllm/distributed/kv_transfer/"
# - vllm/engine/
# - vllm/inputs/
# - vllm/model_executor/
# - vllm/multimodal/
# - vllm/platforms/
# - vllm/sampling_params.py
# - vllm/tracing/
# - vllm/transformers_utils/
# - vllm/utils/
# - vllm/v1/
# - tests/v1/tracing
# - tests/tracing/
# commands:
# - "pip install \
# 'opentelemetry-sdk>=1.26.0' \
# 'opentelemetry-api>=1.26.0' \
# 'opentelemetry-exporter-otlp>=1.26.0' \
# 'opentelemetry-semantic-conventions-ai>=0.4.1'"
# - pytest -v -s v1/tracing
# - pytest -v -s tracing
# mirror:
# amd:
# label: ":amd: (MI300) Metrics, Tracing"
# dind: false
# device: mi300_2
# timeout_in_minutes: 30
# depends_on:
# - image-build-amd
- label: ":nvidia: (H200 MIG 18GB) [Nightly] Python-only Installation"
key: python-only-installation
@@ -601,8 +607,129 @@ steps:
- pytest -v -s multimodal/test_cache.py::test_sleep_wake_preserves_mm_cache_consistency
- pytest -v -s utils_
- label: ":computer: (CPU) Params, Env, Tokenizers, Parser"
key: cpu-params-env-tokenizers-parser
# Disabled until the cmake installation is locked.
# - label: ":computer: (CPU) Params, Env, Tokenizers, Parser"
# key: cpu-params-env-tokenizers-parser
# depends_on:
# - image-build-cpu
# timeout_in_minutes: 35
# source_file_dependencies: &cpu-async-engine-deps
# - vllm/assets/
# - vllm/config/
# - vllm/device_allocator/
# - vllm/connections.py
# - vllm/engine/arg_utils.py
# - vllm/renderers/chat_utils/
# - vllm/entrypoints/mcp/
# - vllm/entrypoints/openai/chat_completion/protocol.py
# - vllm/entrypoints/openai/completion/protocol.py
# - vllm/entrypoints/openai/responses/protocol.py
# - vllm/entrypoints/generate/base/protocol.py
# - vllm/env_override.py
# - vllm/envs.py
# - vllm/exceptions.py
# - vllm/inputs/
# - vllm/logging_utils/
# - vllm/logprobs.py
# - vllm/model_executor/layers/quantization/quark/
# - vllm/model_executor/model_loader/tensorizer.py
# - vllm/models/muse_spark/
# - vllm/multimodal/
# - vllm/outputs.py
# - vllm/parser/
# - vllm/platforms/
# - vllm/pooling_params.py
# - vllm/ray/
# - vllm/reasoning/
# - vllm/renderers/
# - vllm/sampling_params.py
# - vllm/scalar_type.py
# - vllm/tokenizers/
# - vllm/tool_parsers/
# - vllm/transformers_utils/
# - vllm/triton_utils/
# - vllm/utils/
# - vllm/v1/
# - vllm/version.py
# - cmake/utils.cmake
# - cmake/hipify.py
# - tests/test_uvicorn_logging.py
# - tests/test_attention_backend_registry.py
# - tests/test_audio_media_size_precheck.py
# - tests/test_cmake_utils.py
# - tests/test_connections.py
# - tests/test_embedded_commit.py
# - tests/test_envs.py
# - tests/test_force_first_config.py
# - tests/test_fxgraphcache_pickle_patch.py
# - tests/test_logprobs.py
# - tests/test_outputs.py
# - tests/test_pcp_dp.py
# - tests/test_pooling_params.py
# - tests/test_quantization_revision_pin.py
# - tests/test_ray_env.py
# - tests/test_ray_env_utils.py
# - tests/test_request_input_bounds.py
# - tests/test_sampling_params.py
# - tests/test_scalartype.py
# - tests/test_tensorizer_revision_pin.py
# - tests/test_triton_utils.py
# - tests/test_version.py
# - tests/test_zen_cpu_platform_detection.py
# - tests/multimodal
# - tests/renderers
# - tests/standalone_tests/lazy_imports.py
# - tests/standalone_tests/test_precompiled_wheel.py
# - setup.py
# - tests/reasoning
# - tools/pre_commit/check_test_tethering.py
# - tools/pre_commit/test_tethering_allowlist.txt
# - tests/tools/test_check_test_tethering.py
# - tools/pre_commit/check_agent_files.py
# - tests/tools/test_check_agent_files.py
# - tools/pre_commit/check_trust_remote_code.py
# - tests/tools/test_check_trust_remote_code.py
# - tests/tool_parsers
# - tests/tokenizers_
# - tests/parser
# - tests/transformers_utils
# - tests/config
# - tests/device_allocator
# device: cpu-small
# commands:
# - pip install cmake # test_cmake_utils.py shells out to cmake; the CPU image has none
# - python3 standalone_tests/lazy_imports.py
# - python3 standalone_tests/test_precompiled_wheel.py
# - pytest -v -s device_allocator
# - pytest -v -s test_uvicorn_logging.py
# - pytest -v -s test_attention_backend_registry.py
# - pytest -v -s test_audio_media_size_precheck.py
# - pytest -v -s test_cmake_utils.py
# - pytest -v -s test_connections.py
# - pytest -v -s test_embedded_commit.py
# - pytest -v -s test_envs.py
# - pytest -v -s test_force_first_config.py
# - pytest -v -s test_fxgraphcache_pickle_patch.py
# - pytest -v -s test_logprobs.py
# - pytest -v -s test_outputs.py
# - pytest -v -s test_pcp_dp.py
# - pytest -v -s test_pooling_params.py
# - pytest -v -s test_quantization_revision_pin.py
# - pytest -v -s test_ray_env.py
# - pytest -v -s test_ray_env_utils.py
# - pytest -v -s test_request_input_bounds.py
# - pytest -v -s test_sampling_params.py
# - pytest -v -s test_scalartype.py
# - pytest -v -s test_tensorizer_revision_pin.py
# - pytest -v -s test_triton_utils.py
# - pytest -v -s test_version.py
# - pytest -v -s test_zen_cpu_platform_detection.py
# - pytest -v -s tokenizers_
# - pytest -v -s parser
# - pytest -v -s transformers_utils
- label: ":computer: (CPU) Multimodal + Config"
key: cpu-multimodal-config
depends_on:
- image-build-cpu
timeout_in_minutes: 35
@@ -690,45 +817,6 @@ steps:
- tests/device_allocator
device: cpu-small
commands:
- pip install cmake # test_cmake_utils.py shells out to cmake; the CPU image has none
- python3 standalone_tests/lazy_imports.py
- python3 standalone_tests/test_precompiled_wheel.py
- pytest -v -s device_allocator
- pytest -v -s test_uvicorn_logging.py
- pytest -v -s test_attention_backend_registry.py
- pytest -v -s test_audio_media_size_precheck.py
- pytest -v -s test_cmake_utils.py
- pytest -v -s test_connections.py
- pytest -v -s test_embedded_commit.py
- pytest -v -s test_envs.py
- pytest -v -s test_force_first_config.py
- pytest -v -s test_fxgraphcache_pickle_patch.py
- pytest -v -s test_logprobs.py
- pytest -v -s test_outputs.py
- pytest -v -s test_pcp_dp.py
- pytest -v -s test_pooling_params.py
- pytest -v -s test_quantization_revision_pin.py
- pytest -v -s test_ray_env.py
- pytest -v -s test_ray_env_utils.py
- pytest -v -s test_request_input_bounds.py
- pytest -v -s test_sampling_params.py
- pytest -v -s test_scalartype.py
- pytest -v -s test_tensorizer_revision_pin.py
- pytest -v -s test_triton_utils.py
- pytest -v -s test_version.py
- pytest -v -s test_zen_cpu_platform_detection.py
- pytest -v -s tokenizers_
- pytest -v -s parser
- pytest -v -s transformers_utils
- label: ":computer: (CPU) Multimodal + Config"
key: cpu-multimodal-config
depends_on:
- image-build-cpu
timeout_in_minutes: 35
source_file_dependencies: *cpu-async-engine-deps
device: cpu-small
commands:
- pytest -v -s -m 'cpu_test' multimodal
- pytest -v -s config
- pytest -v -s tools/test_check_test_tethering.py
@@ -756,146 +844,149 @@ steps:
commands:
- pytest -v -s tool_parsers
- label: ":nvidia: (A100) Batch Invariance"
key: batch-invariance-a100
timeout_in_minutes: 60
device: a100
source_file_dependencies:
- vllm/v1/attention
- vllm/model_executor/layers
- vllm/model_executor/kernels/linear
- vllm/model_executor/determinism
- tests/v1/determinism/
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pip install pytest-timeout pytest-forked
- pytest -v -s v1/determinism/test_batch_invariance.py
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
mirror:
amd:
label: ":amd: (MI250) Batch Invariance"
dind: false
device: mi250_1
timeout_in_minutes: 60
env:
VLLM_GPU_MEMORY_UTILIZATION: "0.8"
VLLM_TEST_CLEAN_GPU_MEMORY: "1"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/attention
- vllm/model_executor/layers
- vllm/model_executor/kernels/linear
- vllm/model_executor/determinism
- tests/v1/determinism/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
# Disabled until the pytest plugins installation is locked.
# - label: ":nvidia: (A100) Batch Invariance"
# key: batch-invariance-a100
# timeout_in_minutes: 60
# device: a100
# source_file_dependencies:
# - vllm/v1/attention
# - vllm/model_executor/layers
# - vllm/model_executor/kernels/linear
# - vllm/model_executor/determinism
# - tests/v1/determinism/
# commands:
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pip install pytest-timeout pytest-forked
# - pytest -v -s v1/determinism/test_batch_invariance.py
# - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
# mirror:
# amd:
# label: ":amd: (MI250) Batch Invariance"
# dind: false
# device: mi250_1
# timeout_in_minutes: 60
# env:
# VLLM_GPU_MEMORY_UTILIZATION: "0.8"
# VLLM_TEST_CLEAN_GPU_MEMORY: "1"
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/v1/attention
# - vllm/model_executor/layers
# - vllm/model_executor/kernels/linear
# - vllm/model_executor/determinism
# - tests/v1/determinism/
# - vllm/_aiter_ops.py
# - vllm/platforms/rocm.py
- label: ":nvidia: (H100) Batch Invariance"
key: batch-invariance-h100
automatic_shard: true
timeout_in_minutes: 60
device: h100
source_file_dependencies:
- vllm/v1/attention
- vllm/model_executor/layers
- vllm/lora/ops/triton_ops
- vllm/model_executor/kernels/linear
- vllm/model_executor/determinism
- tests/v1/determinism/
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pip install pytest-timeout pytest-forked
- pytest -v -s v1/determinism/test_batch_invariance.py
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
- pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
mirror:
amd:
label: ":amd: (MI300) Batch Invariance"
dind: false
automatic_shard: true
device: mi300_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/attention
- vllm/model_executor/layers
- vllm/lora/ops/triton_ops
- vllm/model_executor/kernels/linear
- vllm/model_executor/determinism
- tests/v1/determinism/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pip install pytest-timeout pytest-forked
- pytest -v -s v1/determinism/test_batch_invariance.py
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
- pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
- pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
- pytest -v -s v1/determinism/test_matmul_batch_invariant.py
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_ATTN
# Disabled until the pytest plugins installation is locked.
# - label: ":nvidia: (H100) Batch Invariance"
# key: batch-invariance-h100
# automatic_shard: true
# timeout_in_minutes: 60
# device: h100
# source_file_dependencies:
# - vllm/v1/attention
# - vllm/model_executor/layers
# - vllm/lora/ops/triton_ops
# - vllm/model_executor/kernels/linear
# - vllm/model_executor/determinism
# - tests/v1/determinism/
# commands:
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pip install pytest-timeout pytest-forked
# - pytest -v -s v1/determinism/test_batch_invariance.py
# - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
# - pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
# - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
# - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
# mirror:
# amd:
# label: ":amd: (MI300) Batch Invariance"
# dind: false
# automatic_shard: true
# device: mi300_1
# timeout_in_minutes: 60
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/v1/attention
# - vllm/model_executor/layers
# - vllm/lora/ops/triton_ops
# - vllm/model_executor/kernels/linear
# - vllm/model_executor/determinism
# - tests/v1/determinism/
# - vllm/_aiter_ops.py
# - vllm/platforms/rocm.py
# commands:
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pip install pytest-timeout pytest-forked
# - pytest -v -s v1/determinism/test_batch_invariance.py
# - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
# - pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
# - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
# - pytest -v -s v1/determinism/test_matmul_batch_invariant.py
# - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
# - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_ATTN
- label: ":nvidia: (B200) Batch Invariance"
key: batch-invariance-b200
automatic_shard: true
timeout_in_minutes: 45
device: b200-k8s
source_file_dependencies:
- vllm/v1/attention
- vllm/model_executor/layers
- vllm/lora/ops/triton_ops
- vllm/model_executor/kernels/linear
- vllm/model_executor/determinism
- tests/v1/determinism/
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pip install pytest-timeout pytest-forked
- pytest -v -s v1/determinism/test_batch_invariance.py
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
- pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
- pytest -v -s v1/determinism/test_batch_invariance_vlm.py
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
- pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
- pytest -v -s v1/determinism/test_matmul_batch_invariant.py
- pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
- pytest -v -s v1/determinism/test_online_batch_invariance.py
mirror:
amd:
label: ":amd: (MI355) Batch Invariance"
dind: false
device: mi355_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/attention
- vllm/model_executor/layers
- vllm/lora/ops/triton_ops
- vllm/model_executor/kernels/linear
- vllm/model_executor/determinism
- tests/v1/determinism/
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pip install pytest-timeout pytest-forked
- pytest -v -s v1/determinism/test_batch_invariance.py
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
- pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
- pytest -v -s v1/determinism/test_batch_invariance_vlm.py
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_ATTN
- pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
- pytest -v -s v1/determinism/test_matmul_batch_invariant.py
- pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
- pytest -v -s v1/determinism/test_online_batch_invariance.py
- pytest -v -s v1/determinism/test_mx_linear_batch_invariant.py
# Disabled until the pytest plugins installation is locked.
# - label: ":nvidia: (B200) Batch Invariance"
# key: batch-invariance-b200
# automatic_shard: true
# timeout_in_minutes: 45
# device: b200-k8s
# source_file_dependencies:
# - vllm/v1/attention
# - vllm/model_executor/layers
# - vllm/lora/ops/triton_ops
# - vllm/model_executor/kernels/linear
# - vllm/model_executor/determinism
# - tests/v1/determinism/
# commands:
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pip install pytest-timeout pytest-forked
# - pytest -v -s v1/determinism/test_batch_invariance.py
# - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
# - pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
# - pytest -v -s v1/determinism/test_batch_invariance_vlm.py
# - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
# - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
# - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
# - pytest -v -s v1/determinism/test_matmul_batch_invariant.py
# - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
# - pytest -v -s v1/determinism/test_online_batch_invariance.py
# mirror:
# amd:
# label: ":amd: (MI355) Batch Invariance"
# dind: false
# device: mi355_1
# timeout_in_minutes: 60
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/v1/attention
# - vllm/model_executor/layers
# - vllm/lora/ops/triton_ops
# - vllm/model_executor/kernels/linear
# - vllm/model_executor/determinism
# - tests/v1/determinism/
# - vllm/_aiter_ops.py
# - vllm/platforms/rocm.py
# commands:
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pip install pytest-timeout pytest-forked
# - pytest -v -s v1/determinism/test_batch_invariance.py
# - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
# - pytest -v -s v1/determinism/test_lora_shrink_batch_invariant.py
# - pytest -v -s v1/determinism/test_batch_invariance_vlm.py
# - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
# - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_ATTN
# - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
# - pytest -v -s v1/determinism/test_matmul_batch_invariant.py
# - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
# - pytest -v -s v1/determinism/test_online_batch_invariance.py
# - pytest -v -s v1/determinism/test_mx_linear_batch_invariant.py
- label: ":nvidia: (H100) [Nightly] Batch Invariance (Distributed)"
key: batch-invariance-distributed-4xh100
.buildkite/test_areas/model_executor.yaml
+68 -40
View File
@@ -2,11 +2,66 @@ group: Model Executor
depends_on:
- image-build
steps:
- label: ":nvidia: (H200 MIG 35GB) Model Executor"
device: h200_35gb
key: model-executor
# Disabled until the apt packages installation is locked.
# - label: ":nvidia: (H200 MIG 35GB) Model Executor"
# device: h200_35gb
# key: model-executor
# automatic_shard: true
# timeout_in_minutes: 60
# source_file_dependencies:
# - vllm/engine/arg_utils.py
# - vllm/config/model.py
# - vllm/model_executor
# - vllm/model_executor/warmup
# - tests/model_executor
# - tests/model_executor/test_jit_warmup.py
# - tests/model_executor/test_jit_warmup_cutedsl_launcher.py
# - tests/model_executor/test_jit_warmup_triton_launcher.py
# - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
# commands:
# - apt-get update && apt-get install -y curl libsodium23
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# # Dump tracebacks of all threads if a test hangs, so a wedged GPU/CUDA
# # init surfaces a stack instead of silently stalling.
# - export PYTHONFAULTHANDLER=1
# # Per-test watchdog: a single hung test (e.g. stuck during engine/CUDA
# # init) fails fast with a traceback instead of running until the global
# # build timeout. The `thread` method also handles hangs inside C/CUDA
# # calls that the signal method cannot interrupt.
# - pytest -v -s model_executor -m '(not slow_test) and (not distributed)' --timeout=900 --timeout-method=thread
# - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
# mirror:
# amd:
# label: ":amd: (MI300) Model Executor"
# dind: false
# device: mi300_1
# timeout_in_minutes: 75
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/engine/arg_utils.py
# - vllm/config/model.py
# - vllm/model_executor
# - vllm/model_executor/warmup
# - tests/model_executor
# - tests/model_executor/test_jit_warmup.py
# - tests/model_executor/test_jit_warmup_cutedsl_launcher.py
# - tests/model_executor/test_jit_warmup_triton_launcher.py
# - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
# - vllm/_aiter_ops.py
# - vllm/platforms/rocm.py
# commands:
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - export PYTHONFAULTHANDLER=1
# - pytest -v -s model_executor -m '(not slow_test) and (not distributed)' --timeout=900 --timeout-method=thread
# - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
# Keep the AMD job: its commands do not run this installation.
- label: ':amd: (MI300) Model Executor'
device: mi300_1
key: amd-model-executor
automatic_shard: true
timeout_in_minutes: 60
timeout_in_minutes: 75
source_file_dependencies:
- vllm/engine/arg_utils.py
- vllm/config/model.py
@@ -17,40 +72,13 @@ steps:
- tests/model_executor/test_jit_warmup_cutedsl_launcher.py
- tests/model_executor/test_jit_warmup_triton_launcher.py
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- apt-get update && apt-get install -y curl libsodium23
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
# Dump tracebacks of all threads if a test hangs, so a wedged GPU/CUDA
# init surfaces a stack instead of silently stalling.
- export PYTHONFAULTHANDLER=1
# Per-test watchdog: a single hung test (e.g. stuck during engine/CUDA
# init) fails fast with a traceback instead of running until the global
# build timeout. The `thread` method also handles hangs inside C/CUDA
# calls that the signal method cannot interrupt.
- pytest -v -s model_executor -m '(not slow_test) and (not distributed)' --timeout=900 --timeout-method=thread
- pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
mirror:
amd:
label: ":amd: (MI300) Model Executor"
dind: false
device: mi300_1
timeout_in_minutes: 75
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/engine/arg_utils.py
- vllm/config/model.py
- vllm/model_executor
- vllm/model_executor/warmup
- tests/model_executor
- tests/model_executor/test_jit_warmup.py
- tests/model_executor/test_jit_warmup_cutedsl_launcher.py
- tests/model_executor/test_jit_warmup_triton_launcher.py
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export PYTHONFAULTHANDLER=1
- pytest -v -s model_executor -m '(not slow_test) and (not distributed)' --timeout=900 --timeout-method=thread
- pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export PYTHONFAULTHANDLER=1
- pytest -v -s model_executor -m '(not slow_test) and (not distributed)' --timeout=900 --timeout-method=thread
- pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
dind: false
depends_on:
- image-build-amd
.buildkite/test_areas/models_basic.yaml
+35 -33
View File
@@ -165,16 +165,17 @@ steps:
commands:
- pytest -v -s models/inkling/rocm
- label: ":nvidia: (B200) Flint"
key: flint-unit-tests-b200
timeout_in_minutes: 30
device: b200-k8s
source_file_dependencies:
- vllm/models/flint/
- tests/models/flint/
commands:
- uv pip install --system 'gguf>=0.19.0'
- pytest -v -s models/flint
# Disabled until the gguf installation is locked.
# - label: ":nvidia: (B200) Flint"
# key: flint-unit-tests-b200
# timeout_in_minutes: 30
# device: b200-k8s
# source_file_dependencies:
# - vllm/models/flint/
# - tests/models/flint/
# commands:
# - uv pip install --system 'gguf>=0.19.0'
# - pytest -v -s models/flint
- label: ":nvidia: (B200) Kimi K3"
key: kimi-k3-unit-tests-b200
@@ -206,29 +207,30 @@ steps:
# cuda.bindings.driver), so it is deliberately left out here.
- pytest -v -s models/kimi_k3
- label: ":nvidia: (4xB200) [Nightly] Kimi K3 Prefix Cache"
key: kimi-k3-prefix-cache-4xb200
# Nightly only: 4-GPU B200 pods wait hours behind smaller jobs on per-PR runs.
optional: true
automatic_shard: true
timeout_in_minutes: 75
device: b200-k8s
num_devices: 4
source_file_dependencies:
- vllm/models/kimi_k3/
- vllm/v1/core/
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/
- vllm/distributed/mooncake_store.py
- vllm/v1/simple_kv_offload/
- vllm/v1/spec_decode/
- vllm/model_executor/layers/mamba/
- tests/models/kimi_k3/test_prefix_cache.py
- .buildkite/scripts/install-kv-connectors.sh
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- pytest -v -s models/kimi_k3/test_prefix_cache.py
# Disabled until the KV connectors installation is locked.
# - label: ":nvidia: (4xB200) [Nightly] Kimi K3 Prefix Cache"
# key: kimi-k3-prefix-cache-4xb200
# # Nightly only: 4-GPU B200 pods wait hours behind smaller jobs on per-PR runs.
# optional: true
# automatic_shard: true
# timeout_in_minutes: 75
# device: b200-k8s
# num_devices: 4
# source_file_dependencies:
# - vllm/models/kimi_k3/
# - vllm/v1/core/
# - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
# - vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
# - vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/
# - vllm/distributed/mooncake_store.py
# - vllm/v1/simple_kv_offload/
# - vllm/v1/spec_decode/
# - vllm/model_executor/layers/mamba/
# - tests/models/kimi_k3/test_prefix_cache.py
# - .buildkite/scripts/install-kv-connectors.sh
# commands:
# - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
# - pytest -v -s models/kimi_k3/test_prefix_cache.py
- label: ":nvidia: (H200) Step5"
key: step5-unit-tests
.buildkite/test_areas/plugins.yaml
+61 -58
View File
@@ -64,63 +64,66 @@ steps:
- vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 18GB) [Nightly] GGUF Plugin"
key: gguf-plugin
device: h200_18gb
timeout_in_minutes: 30
soft_fail: true
optional: true
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/plugins_tests/gguf/
commands:
- pip install "vllm-gguf-plugin >= 0.0.2"
- pytest -v -s plugins_tests/gguf
mirror:
amd:
label: ":amd: (MI355 DPX) GGUF Plugin"
dind: false
device: mi355_dpx
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/plugins_tests/gguf
- vllm/platforms/rocm.py
# Disabled until the GGUF plugin installation is locked.
# - label: ":nvidia: (H200 MIG 18GB) [Nightly] GGUF Plugin"
# key: gguf-plugin
# device: h200_18gb
# timeout_in_minutes: 30
# soft_fail: true
# optional: true
# source_file_dependencies:
# - vllm/model_executor/layers/quantization
# - tests/plugins_tests/gguf/
# commands:
# - pip install "vllm-gguf-plugin >= 0.0.2"
# - pytest -v -s plugins_tests/gguf
# mirror:
# amd:
# label: ":amd: (MI355 DPX) GGUF Plugin"
# dind: false
# device: mi355_dpx
# timeout_in_minutes: 45
# working_dir: "/vllm-workspace/tests"
# depends_on:
# - image-build-amd
# source_file_dependencies:
# - vllm/model_executor/layers/quantization
# - tests/plugins_tests/gguf
# - vllm/platforms/rocm.py
- label: ":nvidia: (H200 MIG 18GB) [Nightly] BitsAndBytes Plugin"
key: bitsandbytes-plugin
device: h200_18gb
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
soft_fail: true
optional: true
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/plugins_tests/bitsandbytes
commands:
# bitsandbytes' int8 quant syncs internally, and it is out-of-tree, so
# there is nothing to wrap on the vLLM side.
- unset VLLM_GPU_SYNC_CHECK
- pip install "vllm-bnb-plugin >= 0.0.1"
- pytest -v -s plugins_tests/bitsandbytes -m 'not distributed'
# Disabled until the BitsAndBytes plugin installation is locked.
# - label: ":nvidia: (H200 MIG 18GB) [Nightly] BitsAndBytes Plugin"
# key: bitsandbytes-plugin
# device: h200_18gb
# timeout_in_minutes: 30
# working_dir: "/vllm-workspace/tests"
# soft_fail: true
# optional: true
# source_file_dependencies:
# - vllm/model_executor/layers/quantization
# - tests/plugins_tests/bitsandbytes
# commands:
# # bitsandbytes' int8 quant syncs internally, and it is out-of-tree, so
# # there is nothing to wrap on the vLLM side.
# - unset VLLM_GPU_SYNC_CHECK
# - pip install "vllm-bnb-plugin >= 0.0.1"
# - pytest -v -s plugins_tests/bitsandbytes -m 'not distributed'
- label: ":nvidia: (L4) [Nightly] BitsAndBytes Plugin"
key: bitsandbytes-plugin-2-gpus
timeout_in_minutes: 15
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
soft_fail: true
optional: true
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/plugins_tests/bitsandbytes
commands:
# bitsandbytes' int8 quant syncs internally, and it is out-of-tree, so
# there is nothing to wrap on the vLLM side.
- unset VLLM_GPU_SYNC_CHECK
- pip install "vllm-bnb-plugin >= 0.0.1"
- pytest -v -s plugins_tests/bitsandbytes -m 'distributed(num_gpus=2)'
# Disabled until the BitsAndBytes plugin installation is locked.
# - label: ":nvidia: (L4) [Nightly] BitsAndBytes Plugin"
# key: bitsandbytes-plugin-2-gpus
# timeout_in_minutes: 15
# working_dir: "/vllm-workspace/tests"
# device: l4
# num_devices: 2
# soft_fail: true
# optional: true
# source_file_dependencies:
# - vllm/model_executor/layers/quantization
# - tests/plugins_tests/bitsandbytes
# commands:
# # bitsandbytes' int8 quant syncs internally, and it is out-of-tree, so
# # there is nothing to wrap on the vLLM side.
# - unset VLLM_GPU_SYNC_CHECK
# - pip install "vllm-bnb-plugin >= 0.0.1"
# - pytest -v -s plugins_tests/bitsandbytes -m 'distributed(num_gpus=2)'
tools/pre_commit/test_tethering_allowlist.txt
+120
View File
@@ -101,3 +101,123 @@ tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py # CI jobs disabled p
tests/quantization/test_turboquant.py # CI jobs disabled pending locked torchao transitive dependencies
tests/rocm/test_moe_padding.py # CI jobs disabled pending locked torchao transitive dependencies
tests/rocm/test_moe_weight_replay.py # CI jobs disabled pending locked torchao transitive dependencies
# Temporary gaps from disabling jobs with unpinned install environments.
tests/device_allocator/test_alloc_conf.py # CI jobs disabled pending locked cmake installation
tests/distributed/test_elastic_ep.py # CI jobs disabled pending locked KV connectors installation
tests/models/flint/test_decode_attention.py # CI jobs disabled pending locked gguf installation
tests/models/flint/test_flint_moe.py # CI jobs disabled pending locked gguf installation
tests/models/flint/test_flint_ops.py # CI jobs disabled pending locked gguf installation
tests/models/flint/test_gguf_nvfp4.py # CI jobs disabled pending locked gguf installation
tests/models/flint/test_qk_norm_rope.py # CI jobs disabled pending locked gguf installation
tests/models/flint/test_qkv_fused.py # CI jobs disabled pending locked gguf installation
tests/parser/cohere/test_citations.py # CI jobs disabled pending locked cmake installation
tests/parser/cohere/test_reasoning.py # CI jobs disabled pending locked cmake installation
tests/parser/cohere/test_structural_tags.py # CI jobs disabled pending locked cmake installation
tests/parser/cohere/test_tool_calls.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_deepseek_v32.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_deepseek_v4.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_deepseek_v41.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_delegating_replay.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_engine.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_gemma4_streaming_reasoning.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_glm47_moe.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_granite.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_inkling.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_ling3.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_minicpmv_reasoning.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_minimax_m2.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_minimax_m3.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_nemotron_v3.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_parser_engine.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_plamo3.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_qwen3.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_qwen3_reasoning.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_replay.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_seed_oss.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_step3p5.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_token_id_scanner.py # CI jobs disabled pending locked cmake installation
tests/parser/engine/test_ufffd_reasoning_transition.py # CI jobs disabled pending locked cmake installation
tests/parser/mistral/test_reasoning.py # CI jobs disabled pending locked cmake installation
tests/parser/mistral/test_tool_calls.py # CI jobs disabled pending locked cmake installation
tests/parser/test_abstract_parser.py # CI jobs disabled pending locked cmake installation
tests/parser/test_beam_parser.py # CI jobs disabled pending locked cmake installation
tests/parser/test_chat_parsing.py # CI jobs disabled pending locked cmake installation
tests/parser/test_harmony.py # CI jobs disabled pending locked cmake installation
tests/parser/test_include_reasoning.py # CI jobs disabled pending locked cmake installation
tests/parser/test_parse.py # CI jobs disabled pending locked cmake installation
tests/parser/test_response_template.py # CI jobs disabled pending locked cmake installation
tests/parser/test_response_template_grammar.py # CI jobs disabled pending locked cmake installation
tests/parser/test_streaming.py # CI jobs disabled pending locked cmake installation
tests/plugins_tests/bitsandbytes/test_bitsandbytes.py # CI jobs disabled pending locked BitsAndBytes plugin installation
tests/plugins_tests/bitsandbytes/test_transformers.py # CI jobs disabled pending locked BitsAndBytes plugin installation
tests/plugins_tests/gguf/test_gguf_plugin_generate.py # CI jobs disabled pending locked GGUF plugin installation
tests/plugins_tests/gguf/test_gguf_plugin_multimodal.py # CI jobs disabled pending locked GGUF plugin installation
tests/standalone_tests/test_precompiled_wheel.py # CI jobs disabled pending locked cmake installation
tests/test_attention_backend_registry.py # CI jobs disabled pending locked cmake installation
tests/test_audio_media_size_precheck.py # CI jobs disabled pending locked cmake installation
tests/test_cmake_utils.py # CI jobs disabled pending locked cmake installation
tests/test_connections.py # CI jobs disabled pending locked cmake installation
tests/test_embedded_commit.py # CI jobs disabled pending locked cmake installation
tests/test_envs.py # CI jobs disabled pending locked cmake installation
tests/test_force_first_config.py # CI jobs disabled pending locked cmake installation
tests/test_fxgraphcache_pickle_patch.py # CI jobs disabled pending locked cmake installation
tests/test_logprobs.py # CI jobs disabled pending locked cmake installation
tests/test_outputs.py # CI jobs disabled pending locked cmake installation
tests/test_pcp_dp.py # CI jobs disabled pending locked cmake installation
tests/test_pooling_params.py # CI jobs disabled pending locked cmake installation
tests/test_quantization_revision_pin.py # CI jobs disabled pending locked cmake installation
tests/test_ray_env.py # CI jobs disabled pending locked cmake installation
tests/test_ray_env_utils.py # CI jobs disabled pending locked cmake installation
tests/test_request_input_bounds.py # CI jobs disabled pending locked cmake installation
tests/test_sampling_params.py # CI jobs disabled pending locked cmake installation
tests/test_scalartype.py # CI jobs disabled pending locked cmake installation
tests/test_tensorizer_revision_pin.py # CI jobs disabled pending locked cmake installation
tests/test_triton_utils.py # CI jobs disabled pending locked cmake installation
tests/test_uvicorn_logging.py # CI jobs disabled pending locked cmake installation
tests/test_version.py # CI jobs disabled pending locked cmake installation
tests/test_zen_cpu_platform_detection.py # CI jobs disabled pending locked cmake installation
tests/tokenizers_/test_basic.py # CI jobs disabled pending locked cmake installation
tests/tokenizers_/test_deepseek_v4.py # CI jobs disabled pending locked cmake installation
tests/tokenizers_/test_deepseek_v41.py # CI jobs disabled pending locked cmake installation
tests/tokenizers_/test_detokenize.py # CI jobs disabled pending locked cmake installation
tests/tokenizers_/test_hf.py # CI jobs disabled pending locked cmake installation
tests/tokenizers_/test_mistral.py # CI jobs disabled pending locked cmake installation
tests/tokenizers_/test_registry.py # CI jobs disabled pending locked cmake installation
tests/tracing/test_loading_tracing.py # CI jobs disabled pending locked tracing packages installation
tests/transformers_utils/processors/test_pixtral.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/processors/test_voxtral.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_bailing_moe_v3_vl_config.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_beam_config.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_config.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_config_parser_registry.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_deepseek_v4_utils.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_dory_config.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_dspark_mla_config.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_hf_overrides_model_type.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_muse_spark_config.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_processor.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_repo_utils.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_speculators_dspark_config.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_speculators_override.py # CI jobs disabled pending locked cmake installation
tests/transformers_utils/test_utils.py # CI jobs disabled pending locked cmake installation
tests/v1/determinism/test_batch_invariance_vlm.py # CI jobs disabled pending locked pytest plugins installation
tests/v1/determinism/test_cutlass_batch_invariance.py # CI jobs disabled pending locked pytest plugins installation
tests/v1/determinism/test_lora_shrink_batch_invariant.py # CI jobs disabled pending locked pytest plugins installation
tests/v1/determinism/test_matmul_batch_invariant.py # CI jobs disabled pending locked pytest plugins installation
tests/v1/determinism/test_mx_linear_batch_invariant.py # CI jobs disabled pending locked pytest plugins installation
tests/v1/determinism/test_nvfp4_batch_invariant.py # CI jobs disabled pending locked pytest plugins installation
tests/v1/determinism/test_rms_norm_batch_invariant.py # CI jobs disabled pending locked pytest plugins installation
tests/v1/fault_tolerance/test_engine_core_sentinel.py # CI jobs disabled pending locked KV connectors installation
tests/v1/fault_tolerance/test_fault_tolerance_e2e.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/mooncake_integration/test_accuracy.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/mooncake_integration/test_accuracy_rocm.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/mooncake_integration/test_mooncake_imports.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/moriio_integration/test_write_dummy_e2e_rocm.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/nixl_integration/test_accuracy.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/nixl_integration/test_accuracy_rocm.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/nixl_integration/test_edge_cases.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/nixl_integration/test_mamba_prefix_cache.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/nixl_integration/test_multi_connector_edge_cases.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/nixl_integration/test_nixl_imports.py # CI jobs disabled pending locked KV connectors installation
tests/v1/kv_connector/nixl_integration/test_spec_decode_acceptance.py # CI jobs disabled pending locked KV connectors installation