[CI] Give every Buildkite step an explicit key, and a hook to enforce it (#56329)

Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com>

.buildkite/intel_jobs/basic_correctness_intel.yaml

@@ -3,6 +3,7 @@ depends_on:
   - image-build-xpu
 steps:
 - label: XPU Sleep Mode
+  key: xpu-sleep-mode
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:

.buildkite/intel_jobs/engine_intel.yaml

@@ -41,6 +41,7 @@ steps:
       pytest -v -s engine/test_arg_utils.py test_sequence.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py'
 
 - label: Engine (1 GPU)
+  key: engine-1-gpu
   timeout_in_minutes: 30
   device: intel_gpu
   agent_tags:
@@ -65,6 +66,7 @@ steps:
       VLLM_XPU_ENABLE_XPU_GRAPH=1 pytest -v -s test_config.py'
 
 - label: V1 e2e (2 GPUs)
+  key: v1-e2e-2-gpus
   timeout_in_minutes: 30
   device: intel_gpu
   agent_tags:

.buildkite/intel_jobs/kernels_intel.yaml

@@ -3,6 +3,7 @@ depends_on:
   - image-build-xpu
 steps:
 - label: vLLM IR Tests
+  key: vllm-ir-tests
   timeout_in_minutes: 30
   device: intel_gpu
   agent_tags:

.buildkite/intel_jobs/lora_intel.yaml

@@ -3,6 +3,7 @@ depends_on:
   - image-build-xpu
 steps:
 - label: LoRA Runtime + Utils
+  key: lora-runtime-utils
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:
@@ -36,6 +37,7 @@ steps:
       pytest -v -s lora/test_worker.py'
 
 - label: LoRA Fused/MoE Kernels
+  key: lora-fused-moe-kernels
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:
@@ -60,6 +62,7 @@ steps:
       pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"'
 
 - label: LoRA Punica Kernels
+  key: lora-punica-kernels
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:
@@ -86,6 +89,7 @@ steps:
       pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit'
 
 - label: LoRA Punica FP8/XPU Ops
+  key: lora-punica-fp8-xpu-ops
   timeout_in_minutes: 60
   device: intel_gpu
   agent_tags:
@@ -110,6 +114,7 @@ steps:
       pytest -v -s lora/test_punica_xpu_ops.py'
 
 - label: LoRA Models
+  key: lora-models
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:
@@ -137,6 +142,7 @@ steps:
       pytest -s -v lora/test_minicpmv_tp.py'
 
 - label: LoRA Multimodal
+  key: lora-multimodal
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:

.buildkite/intel_jobs/misc_intel.yaml

@@ -3,6 +3,7 @@ depends_on:
   - image-build-xpu
 steps:
 - label: V1 Core + KV + Metrics
+  key: v1-core-kv-metrics-intel
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:
@@ -32,9 +33,10 @@ steps:
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
       cd tests &&
       pytest -v -s v1/executor &&
-      VLLM_BATCH_INVARIANT=1 pytest -v -s -m 'not cpu_test' v1/ec_connector/unit'
+      VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit'
 
 - label: V1 Sample + Logits
+  key: v1-sample-logits-intel
   timeout_in_minutes: 90
   device: intel_gpu
   agent_tags:
@@ -80,6 +82,7 @@ steps:
   parallelism: 4
 
 - label: Basic Models Tests (Initialization)
+  key: basic-models-tests-initialization
   timeout_in_minutes: 60
   device: intel_gpu
   agent_tags:
@@ -104,6 +107,7 @@ steps:
       pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]'
 
 - label: XPU CPU Offload
+  key: xpu-cpu-offload
   timeout_in_minutes: 60
   device: intel_gpu
   agent_tags:
@@ -131,6 +135,7 @@ steps:
       pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
 
 - label: NixlConnector PD accuracy (4 GPUs)
+  key: nixlconnector-pd-accuracy-4-gpus
   timeout_in_minutes: 60
   num_devices: 4
   device: intel_gpu
@@ -269,6 +274,7 @@ steps:
     pytest -v -s utils_ --ignore=utils_/test_mem_utils.py'
 
 - label: Fusion Unit Tests
+  key: fusion-unit-tests
   timeout_in_minutes: 30
   device: intel_gpu
   agent_tags:

.buildkite/intel_jobs/model_runner_v2_intel.yaml

@@ -3,6 +3,7 @@ depends_on:
   - image-build-xpu
 steps:
 - label: Model Runner V2 Core Tests (Intel)
+  key: model-runner-v2-core-tests-intel
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:
@@ -34,6 +35,7 @@ steps:
       pytest -v -s v1/e2e/general/test_min_tokens.py'
 
 - label: Model Runner V2 Examples (Intel)
+  key: model-runner-v2-examples-intel
   timeout_in_minutes: 45
   device: intel_gpu
   agent_tags:
@@ -64,6 +66,7 @@ steps:
       python3 features/automatic_prefix_caching/prefix_caching_offline.py'
 
 - label: Model Runner V2 Distributed (2 GPUs)
+  key: model-runner-v2-distributed-2-gpus
   timeout_in_minutes: 50
   device: intel_gpu
   agent_tags:
@@ -90,6 +93,7 @@ steps:
       TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -m "distributed\(num_gpus=2\)" -k "not ray and not True"'
 
 - label: Model Runner V2 Spec Decode
+  key: model-runner-v2-spec-decode
   timeout_in_minutes: 50
   device: intel_gpu
   agent_tags:

.buildkite/intel_jobs/test-intel.yaml

@@ -15,6 +15,7 @@ steps:
         - exit_status: -10  # Agent was lost
           limit: 2
   - label: "XPU example Test"
+    key: xpu-example-test
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 50
@@ -50,6 +51,7 @@ steps:
         python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
         '
   - label: "XPU W8A8 FP8 Linear Examples"
+    key: xpu-w8a8-fp8-linear-examples
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 60
@@ -77,6 +79,7 @@ steps:
         python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
         '
   - label: "XPU V1 test"
+    key: xpu-v1-test
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 70
@@ -107,6 +110,7 @@ steps:
         pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'
     parallelism: 2
   - label: "XPU server test"
+    key: xpu-server-test
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 45
@@ -130,6 +134,7 @@ steps:
         pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
         pytest -v -s benchmarks/test_serve_cli.py'
   - label: "XPU quantization test"
+    key: xpu-quantization-test
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 30
@@ -158,6 +163,7 @@ steps:
         pytest -v -s quantization/test_auto_round.py &&
         pytest -v -s quantization/test_online.py'
   - label: "XPU compressed tensors FP8 test"
+    key: xpu-compressed-tensors-fp8-test
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 60
@@ -182,6 +188,7 @@ steps:
         pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8 &&
         pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8_block_enabled'
   - label: "XPU Graph"
+    key: xpu-graph
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 60

.buildkite/test_areas/docker.yaml

@@ -3,6 +3,7 @@ depends_on:
   - image-build-cpu
 steps:
 - label: ":computer: (CPU) Docker Build Metadata"
+  key: docker-build-metadata
   timeout_in_minutes: 20
   device: cpu-small
   source_file_dependencies:

.buildkite/test_areas/lm_eval.yaml

@@ -470,6 +470,7 @@ steps:
   - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash"
 
 - label: ":nvidia: (H200 MIG 35GB) MRCR Eval Small Models"
+  key: mrcr-eval-small-models
   device: h200_35gb
   timeout_in_minutes: 25
   source_file_dependencies:

.buildkite/test_areas/rust_frontend.yaml

@@ -3,6 +3,7 @@ depends_on:
   - image-build
 steps:
 - label: ":nvidia: (H200 MIG 18GB) Rust Frontend OpenAI Coverage"
+  key: rust-frontend-openai-coverage
   timeout_in_minutes: 30
   device: h200_18gb
   working_dir: "/vllm-workspace/tests"
@@ -67,6 +68,7 @@ steps:
       - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 18GB) Rust Frontend Serve/Admin Coverage"
+  key: rust-frontend-serve-admin-coverage
   timeout_in_minutes: 25
   device: h200_18gb
   working_dir: "/vllm-workspace/tests"
@@ -116,6 +118,7 @@ steps:
       - vllm/platforms/rocm.py
 
 - label: ":nvidia: (H200 MIG 18GB) Rust Frontend Core Correctness"
+  key: rust-frontend-core-correctness
   timeout_in_minutes: 20
   device: h200_18gb
   working_dir: "/vllm-workspace/tests"
@@ -130,6 +133,7 @@ steps:
   - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
 
 - label: ":nvidia: (H200 MIG 35GB) Rust Frontend Tool Use"
+  key: rust-frontend-tool-use
   device: h200_35gb
   timeout_in_minutes: 25
   working_dir: "/vllm-workspace/tests"
@@ -145,6 +149,7 @@ steps:
   - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
 
 - label: ":nvidia: (L4) Rust Frontend Distributed"
+  key: rust-frontend-distributed
   timeout_in_minutes: 25
   device: l4
   num_devices: 4

.pre-commit-config.yaml

@@ -177,6 +177,13 @@ repos:
     files: ^\.github/mergify\.yml$
     pass_filenames: false
     additional_dependencies: [PyYAML, regex]
+  - id: check-buildkite-step-keys
+    name: Check every Buildkite step declares an explicit key
+    language: python
+    entry: python tools/pre_commit/check_buildkite_step_keys.py
+    files: ^\.buildkite/.*\.ya?ml$
+    pass_filenames: false
+    additional_dependencies: [PyYAML]
   - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward
     name: Run mypy for Python 3.10
     entry: python tools/pre_commit/mypy.py "3.10"

tools/pre_commit/check_buildkite_step_keys.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Check that every Buildkite step the CI generator reads declares a key.
+
+A step with no `key:` is published under one the generator derives from its
+label, which makes the job's address a function of its title: retitling the
+step silently renames it and breaks retries, `depends_on:` and build history.
+The generator only reads the directories a pipeline config names in
+`job_dirs:`, so this walks the same files and fails on any step without a key.
+
+Usage:
+    python tools/pre_commit/check_buildkite_step_keys.py
+"""
+
+import sys
+from pathlib import Path
+
+import yaml
+
+BUILDKITE = Path(".buildkite")
+CONFIG_GLOB = "ci_config*.yaml"
+JOB_GLOB = "*.yaml"
+
+
+def job_dirs() -> list[Path]:
+    """Every directory the pipeline generator reads, per the pipeline configs."""
+    configs = sorted(BUILDKITE.glob(CONFIG_GLOB))
+    if not configs:
+        raise SystemExit(f"no pipeline config matched {BUILDKITE / CONFIG_GLOB}")
+    dirs: set[Path] = set()
+    for config in configs:
+        with open(config, encoding="utf-8") as f:
+            doc = yaml.safe_load(f)
+        declared = doc.get("job_dirs") if isinstance(doc, dict) else None
+        if not isinstance(declared, list) or not declared:
+            raise SystemExit(f"{config}: `job_dirs` is missing or not a list")
+        for name in declared:
+            job_dir = Path(name)
+            if not job_dir.is_dir():
+                raise SystemExit(f"{config}: job_dir `{name}` does not exist")
+            dirs.add(job_dir)
+    return sorted(dirs)
+
+
+def scan(steps: object, source: Path) -> list[str]:
+    """Label of every step in a `steps:` list that declares no key."""
+    if not isinstance(steps, list):
+        raise SystemExit(f"{source}: `steps:` is not a list")
+    keyless: list[str] = []
+    for i, step in enumerate(steps):
+        # A scalar shorthand such as `- wait` has nowhere to put a key. Every
+        # other step does, so there are no other exemptions. A `mirror:` block
+        # is not a step here: the generator publishes it as `<hw>-<parent key>`.
+        if not isinstance(step, dict):
+            continue
+        label = step.get("label") or step.get("group") or f"step {i}"
+        if "steps" in step:
+            raise SystemExit(
+                f"{source}: `{label}` nests its own `steps:`. The generator has "
+                "no group step and drops the children without an error, so give "
+                "each one its own entry."
+            )
+        key = step.get("key")
+        if not (isinstance(key, str) and key.strip()):
+            keyless.append(label)
+    return keyless
+
+
+def keyless_steps(path: Path) -> list[str]:
+    with open(path, encoding="utf-8") as f:
+        doc = yaml.safe_load(f)
+    if not isinstance(doc, dict) or "steps" not in doc:
+        return []
+    return scan(doc["steps"], path)
+
+
+def main() -> int:
+    files = sorted({p for d in job_dirs() for p in d.rglob(JOB_GLOB)})
+    if not files:
+        raise SystemExit("no pipeline files under the declared job_dirs")
+    keyless = [(path, label) for path in files for label in keyless_steps(path)]
+    if not keyless:
+        return 0
+    print(f"{len(keyless)} Buildkite step(s) have no explicit key:\n", file=sys.stderr)
+    for path, label in keyless:
+        print(f"  {path}: {label}", file=sys.stderr)
+    print(
+        "\nAdd a `key:` naming the test each step runs. Without one the pipeline"
+        "\ngenerator derives the key from the label, so retitling the step renames"
+        "\nits job and breaks retries and build history.",
+        file=sys.stderr,
+    )
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())