[CI] Bump Transformers version to 5.20.0 (#61983)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

.agents/skills/transformers-upgrade/SKILL.md

@@ -23,12 +23,14 @@ A version change is one of two jobs, and they usually come as a pair:
 2. Read the release notes for config validation changes, renamed auto-mapping
    formats, and removed attributes; these are what break vLLM.
 3. Search `tests/` for `min_transformers_version` and skip markers naming the
-   new version, and un-gate them.
+   new version, and run those tests on the new pin. Keep the gates: older
+   versions in the supported range still lack the feature, so a gate goes
+   only when the floor reaches it.
 4. Run the suites in [Testing](#testing) and fix what breaks. Mark every
    shim with a `# Transformers >= X` comment so the next floor raise finds it.
 
-Done when the [Testing](#testing) suites pass on the new pin and every test
-waiting on this version is un-gated.
+Done when the [Testing](#testing) suites and every test gated on this
+version pass on the new pin.
 
 ## Raise the floor (unvendoring pass)
 

requirements/common.txt

@@ -7,7 +7,7 @@ requests >= 2.33.0
 tqdm
 blake3
 py-cpuinfo
-transformers >= 5.16.1, < 5.20.0
+transformers >= 5.16.1, < 5.21.0
 huggingface_hub >= 1.31.0  # Required for the `huggingface_hub.utils.httpx` re-export
 tokenizers >= 0.23.2  # Required for fast incremental detokenization.
 safetensors >= 0.6.2  # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611

requirements/test/cpu.txt

@@ -1223,7 +1223,7 @@ traitlets==5.16.1
     # via
     #   jupyter-client
     #   jupyter-core
-transformers==5.19.0
+transformers==5.20.2
     # via
     #   -r requirements/test/../common.txt
     #   -r requirements/test/cuda.in

requirements/test/cuda.in

@@ -40,7 +40,7 @@ datamodel_code_generator>=0.64.0 # required for minicpm3 test
 nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs
 lm-eval[api]>=0.4.12 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.19.0
+transformers==5.20.2
 schemathesis>=4.0.0 # Required for openai schema test.
 # quantization
 buildkite-test-collector==0.1.9

requirements/test/cuda.txt

@@ -1330,7 +1330,7 @@ traitlets==5.16.1
     # via
     #   jupyter-client
     #   jupyter-core
-transformers==5.19.0
+transformers==5.20.2
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt

requirements/test/nightly-torch.txt

@@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test
 datamodel_code_generator>=0.64.0 # required for minicpm3 test
 lm-eval[api]>=0.4.12 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.19.0
+transformers==5.20.2
 schemathesis>=4.0.0 # Required for openai schema test.
 # quantization
 buildkite-test-collector==0.1.9

requirements/test/rocm.in

@@ -36,7 +36,7 @@ datamodel_code_generator>=0.64.0 # required for minicpm3 test
 nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs
 lm-eval[api]>=0.4.12 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.19.0
+transformers==5.20.2
 schemathesis>=4.0.0 # Required for openai schema test
 # quantization
 bitsandbytes==0.50.0

requirements/test/rocm.txt

@@ -1281,7 +1281,7 @@ traitlets==5.16.1
     # via
     #   jupyter-client
     #   jupyter-core
-transformers==5.19.0
+transformers==5.20.2
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt

requirements/test/xpu.in

@@ -20,7 +20,7 @@ nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs
 modelscope<1.38
 runai-model-streamer[s3,gcs,azure]>=0.15.7
 tensorizer==2.10.1
-transformers==5.19.0
+transformers==5.20.2
 
 # --- Audio Processing ---
 av

requirements/test/xpu.txt

@@ -1134,7 +1134,7 @@ traitlets==5.16.1
     # via
     #   jupyter-client
     #   jupyter-core
-transformers==5.19.0
+transformers==5.20.2
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt

vllm/model_executor/models/AXK1.py

@@ -94,6 +94,18 @@ from .utils import (
 logger = init_logger(__name__)
 
 
+def _get_mlp_layer_types(config: AXK1Config) -> list[str]:
+    if (mlp_layer_types := getattr(config, "mlp_layer_types", None)) is not None:
+        return mlp_layer_types[: config.num_hidden_layers]
+    # Transformers < 5.20
+    first_k_dense_replace = config.first_k_dense_replace
+    moe_layer_freq = getattr(config, "moe_layer_freq", 1)
+    return [
+        "sparse" if i >= first_k_dense_replace and i % moe_layer_freq == 0 else "dense"
+        for i in range(config.num_hidden_layers)
+    ]
+
+
 class AXK1MLP(DeepseekV2MLP):
     pass
 
@@ -646,8 +658,7 @@ class AXK1DecoderLayer(nn.Module):
     def _is_layer_sparse(self) -> bool:
         return (
             self.config.n_routed_experts is not None
-            and self.layer_idx >= self.config.first_k_dense_replace
-            and self.layer_idx % getattr(self.config, "moe_layer_freq", 1) == 0
+            and _get_mlp_layer_types(self.config)[self.layer_idx] == "sparse"
         )
 
     def forward(
@@ -1097,9 +1108,7 @@ class AXK1ForCausalLM(
             self.model.make_empty_intermediate_tensors
         )
         # Set MoE hyperparameters
-        self.num_moe_layers = (
-            self.config.num_hidden_layers - self.config.first_k_dense_replace
-        )
+        self.num_moe_layers = _get_mlp_layer_types(self.config).count("sparse")
         self.set_moe_parameters()
 
     def set_moe_parameters(self):

vllm/model_executor/models/transformers/base.py

@@ -548,7 +548,7 @@ class Base(
                         "num_nextn_predict_layers",  # Most models
                         "mtp_num_hidden_layers",  # Qwen 3.5
                     )
-                    n_predict = getattr_iter(self.text_config, names, 0)
+                    n_predict = getattr_iter(self.text_config, names, 0) or 0
                     for i in range(num_hidden_layers, num_hidden_layers + n_predict):
                         mtp_prefix = f"{prefix}.{i}."
                         if mtp_prefix not in self.ignore_unexpected_prefixes:

vllm/model_executor/models/transformers/multimodal.py

@@ -1128,7 +1128,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
             vision_embeddings = self._get_features(modality, pixel_values, **kwargs)
             if isinstance(vision_embeddings, torch.Tensor):
                 return self._split_embeddings(vision_embeddings, split_sizes)
-            return list(vision_embeddings)
+            if len(vision_embeddings) == len(split_sizes):
+                return list(vision_embeddings)
+            # Encoders which return one entry per batch row (Transformers >= 5.20.1)
+            return self._split_embeddings(
+                torch.cat(list(vision_embeddings)), split_sizes
+            )
 
         # Items the processor left un-padded arrive as a list once their
         # shapes differ. Encode them one at a time so that none of them is