mirror of
https://github.com/huggingface/transformers.git
synced 2026-10-29 01:06:24 +08:00
Aria, Ernie4.5-VL-MoE, Gemma4, Inkling, MiniMax-M3-VL, Qwen3.5-MoE: run the model tests on the VLM/ALM testers (#49404)
* Aria, Ernie4.5-VL-MoE, Gemma4, Inkling, MiniMax-M3-VL, Qwen3.5-MoE: run the model tests on the VLM/ALM testers The multimodal test classes move onto `VLMModelTest` / `ALMModelTest`, their testers declaring their modality inputs instead of overriding the common tests. The sdpa dispatch test skips a modality tower set to None. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * Address review: simplify the multimodal testers - Drop the torch-availability guards around tester class attributes - Gemma4: collect audio/vision sub-configs from tester kwargs, build image_position_ids in get_additional_inputs, use the base mismatch test - Inkling: drop the num_hidden_layers and vision heads overrides - Qwen3.5-MoE: scrub vision special tokens through _special_token_ids - MTP generate test reads num_hidden_layers from the text config - RoPE frequency test moves the rope modules to the test device Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * MiniMax-M3-VL: use the base test_mismatching_num_image_tokens Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Gemma4 audio: build the chunked attention mask independently of the attention implementation The audio tower always runs its own chunked attention on a boolean mask, but built it with `create_bidirectional_mask`, which follows `config._attn_implementation`. Under eager the additive float mask got inverted by `logical_not()`, and under flash attention the 2D padding mask crashed `_convert_4d_mask_to_blocked_5d` (or was `None`, dropping the sliding window). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Gemma4 audio: run the chunked attention on the eager mask only The audio tower implements its own chunked local attention, so it now declares no flash/sdpa/flex/attention backend support and consumes the standard eager mask instead of a boolean mask built on the side. Output is unchanged from the boolean-mask version. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Address review: tester defaults, mask overrides and rope device - Inkling: set the sliding-window attention shape as kwarg defaults - Drop the full-ones mask overrides MiniMax-M3-VL and Qwen3.5-MoE don't need; document Ernie's - RoPE frequency test no longer passes the removed `device` argument Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilyas MoutawwakilandClaude Opus 5.5 authored and GitHub committed
1 parent
c05860687f
commit
c48824e840
10 files changed
+495
-1047
No files matched your search
@@ -314,7 +314,7 @@ class Gemma4AudioAttention(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
attention_mask: torch.BoolTensor | None = None,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, None]:
|
||||
batch_size, seq_length, _ = hidden_states.shape
|
||||
hidden_shape = (batch_size, seq_length, self.num_heads, self.head_dim)
|
||||
@@ -349,9 +349,7 @@ class Gemma4AudioAttention(nn.Module):
|
||||
attn_weights = attn_weights * self.softcap
|
||||
|
||||
if attention_mask is not None:
|
||||
attn_weights = attn_weights.masked_fill(
|
||||
attention_mask.logical_not(), self.config.attention_invalid_logits_value
|
||||
)
|
||||
attn_weights = attn_weights.masked_fill(attention_mask != 0, self.config.attention_invalid_logits_value)
|
||||
|
||||
attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype)
|
||||
attn_output = attn_weights @ value_states.permute(0, 3, 1, 2, 4)
|
||||
@@ -1894,6 +1892,11 @@ class Gemma4AudioModel(Gemma4PreTrainedModel):
|
||||
config: Gemma4AudioConfig
|
||||
main_input_name = "input_features"
|
||||
base_model_prefix = "model.audio_tower" # prefix for Gemma4ForConditionalGeneration saved checkpoints, required for Gemma4AudioModel.from_pretrained()
|
||||
# The chunked local attention is only implemented in eager
|
||||
_supports_flash_attn = False
|
||||
_supports_sdpa = False
|
||||
_supports_flex_attn = False
|
||||
_supports_attention_backend = False
|
||||
_can_record_outputs = {
|
||||
"hidden_states": Gemma4AudioLayer,
|
||||
"attentions": Gemma4AudioAttention,
|
||||
@@ -1928,9 +1931,10 @@ class Gemma4AudioModel(Gemma4PreTrainedModel):
|
||||
padded_seq_len = num_blocks * chunk_size
|
||||
pad_amount = padded_seq_len - seq_len
|
||||
|
||||
mask_4d = F.pad(mask_4d, (0, pad_amount, 0, pad_amount), value=False)
|
||||
min_dtype = torch.finfo(mask_4d.dtype).min
|
||||
mask_4d = F.pad(mask_4d, (0, pad_amount, 0, pad_amount), value=min_dtype)
|
||||
mask_5d = mask_4d.reshape(batch_size, 1, num_blocks, chunk_size, padded_seq_len)
|
||||
mask_5d = F.pad(mask_5d, (max_past_horizon, max_future_horizon), value=False)
|
||||
mask_5d = F.pad(mask_5d, (max_past_horizon, max_future_horizon), value=min_dtype)
|
||||
|
||||
block_starts = torch.arange(num_blocks, device=device) * chunk_size
|
||||
offsets = torch.arange(chunk_size + max_past_horizon + max_future_horizon, device=device)
|
||||
|
||||
@@ -347,7 +347,7 @@ class Gemma4AudioAttention(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
attention_mask: torch.BoolTensor | None = None,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, None]:
|
||||
batch_size, seq_length, _ = hidden_states.shape
|
||||
hidden_shape = (batch_size, seq_length, self.num_heads, self.head_dim)
|
||||
@@ -382,9 +382,7 @@ class Gemma4AudioAttention(nn.Module):
|
||||
attn_weights = attn_weights * self.softcap
|
||||
|
||||
if attention_mask is not None:
|
||||
attn_weights = attn_weights.masked_fill(
|
||||
attention_mask.logical_not(), self.config.attention_invalid_logits_value
|
||||
)
|
||||
attn_weights = attn_weights.masked_fill(attention_mask != 0, self.config.attention_invalid_logits_value)
|
||||
|
||||
attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype)
|
||||
attn_output = attn_weights @ value_states.permute(0, 3, 1, 2, 4)
|
||||
@@ -1614,6 +1612,11 @@ class Gemma4AudioModel(Gemma4PreTrainedModel):
|
||||
config: Gemma4AudioConfig
|
||||
main_input_name = "input_features"
|
||||
base_model_prefix = "model.audio_tower" # prefix for Gemma4ForConditionalGeneration saved checkpoints, required for Gemma4AudioModel.from_pretrained()
|
||||
# The chunked local attention is only implemented in eager
|
||||
_supports_flash_attn = False
|
||||
_supports_sdpa = False
|
||||
_supports_flex_attn = False
|
||||
_supports_attention_backend = False
|
||||
_can_record_outputs = {
|
||||
"hidden_states": Gemma4AudioLayer,
|
||||
"attentions": Gemma4AudioAttention,
|
||||
@@ -1648,9 +1651,10 @@ class Gemma4AudioModel(Gemma4PreTrainedModel):
|
||||
padded_seq_len = num_blocks * chunk_size
|
||||
pad_amount = padded_seq_len - seq_len
|
||||
|
||||
mask_4d = F.pad(mask_4d, (0, pad_amount, 0, pad_amount), value=False)
|
||||
min_dtype = torch.finfo(mask_4d.dtype).min
|
||||
mask_4d = F.pad(mask_4d, (0, pad_amount, 0, pad_amount), value=min_dtype)
|
||||
mask_5d = mask_4d.reshape(batch_size, 1, num_blocks, chunk_size, padded_seq_len)
|
||||
mask_5d = F.pad(mask_5d, (max_past_horizon, max_future_horizon), value=False)
|
||||
mask_5d = F.pad(mask_5d, (max_past_horizon, max_future_horizon), value=min_dtype)
|
||||
|
||||
block_starts = torch.arange(num_blocks, device=device) * chunk_size
|
||||
offsets = torch.arange(chunk_size + max_past_horizon + max_future_horizon, device=device)
|
||||
|
||||
@@ -2659,8 +2659,9 @@ class GenerationTesterMixin(ExportGenerateTesterMixin):
|
||||
weight_filename = os.path.join(tmpdirname, "model.safetensors")
|
||||
saved_state_dict = load_file(weight_filename)
|
||||
# add mtp weights and resave
|
||||
num_hidden_layers = config.get_text_config().num_hidden_layers
|
||||
layer_mapped_mtp_dict = {
|
||||
k.replace(".0.", f".{config.num_hidden_layers}.").replace(".mtp_block.", "."): v
|
||||
k.replace(".0.", f".{num_hidden_layers}.").replace(".mtp_block.", "."): v
|
||||
for k, v in mtp_non_shared_state_dict.items()
|
||||
}
|
||||
saved_state_dict.update(
|
||||
@@ -2671,7 +2672,7 @@ class GenerationTesterMixin(ExportGenerateTesterMixin):
|
||||
with patch.object(
|
||||
model_class,
|
||||
"_keys_to_ignore_on_load_unexpected",
|
||||
keys_to_ignore_unexpected + [f"{model.base_model_prefix}.layers.{config.num_hidden_layers}"],
|
||||
keys_to_ignore_unexpected + [f"{model.base_model_prefix}.layers.{num_hidden_layers}"],
|
||||
):
|
||||
# Reload model WITHOUT mtp
|
||||
reloaded_model = model_class.from_pretrained(tmpdirname).to(torch_device)
|
||||
|
||||
@@ -39,10 +39,8 @@ from transformers.testing_utils import (
|
||||
torch_device,
|
||||
)
|
||||
|
||||
from ...generation.test_utils import GenerationTesterMixin
|
||||
from ...test_configuration_common import ConfigTester
|
||||
from ...test_image_processing_common import load_coco_image, load_test_image
|
||||
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
|
||||
from ...vlm_tester import VLMModelTest, VLMModelTester
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
@@ -53,146 +51,35 @@ if is_torch_available():
|
||||
IMAGE_OF_VIEW_URL = "https://huggingface.co/datasets/hf-internal-testing/transformers-synthetic-assets/resolve/main/images/llava_view.jpg"
|
||||
|
||||
|
||||
class AriaVisionText2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
batch_size=13,
|
||||
num_channels=3,
|
||||
image_size=16,
|
||||
num_image_tokens=4,
|
||||
ignore_index=-100,
|
||||
image_token_index=9,
|
||||
projector_hidden_act="gelu",
|
||||
seq_length=7,
|
||||
vision_feature_select_strategy="default",
|
||||
vision_feature_layer=-1,
|
||||
text_config=AriaTextConfig(
|
||||
seq_length=7,
|
||||
is_training=True,
|
||||
use_input_mask=True,
|
||||
use_token_type_ids=False,
|
||||
use_labels=True,
|
||||
hidden_act="gelu",
|
||||
hidden_dropout_prob=0.1,
|
||||
attention_probs_dropout_prob=0.1,
|
||||
type_vocab_size=16,
|
||||
type_sequence_label_size=2,
|
||||
initializer_range=0.02,
|
||||
num_labels=3,
|
||||
num_choices=4,
|
||||
pad_token_id=1,
|
||||
hidden_size=32,
|
||||
intermediate_size=16,
|
||||
max_position_embeddings=60,
|
||||
model_type="aria_moe_lm",
|
||||
moe_intermediate_size=4,
|
||||
moe_num_experts=3,
|
||||
moe_topk=2,
|
||||
num_attention_heads=2,
|
||||
num_experts_per_tok=3,
|
||||
num_hidden_layers=2,
|
||||
num_key_value_heads=2,
|
||||
rope_theta=5000000,
|
||||
vocab_size=99,
|
||||
eos_token_id=2,
|
||||
head_dim=4,
|
||||
),
|
||||
is_training=True,
|
||||
vision_config=Idefics3VisionConfig(
|
||||
image_size=16,
|
||||
patch_size=8,
|
||||
num_channels=3,
|
||||
is_training=True,
|
||||
hidden_size=32,
|
||||
projection_dim=4,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=2,
|
||||
intermediate_size=4,
|
||||
dropout=0.1,
|
||||
attention_dropout=0.1,
|
||||
initializer_range=0.02,
|
||||
),
|
||||
):
|
||||
self.parent = parent
|
||||
self.ignore_index = ignore_index
|
||||
self.image_token_index = image_token_index
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.vision_feature_select_strategy = vision_feature_select_strategy
|
||||
self.vision_feature_layer = vision_feature_layer
|
||||
self.text_config = text_config
|
||||
self.vision_config = vision_config
|
||||
self.pad_token_id = text_config.pad_token_id
|
||||
self.eos_token_id = text_config.eos_token_id
|
||||
self.num_hidden_layers = text_config.num_hidden_layers
|
||||
self.vocab_size = text_config.vocab_size
|
||||
self.hidden_size = text_config.hidden_size
|
||||
self.num_attention_heads = text_config.num_attention_heads
|
||||
self.is_training = is_training
|
||||
class AriaVisionText2TextModelTester(VLMModelTester):
|
||||
base_model_class = AriaModel
|
||||
config_class = AriaConfig
|
||||
text_config_class = AriaTextConfig
|
||||
vision_config_class = Idefics3VisionConfig
|
||||
conditional_generation_class = AriaForConditionalGeneration
|
||||
|
||||
self.batch_size = batch_size
|
||||
self.num_channels = num_channels
|
||||
self.image_size = image_size
|
||||
self.num_image_tokens = num_image_tokens
|
||||
self.seq_length = seq_length + self.num_image_tokens
|
||||
self.projector_patch_to_query_dict = {
|
||||
vision_config.image_size**2 // vision_config.patch_size**2: vision_config.projection_dim
|
||||
}
|
||||
|
||||
def get_config(self):
|
||||
return AriaConfig(
|
||||
text_config=self.text_config.to_dict(),
|
||||
vision_config=self.vision_config.to_dict(),
|
||||
ignore_index=self.ignore_index,
|
||||
image_token_index=self.image_token_index,
|
||||
projector_hidden_act=self.projector_hidden_act,
|
||||
vision_feature_select_strategy=self.vision_feature_select_strategy,
|
||||
vision_feature_layer=self.vision_feature_layer,
|
||||
eos_token_id=self.eos_token_id,
|
||||
projector_patch_to_query_dict=self.projector_patch_to_query_dict,
|
||||
)
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
pixel_values = floats_tensor(
|
||||
[
|
||||
self.batch_size,
|
||||
self.vision_config.num_channels,
|
||||
self.vision_config.image_size,
|
||||
self.vision_config.image_size,
|
||||
]
|
||||
)
|
||||
config = self.get_config()
|
||||
|
||||
return config, pixel_values
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config_and_inputs = self.prepare_config_and_inputs()
|
||||
config, pixel_values = config_and_inputs
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 1) + 1
|
||||
attention_mask = input_ids.ne(1).to(torch_device)
|
||||
input_ids[input_ids == config.image_token_index] = self.pad_token_id
|
||||
input_ids[:, : self.num_image_tokens] = config.image_token_index
|
||||
inputs_dict = {
|
||||
"pixel_values": pixel_values,
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
return config, inputs_dict
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("image_token_id", 9)
|
||||
kwargs.setdefault("pad_token_id", 1)
|
||||
kwargs.setdefault("image_size", 16)
|
||||
kwargs.setdefault("patch_size", 8)
|
||||
kwargs.setdefault("num_image_tokens", 4)
|
||||
kwargs.setdefault("intermediate_size", 16)
|
||||
kwargs.setdefault("head_dim", 4)
|
||||
kwargs.setdefault("max_position_embeddings", 60)
|
||||
kwargs.setdefault("moe_num_experts", 3)
|
||||
kwargs.setdefault("moe_topk", 2)
|
||||
super().__init__(parent, **kwargs)
|
||||
self.projector_patch_to_query_dict = {(self.image_size // self.patch_size) ** 2: self.num_image_tokens}
|
||||
|
||||
|
||||
@require_torch
|
||||
class AriaForConditionalGenerationModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
class AriaForConditionalGenerationModelTest(VLMModelTest, unittest.TestCase):
|
||||
"""
|
||||
Model tester for `AriaForConditionalGeneration`.
|
||||
"""
|
||||
|
||||
all_model_classes = (AriaModel, AriaForConditionalGeneration) if is_torch_available() else ()
|
||||
|
||||
_is_composite = True
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = AriaVisionText2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=AriaConfig, has_text_modality=False)
|
||||
model_tester_class = AriaVisionText2TextModelTester
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason="This architecture seems to not compute gradients for the last vision-layernorm because the model uses hidden states pre-norm"
|
||||
|
||||
@@ -24,7 +24,10 @@ from transformers import (
|
||||
Ernie4_5_VLMoeForConditionalGeneration,
|
||||
Ernie4_5_VLMoeModel,
|
||||
is_torch_available,
|
||||
is_vision_available,
|
||||
)
|
||||
from transformers.models.ernie4_5_vl_moe.configuration_ernie4_5_vl_moe import (
|
||||
Ernie4_5_VLMoeTextConfig,
|
||||
Ernie4_5_VLMoeVisionConfig,
|
||||
)
|
||||
from transformers.testing_utils import (
|
||||
Expectations,
|
||||
@@ -35,179 +38,93 @@ from transformers.testing_utils import (
|
||||
slow,
|
||||
torch_device,
|
||||
)
|
||||
from transformers.utils import is_cv2_available
|
||||
|
||||
from ...generation.test_utils import GenerationTesterMixin
|
||||
from ...test_configuration_common import ConfigTester
|
||||
from ...test_modeling_common import (
|
||||
ModelTesterMixin,
|
||||
floats_tensor,
|
||||
ids_tensor,
|
||||
)
|
||||
from ...test_modeling_common import floats_tensor
|
||||
from ...test_processing_common import url_to_local_path
|
||||
from ...vlm_tester import VLMModelTest, VLMModelTester
|
||||
|
||||
|
||||
if is_cv2_available():
|
||||
pass
|
||||
|
||||
if is_torch_available():
|
||||
import torch
|
||||
|
||||
if is_vision_available():
|
||||
pass
|
||||
|
||||
class Ernie4_5_VLMoeVisionText2TextModelTester(VLMModelTester):
|
||||
base_model_class = Ernie4_5_VLMoeModel
|
||||
config_class = Ernie4_5_VLMoeConfig
|
||||
text_config_class = Ernie4_5_VLMoeTextConfig
|
||||
vision_config_class = Ernie4_5_VLMoeVisionConfig
|
||||
conditional_generation_class = Ernie4_5_VLMoeForConditionalGeneration
|
||||
|
||||
class Ernie4_5_VLMoeVisionText2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
batch_size=3,
|
||||
seq_length=7,
|
||||
num_channels=3,
|
||||
ignore_index=-100,
|
||||
image_size=112,
|
||||
video_start_token_id=3,
|
||||
video_end_token_id=4,
|
||||
image_start_token_id=5,
|
||||
image_end_token_id=6,
|
||||
image_token_id=7,
|
||||
video_token_id=8,
|
||||
is_training=True,
|
||||
text_config=None,
|
||||
vision_config=None,
|
||||
):
|
||||
self.parent = parent
|
||||
self.batch_size = batch_size
|
||||
self.seq_length = seq_length
|
||||
self.num_channels = num_channels
|
||||
self.ignore_index = ignore_index
|
||||
self.image_size = image_size
|
||||
self.bos_token_id = 0
|
||||
self.eos_token_id = 0
|
||||
self.pad_token_id = 0
|
||||
self.video_start_token_id = video_start_token_id
|
||||
self.video_end_token_id = video_end_token_id
|
||||
self.image_start_token_id = image_start_token_id
|
||||
self.image_end_token_id = image_end_token_id
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.is_training = is_training
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("video_start_token_id", 3)
|
||||
kwargs.setdefault("video_end_token_id", 4)
|
||||
kwargs.setdefault("image_start_token_id", 5)
|
||||
kwargs.setdefault("image_end_token_id", 6)
|
||||
kwargs.setdefault("image_token_id", 7)
|
||||
kwargs.setdefault("video_token_id", 8)
|
||||
kwargs.setdefault("image_size", 112)
|
||||
kwargs.setdefault("patch_size", 14)
|
||||
kwargs.setdefault("num_image_tokens", 64)
|
||||
kwargs.setdefault("hidden_act", "silu")
|
||||
kwargs.setdefault("num_key_value_heads", 1)
|
||||
kwargs.setdefault("tie_word_embeddings", True)
|
||||
kwargs.setdefault("rope_parameters", {"type": "default", "rope_theta": 500_000.0, "mrope_section": [3, 3, 2]})
|
||||
kwargs.setdefault("mlp_layer_types", ["dense", "sparse"])
|
||||
kwargs.setdefault("moe_intermediate_size", [32, 32])
|
||||
kwargs.setdefault("moe_norm_min", 1e-12)
|
||||
kwargs.setdefault("depth", 2)
|
||||
kwargs.setdefault("num_heads", 2)
|
||||
kwargs.setdefault("spatial_merge_size", 1)
|
||||
super().__init__(parent, **kwargs)
|
||||
|
||||
self.text_config = text_config
|
||||
if text_config is None:
|
||||
self.text_config = {
|
||||
"vocab_size": 99,
|
||||
"hidden_size": 32,
|
||||
"intermediate_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 2,
|
||||
"num_key_value_heads": 1,
|
||||
"hidden_act": "silu",
|
||||
"max_position_embeddings": 512,
|
||||
"tie_word_embeddings": True,
|
||||
"rope_parameters": {"type": "default", "rope_theta": 500_000.0, "mrope_section": [3, 3, 2]},
|
||||
"mlp_layer_types": ["dense", "sparse"],
|
||||
"moe_intermediate_size": [32, 32],
|
||||
"moe_k": 2,
|
||||
"moe_num_experts": 8,
|
||||
"moe_num_shared_experts": 2,
|
||||
"moe_norm_min": 1e-12,
|
||||
}
|
||||
@property
|
||||
def _special_token_ids(self):
|
||||
return super()._special_token_ids | {
|
||||
self.video_token_id,
|
||||
self.video_start_token_id,
|
||||
self.video_end_token_id,
|
||||
self.image_start_token_id,
|
||||
self.image_end_token_id,
|
||||
}
|
||||
|
||||
self.vision_config = vision_config
|
||||
if vision_config is None:
|
||||
self.vision_config = {
|
||||
"depth": 2,
|
||||
"hidden_size": 32,
|
||||
"hidden_act": "silu",
|
||||
"intermediate_size": 32,
|
||||
"num_heads": 2,
|
||||
"spatial_merge_size": 1,
|
||||
}
|
||||
def create_attention_mask(self, input_ids):
|
||||
# The M-RoPE index only counts unmasked tokens, so the default mask would pad out image placeholders
|
||||
return torch.ones_like(input_ids)
|
||||
|
||||
self.hidden_size = self.text_config["hidden_size"]
|
||||
self.num_hidden_layers = self.text_config["num_hidden_layers"]
|
||||
self.num_attention_heads = self.text_config["num_attention_heads"]
|
||||
self.vocab_size = self.text_config["vocab_size"]
|
||||
|
||||
self.num_image_tokens = 64
|
||||
self.seq_length = seq_length + self.num_image_tokens
|
||||
|
||||
def get_config(self):
|
||||
return Ernie4_5_VLMoeConfig(
|
||||
text_config=self.text_config,
|
||||
vision_config=self.vision_config,
|
||||
image_token_id=self.image_token_id,
|
||||
video_token_id=self.video_token_id,
|
||||
video_start_token_id=self.video_start_token_id,
|
||||
video_end_token_id=self.video_end_token_id,
|
||||
image_start_token_id=self.image_start_token_id,
|
||||
image_end_token_id=self.image_end_token_id,
|
||||
def create_pixel_values(self, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
return floats_tensor(
|
||||
[
|
||||
batch_size * (self.image_size**2) // (self.patch_size**2),
|
||||
self.num_channels * (self.patch_size**2),
|
||||
]
|
||||
)
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
config = self.get_config()
|
||||
patch_size = config.vision_config.patch_size
|
||||
pixel_values = floats_tensor(
|
||||
[self.batch_size * (self.image_size**2) // (patch_size**2), self.num_channels * (patch_size**2)]
|
||||
)
|
||||
|
||||
return config, pixel_values
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config_and_inputs = self.prepare_config_and_inputs()
|
||||
config, pixel_values = config_and_inputs
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
|
||||
attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device=torch_device)
|
||||
|
||||
input_ids[input_ids == self.video_token_id] = self.pad_token_id
|
||||
input_ids[input_ids == self.image_token_id] = self.pad_token_id
|
||||
input_ids[input_ids == self.video_start_token_id] = self.pad_token_id
|
||||
input_ids[input_ids == self.image_start_token_id] = self.pad_token_id
|
||||
input_ids[input_ids == self.video_end_token_id] = self.pad_token_id
|
||||
input_ids[input_ids == self.image_end_token_id] = self.pad_token_id
|
||||
|
||||
def place_image_tokens(self, input_ids, config):
|
||||
input_ids = input_ids.clone()
|
||||
input_ids[:, 0] = self.image_start_token_id
|
||||
input_ids[:, 1 : 1 + self.num_image_tokens] = self.image_token_id
|
||||
input_ids[:, 1 + self.num_image_tokens] = self.image_end_token_id
|
||||
return input_ids
|
||||
|
||||
patch_size = config.vision_config.patch_size
|
||||
patches_per_side = self.image_size // patch_size
|
||||
|
||||
def get_additional_inputs(self, config, input_ids, modality_inputs, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
patches_per_side = self.image_size // self.patch_size
|
||||
mm_token_type_ids = torch.zeros_like(input_ids)
|
||||
mm_token_type_ids[input_ids == self.image_token_id] = 1
|
||||
inputs_dict = {
|
||||
"pixel_values": pixel_values,
|
||||
return {
|
||||
"image_grid_thw": torch.tensor(
|
||||
[[1, patches_per_side, patches_per_side]] * self.batch_size, device=torch_device
|
||||
[[1, patches_per_side, patches_per_side]] * batch_size, device=torch_device
|
||||
),
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
"mm_token_type_ids": mm_token_type_ids,
|
||||
}
|
||||
return config, inputs_dict
|
||||
|
||||
|
||||
@require_torch
|
||||
class Ernie4_5_VLMoeModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
all_model_classes = (
|
||||
(
|
||||
Ernie4_5_VLMoeModel,
|
||||
Ernie4_5_VLMoeForConditionalGeneration,
|
||||
)
|
||||
if is_torch_available()
|
||||
else ()
|
||||
)
|
||||
class Ernie4_5_VLMoeModelTest(VLMModelTest, unittest.TestCase):
|
||||
model_tester_class = Ernie4_5_VLMoeVisionText2TextModelTester
|
||||
model_split_percents = [0.7, 0.9] # model too big to split at 0.5
|
||||
test_all_params_have_gradient = False # e score correction bias + moe
|
||||
_is_composite = True
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = Ernie4_5_VLMoeVisionText2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=Ernie4_5_VLMoeConfig, has_text_modality=False)
|
||||
|
||||
def test_config(self):
|
||||
self.config_tester.run_common_tests()
|
||||
|
||||
def _video_features_prepare_config_and_inputs(self):
|
||||
"""
|
||||
|
||||
@@ -22,8 +22,10 @@ from parameterized import parameterized
|
||||
|
||||
from transformers import (
|
||||
AutoTokenizer,
|
||||
Gemma4AudioConfig,
|
||||
Gemma4Config,
|
||||
Gemma4TextConfig,
|
||||
Gemma4VisionConfig,
|
||||
is_torch_available,
|
||||
logging,
|
||||
set_seed,
|
||||
@@ -42,11 +44,11 @@ from transformers.testing_utils import (
|
||||
)
|
||||
from transformers.utils import ModelOutput
|
||||
|
||||
from ...alm_tester import ALMModelTest, ALMModelTester
|
||||
from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester
|
||||
from ...generation.test_utils import GenerationTesterMixin
|
||||
from ...test_configuration_common import ConfigTester
|
||||
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
|
||||
from ...test_modeling_common import floats_tensor
|
||||
from ...test_processing_common import url_to_local_path
|
||||
from ...vlm_tester import VLMModelTest, VLMModelTester
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
@@ -190,114 +192,80 @@ class Gemma4TextModelTest(CausalLMModelTest, unittest.TestCase):
|
||||
pass
|
||||
|
||||
|
||||
class Gemma4Audio2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
image_token_id=4,
|
||||
boi_token_id=5,
|
||||
eoi_token_id=6,
|
||||
audio_token_id=7,
|
||||
boa_token_id=8,
|
||||
eoa_token_index=9,
|
||||
video_token_id=10,
|
||||
seq_length=50,
|
||||
audio_seq_length=96,
|
||||
audio_num_channels=16,
|
||||
is_training=True,
|
||||
audio_config={
|
||||
"hidden_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"hidden_act": "silu",
|
||||
"subsampling_conv_channels": [16, 8],
|
||||
"conv_kernel_size": 3,
|
||||
"attention_chunk_size": 4,
|
||||
"attention_context_left": 5,
|
||||
"attention_context_right": 0,
|
||||
"output_proj_dims": 32,
|
||||
# Clipped linears register inf/-inf buffers which cause NaN in test_torch_save_load's
|
||||
# comparison logic (inf - inf = NaN). Disable for testing.
|
||||
"use_clipped_linears": False,
|
||||
},
|
||||
):
|
||||
self.parent = parent
|
||||
self.image_token_id = image_token_id
|
||||
self.boi_token_id = boi_token_id
|
||||
self.eoi_token_id = eoi_token_id
|
||||
self.audio_token_id = audio_token_id
|
||||
self.boa_token_id = boa_token_id
|
||||
self.eoa_token_index = eoa_token_index
|
||||
self.video_token_id = video_token_id
|
||||
self.llm_tester = Gemma4TextModelTester(self.parent)
|
||||
self.llm_tester.use_bidirectional_attention = None
|
||||
self.text_config = self.llm_tester.get_config()
|
||||
self.audio_config = audio_config
|
||||
self.seq_length = seq_length
|
||||
self.audio_seq_length = audio_seq_length
|
||||
self.audio_num_channels = audio_num_channels
|
||||
self.pad_token_id = self.text_config.pad_token_id
|
||||
class Gemma4Audio2TextModelTester(ALMModelTester):
|
||||
base_model_class = Gemma4Model
|
||||
conditional_generation_class = Gemma4ForConditionalGeneration
|
||||
config_class = Gemma4Config
|
||||
text_config_class = Gemma4TextConfig
|
||||
audio_config_class = Gemma4AudioConfig
|
||||
audio_mask_key = "input_features_mask"
|
||||
|
||||
self.num_hidden_layers = self.text_config.num_hidden_layers
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
self.hidden_size = self.text_config.hidden_size
|
||||
self.num_attention_heads = self.text_config.num_attention_heads
|
||||
self.is_training = is_training
|
||||
|
||||
self.batch_size = 3
|
||||
self.encoder_seq_length = seq_length
|
||||
|
||||
def get_config(self):
|
||||
return Gemma4Config(
|
||||
text_config=self.text_config,
|
||||
vision_config=None,
|
||||
audio_config=self.audio_config,
|
||||
image_token_id=self.image_token_id,
|
||||
boi_token_id=self.boi_token_id,
|
||||
eoi_token_id=self.eoi_token_id,
|
||||
audio_token_id=self.audio_token_id,
|
||||
boa_token_id=self.boa_token_id,
|
||||
eoa_token_index=self.eoa_token_index,
|
||||
video_token_id=self.video_token_id,
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("image_token_id", 4)
|
||||
kwargs.setdefault("boi_token_id", 5)
|
||||
kwargs.setdefault("eoi_token_id", 6)
|
||||
kwargs.setdefault("audio_token_id", 7)
|
||||
kwargs.setdefault("boa_token_id", 8)
|
||||
kwargs.setdefault("eoa_token_index", 9)
|
||||
kwargs.setdefault("video_token_id", 10)
|
||||
kwargs.setdefault("pad_token_id", 0)
|
||||
kwargs.setdefault("seq_length", 50)
|
||||
kwargs.setdefault("feat_seq_length", 96)
|
||||
kwargs.setdefault("num_mel_bins", 16)
|
||||
kwargs.setdefault("num_hidden_layers", 4)
|
||||
kwargs.setdefault("num_kv_shared_layers", 2)
|
||||
kwargs.setdefault(
|
||||
"layer_types", ["sliding_attention", "full_attention", "sliding_attention", "full_attention"]
|
||||
)
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
input_features = floats_tensor([self.batch_size, self.audio_seq_length, self.audio_num_channels])
|
||||
input_features_mask = torch.ones(self.batch_size, self.audio_seq_length, dtype=torch.bool, device=torch_device)
|
||||
config = self.get_config()
|
||||
return config, input_features, input_features_mask
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config, input_features, input_features_mask = self.prepare_config_and_inputs()
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 1) + 1
|
||||
attention_mask = input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
# Ensure no tokens accidentally match special token IDs
|
||||
for token_id in [config.image_token_id, config.video_token_id, config.audio_token_id]:
|
||||
input_ids[input_ids == token_id] = self.pad_token_id
|
||||
|
||||
# The audio encoder produces audio_seq_length / 4 tokens per audio sample after subsampling.
|
||||
# We need that many audio placeholder tokens per sequence in input_ids.
|
||||
num_audio_tokens = self.audio_seq_length // 4
|
||||
input_ids[:, :num_audio_tokens] = config.audio_token_id
|
||||
|
||||
inputs_dict = {
|
||||
"input_features": input_features,
|
||||
"input_features_mask": input_features_mask,
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
kwargs.setdefault("vocab_size_per_layer_input", 99)
|
||||
kwargs.setdefault("hidden_size_per_layer_input", 16)
|
||||
kwargs.setdefault("enable_moe_block", True)
|
||||
kwargs.setdefault("moe_intermediate_size", 16)
|
||||
kwargs.setdefault("top_k_experts", 2)
|
||||
kwargs.setdefault("subsampling_conv_channels", [16, 8])
|
||||
kwargs.setdefault("conv_kernel_size", 3)
|
||||
kwargs.setdefault("attention_chunk_size", 4)
|
||||
kwargs.setdefault("attention_context_left", 5)
|
||||
kwargs.setdefault("attention_context_right", 0)
|
||||
kwargs.setdefault("output_proj_dims", 32)
|
||||
# Clipped linears register inf/-inf buffers which cause NaN in test_torch_save_load's
|
||||
# comparison logic (inf - inf = NaN). Disable for testing.
|
||||
kwargs.setdefault("use_clipped_linears", False)
|
||||
super().__init__(parent, **kwargs)
|
||||
self.head_dim = self.hidden_size // self.num_attention_heads
|
||||
self.per_layer_config = {
|
||||
layer_idx: {"head_dim": 2 * self.head_dim}
|
||||
for layer_idx, layer_type in enumerate(self.layer_types)
|
||||
if layer_type == "full_attention"
|
||||
}
|
||||
return config, inputs_dict
|
||||
|
||||
@property
|
||||
def _special_token_ids(self):
|
||||
return super()._special_token_ids | {self.image_token_id, self.video_token_id}
|
||||
|
||||
@property
|
||||
def text_config_args(self):
|
||||
return super().text_config_args + ["per_layer_config"]
|
||||
|
||||
def create_attention_mask(self, input_ids):
|
||||
return input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
def create_audio_features(self, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
# (num_audios, num_frames, num_mel_bins)
|
||||
return floats_tensor([batch_size, self.feat_seq_length, self.num_mel_bins])
|
||||
|
||||
def create_audio_mask(self, batch_size: int | None = None):
|
||||
return super().create_audio_mask(batch_size).bool()
|
||||
|
||||
def get_audio_embeds_mask(self, audio_mask):
|
||||
# Each of the two stride-2 subsampling convs keeps every other mask position
|
||||
return audio_mask[:, ::4]
|
||||
|
||||
|
||||
@require_torch
|
||||
class Gemma4Audio2TextModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
all_model_classes = (Gemma4Model, Gemma4ForConditionalGeneration) if is_torch_available() else ()
|
||||
all_generative_model_classes = (Gemma4ForConditionalGeneration,) if is_torch_available() else ()
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = Gemma4Audio2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=Gemma4Config, hidden_size=37)
|
||||
class Gemma4Audio2TextModelTest(ALMModelTest, unittest.TestCase):
|
||||
model_tester_class = Gemma4Audio2TextModelTester
|
||||
|
||||
@unittest.skip("The tester has no image in input dict and mm-encoder-output don't yet support audio")
|
||||
def test_generate_from_multimodal_encoder_outputs_and_raw_data(self):
|
||||
@@ -379,134 +347,87 @@ class Gemma4Audio2TextModelTest(ModelTesterMixin, GenerationTesterMixin, unittes
|
||||
torch.testing.assert_close(pos, expected)
|
||||
|
||||
|
||||
class Gemma4Vision2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
mm_tokens_per_image=2,
|
||||
image_token_id=4,
|
||||
video_token_id=7,
|
||||
audio_token_id=8,
|
||||
boi_token_id=5,
|
||||
eoi_token_id=6,
|
||||
seq_length=25,
|
||||
is_training=True,
|
||||
vision_config={
|
||||
"use_labels": True,
|
||||
"image_size": 20,
|
||||
"patch_size": 5,
|
||||
"num_channels": 3,
|
||||
"is_training": True,
|
||||
"hidden_size": 32,
|
||||
"num_key_value_heads": 1,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"intermediate_size": 37,
|
||||
"dropout": 0.1,
|
||||
"attention_dropout": 0.1,
|
||||
"initializer_range": 0.02,
|
||||
},
|
||||
):
|
||||
self.parent = parent
|
||||
# `image_token_id` is set to 0 to pass "resize_embeddings" test, do not modify
|
||||
self.mm_tokens_per_image = mm_tokens_per_image
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.audio_token_id = audio_token_id
|
||||
self.boi_token_id = boi_token_id
|
||||
self.eoi_token_id = eoi_token_id
|
||||
self.llm_tester = Gemma4TextModelTester(self.parent)
|
||||
self.text_config = self.llm_tester.get_config()
|
||||
self.vision_config = vision_config
|
||||
self.seq_length = seq_length
|
||||
self.pad_token_id = self.text_config.pad_token_id
|
||||
class Gemma4Vision2TextModelTester(VLMModelTester):
|
||||
base_model_class = Gemma4Model
|
||||
conditional_generation_class = Gemma4ForConditionalGeneration
|
||||
config_class = Gemma4Config
|
||||
text_config_class = Gemma4TextConfig
|
||||
vision_config_class = Gemma4VisionConfig
|
||||
|
||||
self.num_hidden_layers = self.text_config.num_hidden_layers
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
self.hidden_size = self.text_config.hidden_size
|
||||
self.num_attention_heads = self.text_config.num_attention_heads
|
||||
self.is_training = is_training
|
||||
|
||||
self.batch_size = 3
|
||||
self.num_channels = vision_config["num_channels"]
|
||||
self.image_size = vision_config["image_size"]
|
||||
self.encoder_seq_length = seq_length
|
||||
|
||||
def get_config(self):
|
||||
return Gemma4Config(
|
||||
text_config=self.text_config,
|
||||
vision_config=self.vision_config,
|
||||
image_token_id=self.image_token_id,
|
||||
video_token_id=self.video_token_id,
|
||||
audio_token_id=self.audio_token_id,
|
||||
boi_token_id=self.boi_token_id,
|
||||
eoi_token_id=self.eoi_token_id,
|
||||
mm_tokens_per_image=self.mm_tokens_per_image,
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("image_token_id", 4)
|
||||
kwargs.setdefault("boi_token_id", 5)
|
||||
kwargs.setdefault("eoi_token_id", 6)
|
||||
kwargs.setdefault("video_token_id", 7)
|
||||
kwargs.setdefault("audio_token_id", 8)
|
||||
kwargs.setdefault("patch_size", 5)
|
||||
kwargs.setdefault("pooling_kernel_size", 2)
|
||||
kwargs.setdefault("num_image_tokens", 5)
|
||||
kwargs.setdefault("seq_length", 25)
|
||||
kwargs.setdefault("num_hidden_layers", 4)
|
||||
kwargs.setdefault("num_kv_shared_layers", 2)
|
||||
kwargs.setdefault(
|
||||
"layer_types", ["sliding_attention", "full_attention", "sliding_attention", "full_attention"]
|
||||
)
|
||||
kwargs.setdefault("vocab_size_per_layer_input", 99)
|
||||
kwargs.setdefault("hidden_size_per_layer_input", 16)
|
||||
kwargs.setdefault("enable_moe_block", True)
|
||||
kwargs.setdefault("moe_intermediate_size", 16)
|
||||
kwargs.setdefault("top_k_experts", 2)
|
||||
kwargs.setdefault("use_bidirectional_attention", "vision")
|
||||
kwargs.setdefault("tie_word_embeddings", True)
|
||||
super().__init__(parent, **kwargs)
|
||||
self.per_layer_config = {
|
||||
layer_idx: {"head_dim": 2 * self.head_dim}
|
||||
for layer_idx, layer_type in enumerate(self.layer_types)
|
||||
if layer_type == "full_attention"
|
||||
}
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
config = self.get_config()
|
||||
config.vision_config.pooling_kernel_size = 2
|
||||
@property
|
||||
def _special_token_ids(self):
|
||||
return super()._special_token_ids | {self.video_token_id, self.audio_token_id}
|
||||
|
||||
@property
|
||||
def text_config_args(self):
|
||||
return super().text_config_args + ["per_layer_config"]
|
||||
|
||||
def create_attention_mask(self, input_ids):
|
||||
return input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
def create_pixel_values(self, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
# (num_images, max_num_patches, patch_size * patch_size * num_channels)
|
||||
patch_size = config.vision_config.patch_size
|
||||
pixel_values = floats_tensor(
|
||||
[
|
||||
self.batch_size,
|
||||
self.vision_config["image_size"],
|
||||
patch_size * patch_size * self.vision_config["num_channels"],
|
||||
]
|
||||
)
|
||||
# (num_images, max_num_patches, 2) for height/width positions. Let it be all ones for testign
|
||||
pixel_position_ids = torch.ones(self.vision_config["image_size"], device=torch_device, dtype=torch.long)
|
||||
pixel_position_ids = pixel_position_ids[None, :, None].repeat(self.batch_size, 1, 2)
|
||||
num_patches = self.num_image_tokens * self.pooling_kernel_size**2
|
||||
return floats_tensor([batch_size, num_patches, self.patch_size**2 * self.num_channels])
|
||||
|
||||
# create (h*w, 2) grid of (x, y) coords for a non-square input image
|
||||
num_patches = self.vision_config["image_size"]
|
||||
def create_image_position_ids(self, num_images):
|
||||
# (num_images, max_num_patches, 2) grid of (x, y) coords for a non-square image
|
||||
num_patches = self.num_image_tokens * self.pooling_kernel_size**2
|
||||
h = int(num_patches**0.5)
|
||||
w = num_patches // h
|
||||
|
||||
xs = torch.arange(w).repeat(h)
|
||||
ys = torch.arange(h).repeat_interleave(w)
|
||||
pixel_position_ids = torch.stack([xs, ys], dim=-1).to(device=torch_device)
|
||||
pixel_position_ids = pixel_position_ids.unsqueeze(0).repeat(self.batch_size, 1, 1)
|
||||
|
||||
return config, pixel_values, pixel_position_ids
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config_and_inputs = self.prepare_config_and_inputs()
|
||||
config, pixel_values, pixel_position_ids = config_and_inputs
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 1) + 1
|
||||
attention_mask = input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
# Ensure no tokens accidentally match special token IDs
|
||||
for token_id in [config.image_token_id, config.video_token_id, config.audio_token_id]:
|
||||
input_ids[input_ids == token_id] = self.pad_token_id
|
||||
input_ids[:, :5] = config.image_token_id
|
||||
position_ids = torch.stack([xs, ys], dim=-1).to(device=torch_device)
|
||||
return position_ids.unsqueeze(0).repeat(num_images, 1, 1)
|
||||
|
||||
def get_additional_inputs(self, config, input_ids, modality_inputs, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
mm_token_type_ids = torch.zeros_like(input_ids)
|
||||
mm_token_type_ids[input_ids == config.image_token_id] = 1
|
||||
|
||||
inputs_dict = {
|
||||
"pixel_values": pixel_values,
|
||||
"image_position_ids": pixel_position_ids,
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
return {
|
||||
"image_position_ids": self.create_image_position_ids(batch_size),
|
||||
"mm_token_type_ids": mm_token_type_ids,
|
||||
}
|
||||
return config, inputs_dict
|
||||
|
||||
|
||||
@require_torch
|
||||
class Gemma4Vision2TextModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
all_model_classes = (Gemma4Model, Gemma4ForConditionalGeneration) if is_torch_available() else ()
|
||||
all_generative_model_classes = (Gemma4ForConditionalGeneration,) if is_torch_available() else ()
|
||||
class Gemma4Vision2TextModelTest(VLMModelTest, unittest.TestCase):
|
||||
model_tester_class = Gemma4Vision2TextModelTester
|
||||
additional_model_inputs = ["mm_token_type_ids", "image_position_ids"]
|
||||
model_split_percents = [0.85, 0.9]
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = Gemma4Vision2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=Gemma4Config, hidden_size=37)
|
||||
super().setUp()
|
||||
self.skip_flash_attn_inference_equivalence_tests()
|
||||
|
||||
def skip_flash_attn_inference_equivalence_tests(self):
|
||||
|
||||
@@ -23,8 +23,10 @@ from safetensors.torch import load_file
|
||||
|
||||
from transformers import (
|
||||
AutoProcessor,
|
||||
InklingAudioConfig,
|
||||
InklingConfig,
|
||||
InklingTextConfig,
|
||||
InklingVisionConfig,
|
||||
is_torch_available,
|
||||
)
|
||||
from transformers.testing_utils import (
|
||||
@@ -35,10 +37,10 @@ from transformers.testing_utils import (
|
||||
torch_device,
|
||||
)
|
||||
|
||||
from ...alm_tester import ALMModelTest, ALMModelTester
|
||||
from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester
|
||||
from ...generation.test_utils import GenerationTesterMixin
|
||||
from ...test_configuration_common import ConfigTester
|
||||
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
|
||||
from ...test_modeling_common import floats_tensor, ids_tensor
|
||||
from ...vlm_tester import VLMModelTest, VLMModelTester
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
@@ -89,103 +91,64 @@ class InklingTextModelTests(CausalLMModelTest, unittest.TestCase):
|
||||
self.assertEqual(config.moe_intermediate_size, 3072)
|
||||
|
||||
|
||||
class InklingAudio2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
image_token_id=4,
|
||||
boi_token_id=5,
|
||||
eoi_token_id=6,
|
||||
audio_token_id=7,
|
||||
video_token_id=10,
|
||||
seq_length=50,
|
||||
audio_num_frames=4,
|
||||
n_mel_bins=4,
|
||||
mel_vocab_size=8,
|
||||
is_training=True,
|
||||
):
|
||||
self.parent = parent
|
||||
self.image_token_id = image_token_id
|
||||
self.boi_token_id = boi_token_id
|
||||
self.eoi_token_id = eoi_token_id
|
||||
self.audio_token_id = audio_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.llm_tester = InklingTextModelTester(self.parent)
|
||||
self.llm_tester.use_bidirectional_attention = None
|
||||
self.text_config = self.llm_tester.get_config()
|
||||
self.audio_num_frames = audio_num_frames
|
||||
self.n_mel_bins = n_mel_bins
|
||||
self.mel_vocab_size = mel_vocab_size
|
||||
self.audio_config = {
|
||||
"hidden_size": self.text_config.hidden_size,
|
||||
"n_mel_bins": n_mel_bins,
|
||||
"mel_vocab_size": mel_vocab_size,
|
||||
class InklingAudio2TextModelTester(ALMModelTester):
|
||||
base_model_class = InklingModel
|
||||
conditional_generation_class = InklingForConditionalGeneration
|
||||
config_class = InklingConfig
|
||||
text_config_class = InklingTextConfig
|
||||
audio_config_class = InklingAudioConfig
|
||||
audio_mask_key = "audio_input_ids_mask"
|
||||
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("image_token_id", 4)
|
||||
kwargs.setdefault("audio_token_id", 7)
|
||||
kwargs.setdefault("pad_token_id", 0)
|
||||
kwargs.setdefault("seq_length", 50)
|
||||
kwargs.setdefault("feat_seq_length", 4)
|
||||
kwargs.setdefault("n_mel_bins", 4)
|
||||
kwargs.setdefault("mel_vocab_size", 8)
|
||||
kwargs.setdefault("layer_types", ["hybrid_sliding", "hybrid"])
|
||||
kwargs.setdefault("mlp_layer_types", ["dense", "sparse"])
|
||||
kwargs.setdefault("moe_intermediate_size", 16)
|
||||
kwargs.setdefault("n_routed_experts", 16)
|
||||
kwargs.setdefault("head_dim", 16)
|
||||
kwargs.setdefault("swa_num_attention_heads", 2)
|
||||
kwargs.setdefault("swa_num_key_value_heads", 2)
|
||||
kwargs.setdefault("swa_head_dim", 16)
|
||||
super().__init__(parent, **kwargs)
|
||||
|
||||
@property
|
||||
def _special_token_ids(self):
|
||||
return super()._special_token_ids | {self.image_token_id}
|
||||
|
||||
def create_attention_mask(self, input_ids):
|
||||
return input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
def create_audio_features(self, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
# Quantized mel frames: (num_audios, num_frames, n_mel_bins)
|
||||
return ids_tensor([batch_size, self.feat_seq_length, self.n_mel_bins], self.mel_vocab_size)
|
||||
|
||||
def get_audio_embeds_mask(self, audio_mask):
|
||||
return audio_mask
|
||||
|
||||
def get_audio_feature_key(self):
|
||||
return "audio_input_ids"
|
||||
|
||||
def _build_modality_sub_configs(self):
|
||||
return {
|
||||
"audio_config": self.get_audio_config(),
|
||||
"vision_config": InklingVisionConfig(patch_size=5, num_hidden_layers=2, num_channels=3),
|
||||
}
|
||||
self.seq_length = seq_length
|
||||
self.pad_token_id = self.text_config.pad_token_id
|
||||
|
||||
self.num_hidden_layers = self.text_config.num_hidden_layers
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
self.hidden_size = self.text_config.hidden_size
|
||||
self.num_attention_heads = self.text_config.num_attention_heads
|
||||
self.is_training = is_training
|
||||
|
||||
self.batch_size = 3
|
||||
self.encoder_seq_length = seq_length
|
||||
|
||||
def get_config(self):
|
||||
config = InklingConfig(
|
||||
text_config=self.text_config,
|
||||
vision_config={"patch_size": 5, "num_hidden_layers": 2, "num_channels": 3},
|
||||
audio_config=self.audio_config,
|
||||
image_token_id=self.image_token_id,
|
||||
boi_token_id=self.boi_token_id,
|
||||
eoi_token_id=self.eoi_token_id,
|
||||
audio_token_id=self.audio_token_id,
|
||||
video_token_id=self.video_token_id,
|
||||
)
|
||||
config.num_hidden_layers = config.text_config.num_hidden_layers
|
||||
return config
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
audio_input_ids = ids_tensor([self.batch_size, self.audio_num_frames, self.n_mel_bins], self.mel_vocab_size)
|
||||
audio_input_ids_mask = torch.ones(self.batch_size, self.audio_num_frames, dtype=torch.bool)
|
||||
config = self.get_config()
|
||||
return config, audio_input_ids, audio_input_ids_mask
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config, audio_input_ids, audio_input_ids_mask = self.prepare_config_and_inputs()
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 1) + 1
|
||||
attention_mask = input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
# Ensure no tokens accidentally match special token IDs
|
||||
for token_id in [config.image_token_id, config.video_token_id, config.audio_token_id]:
|
||||
input_ids[input_ids == token_id] = self.pad_token_id
|
||||
|
||||
# One audio embedding is produced per valid frame; place that many audio placeholders per sequence
|
||||
input_ids[:, : self.audio_num_frames] = config.audio_token_id
|
||||
|
||||
inputs_dict = {
|
||||
"audio_input_ids": audio_input_ids,
|
||||
"audio_input_ids_mask": audio_input_ids_mask,
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
return config, inputs_dict
|
||||
|
||||
|
||||
@require_torch
|
||||
class InklingAudio2TextModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
all_model_classes = (InklingModel, InklingForConditionalGeneration) if is_torch_available() else ()
|
||||
all_generative_model_classes = (InklingForConditionalGeneration,) if is_torch_available() else ()
|
||||
class InklingAudio2TextModelTest(ALMModelTest, unittest.TestCase):
|
||||
model_tester_class = InklingAudio2TextModelTester
|
||||
test_all_params_have_gradient = False # e-score correction bias is only used for expert routing
|
||||
# Audio embeddings are packed per valid frame, so last_hidden_state[0] is the total frame count, not batch size
|
||||
skip_test_audio_features_output_shape = True
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = InklingAudio2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=InklingConfig, hidden_size=37)
|
||||
|
||||
@unittest.skip(
|
||||
"Inkling chains tower namespace and internal renames, so intermediate source keys are absent after reverse mapping"
|
||||
)
|
||||
@@ -271,122 +234,57 @@ class InklingAudio2TextModelTest(ModelTesterMixin, GenerationTesterMixin, unitte
|
||||
pass
|
||||
|
||||
|
||||
class InklingVision2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
mm_tokens_per_image=2,
|
||||
image_token_id=4,
|
||||
video_token_id=7,
|
||||
audio_token_id=8,
|
||||
boi_token_id=5,
|
||||
eoi_token_id=6,
|
||||
seq_length=25,
|
||||
is_training=True,
|
||||
vision_config={
|
||||
"use_labels": True,
|
||||
"image_size": 20,
|
||||
"patch_size": 5,
|
||||
"num_channels": 3,
|
||||
"is_training": True,
|
||||
"hidden_size": 32,
|
||||
"num_key_value_heads": 1,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"intermediate_size": 37,
|
||||
"dropout": 0.1,
|
||||
"attention_dropout": 0.1,
|
||||
"initializer_range": 0.02,
|
||||
},
|
||||
):
|
||||
self.parent = parent
|
||||
# `image_token_id` is set to 0 to pass "resize_embeddings" test, do not modify
|
||||
self.mm_tokens_per_image = mm_tokens_per_image
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.audio_token_id = audio_token_id
|
||||
self.boi_token_id = boi_token_id
|
||||
self.eoi_token_id = eoi_token_id
|
||||
self.llm_tester = InklingTextModelTester(self.parent)
|
||||
self.text_config = self.llm_tester.get_config()
|
||||
self.vision_config = vision_config
|
||||
self.seq_length = seq_length
|
||||
self.pad_token_id = self.text_config.pad_token_id
|
||||
class InklingVision2TextModelTester(VLMModelTester):
|
||||
base_model_class = InklingModel
|
||||
conditional_generation_class = InklingForConditionalGeneration
|
||||
config_class = InklingConfig
|
||||
text_config_class = InklingTextConfig
|
||||
vision_config_class = InklingVisionConfig
|
||||
|
||||
self.num_hidden_layers = self.text_config.num_hidden_layers
|
||||
self.vocab_size = self.text_config.vocab_size
|
||||
self.hidden_size = self.text_config.hidden_size
|
||||
self.num_attention_heads = self.text_config.num_attention_heads
|
||||
self.is_training = is_training
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("image_token_id", 4)
|
||||
kwargs.setdefault("audio_token_id", 8)
|
||||
kwargs.setdefault("seq_length", 25)
|
||||
kwargs.setdefault("num_image_tokens", 1)
|
||||
kwargs.setdefault("patch_size", 5)
|
||||
kwargs.setdefault("temporal_patch_size", 2)
|
||||
kwargs.setdefault("layer_types", ["hybrid_sliding", "hybrid"])
|
||||
kwargs.setdefault("mlp_layer_types", ["dense", "sparse"])
|
||||
kwargs.setdefault("moe_intermediate_size", 16)
|
||||
kwargs.setdefault("n_routed_experts", 16)
|
||||
kwargs.setdefault("swa_num_attention_heads", 2)
|
||||
kwargs.setdefault("swa_num_key_value_heads", 2)
|
||||
kwargs.setdefault("swa_head_dim", 16)
|
||||
super().__init__(parent, **kwargs)
|
||||
|
||||
self.batch_size = 3
|
||||
self.num_channels = vision_config["num_channels"]
|
||||
self.image_size = vision_config["image_size"]
|
||||
self.encoder_seq_length = seq_length
|
||||
@property
|
||||
def _special_token_ids(self):
|
||||
return super()._special_token_ids | {self.audio_token_id}
|
||||
|
||||
def get_config(self):
|
||||
config = InklingConfig(
|
||||
text_config=self.text_config,
|
||||
vision_config=self.vision_config,
|
||||
audio_config={"hidden_size": self.text_config.hidden_size, "n_mel_bins": 4, "mel_vocab_size": 8},
|
||||
image_token_id=self.image_token_id,
|
||||
video_token_id=self.video_token_id,
|
||||
audio_token_id=self.audio_token_id,
|
||||
boi_token_id=self.boi_token_id,
|
||||
eoi_token_id=self.eoi_token_id,
|
||||
mm_tokens_per_image=self.mm_tokens_per_image,
|
||||
)
|
||||
config.num_hidden_layers = config.text_config.num_hidden_layers
|
||||
return config
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
config = self.get_config()
|
||||
config.vision_config.pooling_kernel_size = 2
|
||||
def create_attention_mask(self, input_ids):
|
||||
return input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
def create_pixel_values(self, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
# One packed patch per image placeholder: (num_patches, time, height, width, channels)
|
||||
patch_size = config.vision_config.patch_size
|
||||
pixel_values = floats_tensor(
|
||||
[
|
||||
self.batch_size,
|
||||
config.vision_config.temporal_patch_size,
|
||||
patch_size,
|
||||
patch_size,
|
||||
self.vision_config["num_channels"],
|
||||
]
|
||||
return floats_tensor(
|
||||
[batch_size, self.temporal_patch_size, self.patch_size, self.patch_size, self.num_channels]
|
||||
)
|
||||
return config, pixel_values
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config_and_inputs = self.prepare_config_and_inputs()
|
||||
config, pixel_values = config_and_inputs
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 1) + 1
|
||||
attention_mask = input_ids.ne(self.pad_token_id).to(torch_device)
|
||||
|
||||
# Ensure no tokens accidentally match special token IDs
|
||||
for token_id in [config.image_token_id, config.video_token_id, config.audio_token_id]:
|
||||
input_ids[input_ids == token_id] = self.pad_token_id
|
||||
input_ids[:, :1] = config.image_token_id
|
||||
|
||||
inputs_dict = {
|
||||
"pixel_values": pixel_values,
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
def _build_modality_sub_configs(self):
|
||||
return {
|
||||
"vision_config": self.get_vision_config(),
|
||||
"audio_config": InklingAudioConfig(n_mel_bins=4, mel_vocab_size=8),
|
||||
}
|
||||
return config, inputs_dict
|
||||
|
||||
|
||||
@require_torch
|
||||
class InklingVision2TextModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
all_model_classes = (InklingModel, InklingForConditionalGeneration) if is_torch_available() else ()
|
||||
all_generative_model_classes = (InklingForConditionalGeneration,) if is_torch_available() else ()
|
||||
class InklingVision2TextModelTest(VLMModelTest, unittest.TestCase):
|
||||
model_tester_class = InklingVision2TextModelTester
|
||||
test_all_params_have_gradient = False # e-score correction bias is only used for expert routing
|
||||
test_torch_exportable = False # data-dependent control flow in the HMLP vision tower (time/space folding)
|
||||
model_split_percents = [0.85, 0.9]
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = InklingVision2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=InklingConfig, hidden_size=37)
|
||||
|
||||
@unittest.skip(
|
||||
"Inkling chains tower namespace and internal renames, so intermediate source keys are absent after reverse mapping"
|
||||
)
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
# limitations under the License.
|
||||
"""Testing suite for the PyTorch MiniMax-M3-VL model."""
|
||||
|
||||
import copy
|
||||
import unittest
|
||||
|
||||
from parameterized import parameterized
|
||||
@@ -25,7 +24,9 @@ from transformers import (
|
||||
MiniMaxM3VLImageProcessorFast,
|
||||
MiniMaxM3VLModel,
|
||||
MiniMaxM3VLProcessor,
|
||||
MiniMaxM3VLTextConfig,
|
||||
MiniMaxM3VLVideoProcessor,
|
||||
MiniMaxM3VLVisionConfig,
|
||||
is_torch_available,
|
||||
is_vision_available,
|
||||
)
|
||||
@@ -35,17 +36,14 @@ from transformers.testing_utils import (
|
||||
torch_device,
|
||||
)
|
||||
|
||||
from ...generation.test_utils import GenerationTesterMixin
|
||||
from ...test_configuration_common import ConfigTester
|
||||
from ...test_image_processing_common import load_coco_image, load_test_image
|
||||
from ...test_modeling_common import (
|
||||
TEST_EAGER_MATCHES_BATCHED_AND_GROUPED_INFERENCE_PARAMETERIZATION,
|
||||
ModelTesterMixin,
|
||||
_test_eager_matches_batched_and_grouped_inference,
|
||||
floats_tensor,
|
||||
ids_tensor,
|
||||
)
|
||||
from ...test_pipeline_mixin import PipelineTesterMixin
|
||||
from ...vlm_tester import VLMModelTest, VLMModelTester
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
@@ -62,153 +60,74 @@ if is_vision_available():
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class MiniMaxM3VLVisionText2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
batch_size=3,
|
||||
seq_length=7,
|
||||
ignore_index=-100,
|
||||
image_token_index=4,
|
||||
video_token_index=5,
|
||||
is_training=True,
|
||||
text_config={
|
||||
"hidden_size": 32,
|
||||
"intermediate_size": 64,
|
||||
"dense_intermediate_size": 128,
|
||||
"shared_intermediate_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 2,
|
||||
"head_dim": 32,
|
||||
"rotary_dim": 16,
|
||||
"hidden_act": "silu",
|
||||
"max_position_embeddings": 512,
|
||||
"rms_norm_eps": 1e-6,
|
||||
"vocab_size": 99,
|
||||
"bos_token_id": 0,
|
||||
"eos_token_id": 1,
|
||||
"pad_token_id": 2,
|
||||
"num_local_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
"n_shared_experts": 1,
|
||||
"moe_layer_freq": [0, 1],
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"minimax_m3_sparse",
|
||||
],
|
||||
"use_routing_bias": True,
|
||||
"routed_scaling_factor": 2.0,
|
||||
"swiglu_alpha": 1.702,
|
||||
"swiglu_limit": 7.0,
|
||||
"tie_word_embeddings": False,
|
||||
"rope_parameters": {
|
||||
"rope_type": "default",
|
||||
"rope_theta": 5000000.0,
|
||||
"partial_rotary_factor": 0.5,
|
||||
},
|
||||
"index_n_heads": 2,
|
||||
"index_head_dim": 16,
|
||||
"index_block_size": 8,
|
||||
"index_topk_blocks": 4,
|
||||
"index_local_blocks": 1,
|
||||
},
|
||||
vision_config={
|
||||
"hidden_size": 32,
|
||||
"intermediate_size": 64,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"num_channels": 3,
|
||||
"image_size": 14,
|
||||
"patch_size": 14,
|
||||
"temporal_patch_size": 2,
|
||||
"spatial_merge_size": 1,
|
||||
"rope_theta": 10000.0,
|
||||
},
|
||||
):
|
||||
self.parent = parent
|
||||
self.batch_size = batch_size
|
||||
self.ignore_index = ignore_index
|
||||
self.image_token_index = image_token_index
|
||||
self.video_token_index = video_token_index
|
||||
self.is_training = is_training
|
||||
self.text_config = text_config
|
||||
self.vision_config = vision_config
|
||||
|
||||
self.pad_token_id = text_config["pad_token_id"]
|
||||
self.num_hidden_layers = text_config["num_hidden_layers"]
|
||||
self.num_attention_heads = text_config["num_attention_heads"]
|
||||
self.hidden_size = text_config["hidden_size"]
|
||||
self.vocab_size = text_config["vocab_size"]
|
||||
|
||||
self.num_channels = vision_config["num_channels"]
|
||||
self.image_size = vision_config["image_size"]
|
||||
self.patch_size = vision_config["patch_size"]
|
||||
self.temporal_patch_size = vision_config["temporal_patch_size"]
|
||||
self.spatial_merge_size = vision_config["spatial_merge_size"]
|
||||
class MiniMaxM3VLVisionText2TextModelTester(VLMModelTester):
|
||||
base_model_class = MiniMaxM3VLModel
|
||||
config_class = MiniMaxM3VLConfig
|
||||
text_config_class = MiniMaxM3VLTextConfig
|
||||
vision_config_class = MiniMaxM3VLVisionConfig
|
||||
conditional_generation_class = MiniMaxM3SparseForConditionalGeneration
|
||||
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("image_token_id", 4)
|
||||
kwargs.setdefault("video_token_id", 5)
|
||||
kwargs.setdefault("bos_token_id", 0)
|
||||
kwargs.setdefault("eos_token_id", 1)
|
||||
kwargs.setdefault("pad_token_id", 2)
|
||||
kwargs.setdefault("image_size", 14)
|
||||
kwargs.setdefault("patch_size", 14)
|
||||
# One patch per image (grid [1, 1, 1]) so that the generation common tests, which crop
|
||||
# all inputs along the batch dim, keep ``pixel_values`` and ``image_grid_thw`` consistent.
|
||||
self.num_patches = 1
|
||||
self.num_image_tokens = self.num_patches // (self.spatial_merge_size**2)
|
||||
self.seq_length = seq_length + self.num_image_tokens
|
||||
self.encoder_seq_length = self.seq_length
|
||||
kwargs.setdefault("num_image_tokens", 1)
|
||||
kwargs.setdefault("temporal_patch_size", 2)
|
||||
kwargs.setdefault("spatial_merge_size", 1)
|
||||
kwargs.setdefault("projector_hidden_size", 32)
|
||||
kwargs.setdefault("intermediate_size", 64)
|
||||
kwargs.setdefault("dense_intermediate_size", 128)
|
||||
kwargs.setdefault("shared_intermediate_size", 32)
|
||||
kwargs.setdefault("num_attention_heads", 4)
|
||||
kwargs.setdefault("num_key_value_heads", 2)
|
||||
kwargs.setdefault("head_dim", 32)
|
||||
kwargs.setdefault("rotary_dim", 16)
|
||||
kwargs.setdefault("rms_norm_eps", 1e-6)
|
||||
kwargs.setdefault("num_experts", 4)
|
||||
kwargs.setdefault("num_experts_per_tok", 2)
|
||||
kwargs.setdefault("mlp_layer_types", ["dense", "sparse"])
|
||||
kwargs.setdefault("layer_types", ["full_attention", "minimax_m3_sparse"])
|
||||
kwargs.setdefault("routed_scaling_factor", 2.0)
|
||||
kwargs.setdefault("swiglu_alpha", 1.702)
|
||||
kwargs.setdefault("swiglu_limit", 7.0)
|
||||
kwargs.setdefault(
|
||||
"rope_parameters", {"rope_type": "default", "rope_theta": 5000000.0, "partial_rotary_factor": 0.5}
|
||||
)
|
||||
kwargs.setdefault("index_n_heads", 2)
|
||||
kwargs.setdefault("index_head_dim", 16)
|
||||
kwargs.setdefault("index_block_size", 8)
|
||||
kwargs.setdefault("index_topk_blocks", 4)
|
||||
kwargs.setdefault("index_local_blocks", 1)
|
||||
super().__init__(parent, **kwargs)
|
||||
|
||||
def get_config(self):
|
||||
return MiniMaxM3VLConfig(
|
||||
text_config=self.text_config,
|
||||
vision_config=self.vision_config,
|
||||
image_token_index=self.image_token_index,
|
||||
video_token_index=self.video_token_index,
|
||||
projector_hidden_size=self.text_config["hidden_size"],
|
||||
@property
|
||||
def _special_token_ids(self):
|
||||
return super()._special_token_ids | {self.video_token_id}
|
||||
|
||||
def create_pixel_values(self, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
return floats_tensor(
|
||||
[
|
||||
batch_size * self.num_image_tokens,
|
||||
self.num_channels * (self.patch_size**2) * self.temporal_patch_size,
|
||||
]
|
||||
)
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
config = self.get_config()
|
||||
patch_dim = self.num_channels * (self.patch_size**2) * self.temporal_patch_size
|
||||
pixel_values = floats_tensor([self.batch_size * self.num_patches, patch_dim])
|
||||
return config, pixel_values
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config, pixel_values = self.prepare_config_and_inputs()
|
||||
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], config.text_config.vocab_size - 2) + 2
|
||||
attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device=torch_device)
|
||||
input_ids[input_ids == self.image_token_index] = self.pad_token_id
|
||||
input_ids[input_ids == self.video_token_index] = self.pad_token_id
|
||||
input_ids[:, : self.num_image_tokens] = self.image_token_index
|
||||
|
||||
inputs_dict = {
|
||||
"pixel_values": pixel_values,
|
||||
"image_grid_thw": torch.tensor([[1, 1, 1]] * self.batch_size, device=torch_device),
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
}
|
||||
return config, inputs_dict
|
||||
def get_additional_inputs(self, config, input_ids, modality_inputs, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
return {"image_grid_thw": torch.tensor([[1, 1, 1]] * batch_size, device=torch_device)}
|
||||
|
||||
|
||||
@require_torch
|
||||
class MiniMaxM3VLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
|
||||
"""
|
||||
Model tester for `MiniMaxM3SparseForConditionalGeneration`.
|
||||
"""
|
||||
class MiniMaxM3VLModelTest(VLMModelTest, unittest.TestCase):
|
||||
model_tester_class = MiniMaxM3VLVisionText2TextModelTester
|
||||
|
||||
all_model_classes = (
|
||||
(
|
||||
MiniMaxM3VLModel,
|
||||
MiniMaxM3SparseForConditionalGeneration,
|
||||
)
|
||||
if is_torch_available()
|
||||
else ()
|
||||
)
|
||||
pipeline_model_mapping = (
|
||||
{
|
||||
"image-text-to-text": MiniMaxM3SparseForConditionalGeneration,
|
||||
}
|
||||
if is_torch_available()
|
||||
else {}
|
||||
)
|
||||
_is_composite = True
|
||||
# The vision tower packs every image's (and video frame's) patches into a single sequence
|
||||
# (batch dim 1), so ``last_hidden_state`` does not carry a per-item batch axis to shape-check.
|
||||
skip_test_image_features_output_shape = True
|
||||
@@ -219,10 +138,6 @@ class MiniMaxM3VLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTest
|
||||
# the upstream training recipe, not the main causal-LM loss (same as DeepSeek-V4).
|
||||
test_all_params_have_gradient = False
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = MiniMaxM3VLVisionText2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=MiniMaxM3VLConfig, has_text_modality=False)
|
||||
|
||||
def test_indexer_selects_blocks_per_head(self):
|
||||
"""The lightning indexer must select key blocks *per head* (one selection per KV / GQA group),
|
||||
matching the deployment kernel -- not a single selection shared across all heads. This guards
|
||||
@@ -261,9 +176,6 @@ class MiniMaxM3VLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTest
|
||||
heads_differ = (block_indices[0, 0] != block_indices[0, 1]).any()
|
||||
self.assertTrue(bool(heads_differ), "indexer heads selected identical blocks -- head axis was collapsed")
|
||||
|
||||
def test_config(self):
|
||||
self.config_tester.run_common_tests()
|
||||
|
||||
@unittest.skip(reason="IDK exactly why, can be adressed later")
|
||||
def test_reverse_loading_mapping(self):
|
||||
pass
|
||||
@@ -329,7 +241,7 @@ class MiniMaxM3VLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTest
|
||||
vocab = config.text_config.vocab_size
|
||||
pad_id = config.text_config.pad_token_id
|
||||
# Text-only inputs: keep ids clear of the image/video placeholder ids so no vision tower runs.
|
||||
low = max(self.model_tester.image_token_index, self.model_tester.video_token_index) + 1
|
||||
low = max(self.model_tester.image_token_id, self.model_tester.video_token_id) + 1
|
||||
torch.manual_seed(0)
|
||||
lengths = [self.model_tester.seq_length + 6, self.model_tester.seq_length + 1]
|
||||
seqs = [torch.randint(low, vocab - 2, (n,), device=torch_device) for n in lengths]
|
||||
@@ -357,40 +269,6 @@ class MiniMaxM3VLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTest
|
||||
for i, seq in enumerate(seqs):
|
||||
torch.testing.assert_close(batched_logits[i, : len(seq)], per_seq_logits[i], rtol=1e-4, atol=1e-4)
|
||||
|
||||
def test_mismatching_num_image_tokens(self):
|
||||
"""
|
||||
Tests that VLMs raise an explicit error when the number of images doesn't match the number
|
||||
of image tokens in the text, and that genuine multi-image cases are accepted.
|
||||
"""
|
||||
config, input_dict = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
num_patches = self.model_tester.num_patches
|
||||
for model_class in self.all_model_classes:
|
||||
model = model_class(config).to(torch_device)
|
||||
model.eval()
|
||||
curr_input_dict = copy.deepcopy(input_dict)
|
||||
_ = model(**curr_input_dict) # successful forward with no modifications
|
||||
|
||||
# remove one image but leave its image tokens in text
|
||||
curr_input_dict["pixel_values"] = curr_input_dict["pixel_values"][:-num_patches, ...]
|
||||
curr_input_dict["image_grid_thw"] = curr_input_dict["image_grid_thw"][:-1, ...]
|
||||
with self.assertRaisesRegex(ValueError, "Image features and image tokens do not match"):
|
||||
_ = model(**curr_input_dict)
|
||||
|
||||
# simulate multi-image case by concatenating inputs where each has exactly one image
|
||||
input_ids = curr_input_dict["input_ids"][:1]
|
||||
pixel_values = curr_input_dict["pixel_values"][:num_patches]
|
||||
image_grid_thw = curr_input_dict["image_grid_thw"][:1]
|
||||
input_ids = torch.cat([input_ids, input_ids], dim=0)
|
||||
|
||||
# two image-token groups but one image raises an error
|
||||
with self.assertRaisesRegex(ValueError, "Image features and image tokens do not match"):
|
||||
_ = model(input_ids=input_ids, pixel_values=pixel_values, image_grid_thw=image_grid_thw)
|
||||
|
||||
# two images and two image-token groups don't raise an error
|
||||
pixel_values = torch.cat([pixel_values, pixel_values], dim=0)
|
||||
image_grid_thw = torch.cat([image_grid_thw, image_grid_thw], dim=0)
|
||||
_ = model(input_ids=input_ids, pixel_values=pixel_values, image_grid_thw=image_grid_thw)
|
||||
|
||||
def test_video_forward(self):
|
||||
"""Video frames flow through the same vision tower as images and scatter into the video-token slots."""
|
||||
config, _ = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
@@ -416,11 +294,11 @@ class MiniMaxM3VLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTest
|
||||
self.assertEqual(pixel_values_videos.shape[0], int(video_grid_thw.prod(dim=1).sum()))
|
||||
|
||||
input_ids = ids_tensor([batch_size, self.model_tester.seq_length], config.text_config.vocab_size - 2) + 2
|
||||
input_ids[input_ids == self.model_tester.image_token_index] = self.model_tester.pad_token_id
|
||||
input_ids[input_ids == self.model_tester.video_token_index] = self.model_tester.pad_token_id
|
||||
input_ids[input_ids == self.model_tester.image_token_id] = self.model_tester.pad_token_id
|
||||
input_ids[input_ids == self.model_tester.video_token_id] = self.model_tester.pad_token_id
|
||||
# Carve out one contiguous block of video-token slots per sequence.
|
||||
self.assertLessEqual(tokens_per_video, self.model_tester.seq_length)
|
||||
input_ids[:, :tokens_per_video] = self.model_tester.video_token_index
|
||||
input_ids[:, :tokens_per_video] = self.model_tester.video_token_id
|
||||
attention_mask = torch.ones_like(input_ids)
|
||||
|
||||
for model_class in self.all_model_classes:
|
||||
@@ -459,10 +337,10 @@ class MiniMaxM3VLModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTest
|
||||
video_grid_thw = torch.tensor([[grid_t, grid_h, grid_w]] * batch_size, device=torch_device)
|
||||
|
||||
input_ids = ids_tensor([batch_size, self.model_tester.seq_length], config.text_config.vocab_size - 2) + 2
|
||||
input_ids[input_ids == self.model_tester.image_token_index] = self.model_tester.pad_token_id
|
||||
input_ids[input_ids == self.model_tester.video_token_index] = self.model_tester.pad_token_id
|
||||
input_ids[input_ids == self.model_tester.image_token_id] = self.model_tester.pad_token_id
|
||||
input_ids[input_ids == self.model_tester.video_token_id] = self.model_tester.pad_token_id
|
||||
# One fewer video-token slot than features -> mismatch.
|
||||
input_ids[:, : tokens_per_video - 1] = self.model_tester.video_token_index
|
||||
input_ids[:, : tokens_per_video - 1] = self.model_tester.video_token_id
|
||||
|
||||
for model_class in self.all_model_classes:
|
||||
model = model_class(config).to(torch_device)
|
||||
|
||||
@@ -29,13 +29,8 @@ from transformers.testing_utils import (
|
||||
)
|
||||
|
||||
from ...causal_lm_tester import CausalLMModelTest, CausalLMModelTester
|
||||
from ...generation.test_utils import GenerationTesterMixin
|
||||
from ...test_configuration_common import ConfigTester
|
||||
from ...test_modeling_common import (
|
||||
ModelTesterMixin,
|
||||
floats_tensor,
|
||||
ids_tensor,
|
||||
)
|
||||
from ...test_modeling_common import floats_tensor, ids_tensor
|
||||
from ...vlm_tester import VLMModelTest, VLMModelTester
|
||||
|
||||
|
||||
if is_torch_available():
|
||||
@@ -50,6 +45,7 @@ if is_torch_available():
|
||||
Qwen3_5MoeModel,
|
||||
Qwen3_5MoeTextConfig,
|
||||
Qwen3_5MoeTextModel,
|
||||
Qwen3_5MoeVisionConfig,
|
||||
)
|
||||
|
||||
|
||||
@@ -196,105 +192,99 @@ class Qwen3_5MoeTextModelTest(CausalLMModelTest, unittest.TestCase):
|
||||
torch.testing.assert_close(logits_padded, logits_padfree, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
class Qwen3_5MoeVisionText2TextModelTester:
|
||||
def __init__(
|
||||
self,
|
||||
parent,
|
||||
batch_size=3,
|
||||
seq_length=7,
|
||||
num_channels=3,
|
||||
ignore_index=-100,
|
||||
image_size=16,
|
||||
text_config={
|
||||
"bos_token_id": 0,
|
||||
"eos_token_id": 1,
|
||||
"pad_token_id": 2,
|
||||
"hidden_act": "silu",
|
||||
"head_dim": 32,
|
||||
"hidden_size": 32,
|
||||
"vocab_size": 99,
|
||||
"intermediate_size": 37,
|
||||
"max_position_embeddings": 512,
|
||||
"model_type": "qwen3_5_moe_text",
|
||||
"num_attention_heads": 4,
|
||||
"num_hidden_layers": 2,
|
||||
"layer_types": ["full_attention", "linear_attention"],
|
||||
"num_key_value_heads": 2,
|
||||
"tie_word_embeddings": True,
|
||||
"rope_parameters": {
|
||||
"rope_type": "default",
|
||||
"rope_theta": 10_000,
|
||||
"mrope_section": [2, 1, 1],
|
||||
"mrope_interleaved": True,
|
||||
},
|
||||
"linear_conv_kernel_dim": 2,
|
||||
"linear_key_head_dim": 16,
|
||||
"linear_value_head_dim": 16,
|
||||
"linear_num_key_heads": 4,
|
||||
"linear_num_value_heads": 8,
|
||||
"moe_intermediate_size": 16,
|
||||
"shared_expert_intermediate_size": 36,
|
||||
"num_experts_per_tok": 2,
|
||||
"num_experts": 8,
|
||||
},
|
||||
vision_config={
|
||||
"depth": 2,
|
||||
"in_chans": 3,
|
||||
"hidden_act": "gelu_pytorch_tanh",
|
||||
"intermediate_size": 32,
|
||||
"out_hidden_size": 32,
|
||||
"hidden_size": 32,
|
||||
"num_heads": 4,
|
||||
"patch_size": 16,
|
||||
"spatial_merge_size": 1,
|
||||
"temporal_patch_size": 2,
|
||||
"num_position_embeddings": 16,
|
||||
},
|
||||
image_token_id=3,
|
||||
video_token_id=4,
|
||||
vision_start_token_id=5,
|
||||
vision_end_token_id=6,
|
||||
tie_word_embeddings=True,
|
||||
is_training=True,
|
||||
):
|
||||
self.parent = parent
|
||||
self.ignore_index = ignore_index
|
||||
self.is_training = is_training
|
||||
class Qwen3_5MoeVisionText2TextModelTester(VLMModelTester):
|
||||
base_model_class = Qwen3_5MoeModel
|
||||
config_class = Qwen3_5MoeConfig
|
||||
text_config_class = Qwen3_5MoeTextConfig
|
||||
vision_config_class = Qwen3_5MoeVisionConfig
|
||||
conditional_generation_class = Qwen3_5MoeForConditionalGeneration
|
||||
|
||||
self.vision_config = vision_config
|
||||
self.text_config = text_config
|
||||
def __init__(self, parent, **kwargs):
|
||||
kwargs.setdefault("bos_token_id", 0)
|
||||
kwargs.setdefault("eos_token_id", 1)
|
||||
kwargs.setdefault("pad_token_id", 2)
|
||||
kwargs.setdefault("image_token_id", 3)
|
||||
kwargs.setdefault("video_token_id", 4)
|
||||
kwargs.setdefault("vision_start_token_id", 5)
|
||||
kwargs.setdefault("vision_end_token_id", 6)
|
||||
kwargs.setdefault("image_size", 16)
|
||||
kwargs.setdefault("patch_size", 16)
|
||||
kwargs.setdefault("num_image_tokens", 32)
|
||||
kwargs.setdefault("tie_word_embeddings", True)
|
||||
kwargs.setdefault("hidden_act", "silu")
|
||||
kwargs.setdefault("head_dim", 32)
|
||||
kwargs.setdefault("intermediate_size", 37)
|
||||
kwargs.setdefault("num_attention_heads", 4)
|
||||
kwargs.setdefault("num_key_value_heads", 2)
|
||||
kwargs.setdefault("layer_types", ["full_attention", "linear_attention"])
|
||||
kwargs.setdefault(
|
||||
"rope_parameters",
|
||||
{"rope_type": "default", "rope_theta": 10_000, "mrope_section": [2, 1, 1], "mrope_interleaved": True},
|
||||
)
|
||||
kwargs.setdefault("linear_conv_kernel_dim", 2)
|
||||
kwargs.setdefault("linear_key_head_dim", 16)
|
||||
kwargs.setdefault("linear_value_head_dim", 16)
|
||||
kwargs.setdefault("linear_num_key_heads", 4)
|
||||
kwargs.setdefault("linear_num_value_heads", 8)
|
||||
kwargs.setdefault("moe_intermediate_size", 16)
|
||||
kwargs.setdefault("shared_expert_intermediate_size", 36)
|
||||
kwargs.setdefault("num_experts_per_tok", 2)
|
||||
kwargs.setdefault("num_experts", 8)
|
||||
kwargs.setdefault("depth", 2)
|
||||
kwargs.setdefault("num_heads", 4)
|
||||
kwargs.setdefault("spatial_merge_size", 1)
|
||||
kwargs.setdefault("temporal_patch_size", 2)
|
||||
kwargs.setdefault("num_position_embeddings", 16)
|
||||
kwargs.setdefault("vision_hidden_act", "gelu_pytorch_tanh")
|
||||
kwargs.setdefault("vision_intermediate_size", 32)
|
||||
super().__init__(parent, **kwargs)
|
||||
|
||||
self.vocab_size = text_config["vocab_size"]
|
||||
self.bos_token_id = text_config["bos_token_id"]
|
||||
self.eos_token_id = text_config["eos_token_id"]
|
||||
self.pad_token_id = text_config["pad_token_id"]
|
||||
self.head_dim = text_config["head_dim"]
|
||||
self.hidden_size = text_config["hidden_size"]
|
||||
self.intermediate_size = text_config["intermediate_size"]
|
||||
self.num_hidden_layers = text_config["num_hidden_layers"]
|
||||
self.num_attention_heads = text_config["num_attention_heads"]
|
||||
self.num_key_value_heads = text_config["num_key_value_heads"]
|
||||
self.rope_parameters = text_config["rope_parameters"]
|
||||
self.hidden_act = text_config["hidden_act"]
|
||||
self.max_position_embeddings = text_config["max_position_embeddings"]
|
||||
self.model_type = text_config["model_type"]
|
||||
self.in_channels = self.num_channels
|
||||
self.out_hidden_size = self.hidden_size
|
||||
|
||||
self.vision_start_token_id = vision_start_token_id
|
||||
self.vision_end_token_id = vision_end_token_id
|
||||
self.image_token_id = image_token_id
|
||||
self.video_token_id = video_token_id
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
def create_pixel_values(self, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
return floats_tensor(
|
||||
[
|
||||
batch_size * (self.image_size**2) // (self.patch_size**2),
|
||||
self.num_channels * (self.patch_size**2) * self.temporal_patch_size,
|
||||
]
|
||||
)
|
||||
|
||||
self.batch_size = batch_size
|
||||
self.num_channels = num_channels
|
||||
self.image_size = image_size
|
||||
self.num_image_tokens = 32
|
||||
self.seq_length = seq_length + self.num_image_tokens
|
||||
@property
|
||||
def _special_token_ids(self):
|
||||
return super()._special_token_ids | {
|
||||
self.video_token_id,
|
||||
self.vision_start_token_id,
|
||||
self.vision_end_token_id,
|
||||
}
|
||||
|
||||
def place_image_tokens(self, input_ids, config):
|
||||
input_ids = input_ids.clone()
|
||||
input_ids[:, -1] = self.pad_token_id
|
||||
input_ids[:, self.num_image_tokens] = self.image_token_id
|
||||
input_ids[:, self.num_image_tokens - 1] = self.vision_start_token_id
|
||||
return input_ids
|
||||
|
||||
def get_additional_inputs(self, config, input_ids, modality_inputs, batch_size: int | None = None):
|
||||
batch_size = batch_size if batch_size is not None else self.batch_size
|
||||
mm_token_type_ids = torch.zeros_like(input_ids)
|
||||
mm_token_type_ids[input_ids == self.image_token_id] = 1
|
||||
return {
|
||||
"image_grid_thw": torch.tensor([[1, 1, 1]] * batch_size, device=torch_device),
|
||||
"mm_token_type_ids": mm_token_type_ids,
|
||||
}
|
||||
|
||||
def get_vision_config(self):
|
||||
vision_config = super().get_vision_config()
|
||||
vision_config.hidden_act = self.vision_hidden_act
|
||||
vision_config.intermediate_size = self.vision_intermediate_size
|
||||
return vision_config
|
||||
|
||||
def get_config(self):
|
||||
return Qwen3_5MoeConfig(
|
||||
text_config=self.text_config,
|
||||
vision_config=self.vision_config,
|
||||
return self.config_class(
|
||||
text_config=self.get_text_config().to_dict(),
|
||||
vision_config=self.get_vision_config().to_dict(),
|
||||
image_token_id=self.image_token_id,
|
||||
video_token_id=self.video_token_id,
|
||||
vision_start_token_id=self.vision_start_token_id,
|
||||
@@ -302,64 +292,10 @@ class Qwen3_5MoeVisionText2TextModelTester:
|
||||
tie_word_embeddings=self.tie_word_embeddings,
|
||||
)
|
||||
|
||||
def prepare_config_and_inputs(self):
|
||||
config = self.get_config()
|
||||
patch_size = config.vision_config.patch_size
|
||||
temporal_patch_size = config.vision_config.temporal_patch_size
|
||||
pixel_values = floats_tensor(
|
||||
[
|
||||
self.batch_size * (self.image_size**2) // (patch_size**2),
|
||||
self.num_channels * (patch_size**2) * temporal_patch_size,
|
||||
]
|
||||
)
|
||||
|
||||
return config, pixel_values
|
||||
|
||||
def prepare_config_and_inputs_for_common(self):
|
||||
config_and_inputs = self.prepare_config_and_inputs()
|
||||
config, pixel_values = config_and_inputs
|
||||
input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
|
||||
attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device=torch_device)
|
||||
|
||||
input_ids[:, -1] = self.pad_token_id
|
||||
input_ids[input_ids == self.video_token_id] = self.pad_token_id
|
||||
input_ids[input_ids == self.image_token_id] = self.pad_token_id
|
||||
input_ids[input_ids == self.vision_start_token_id] = self.pad_token_id
|
||||
input_ids[:, self.num_image_tokens] = self.image_token_id
|
||||
input_ids[:, self.num_image_tokens - 1] = self.vision_start_token_id
|
||||
mm_token_type_ids = torch.zeros_like(input_ids)
|
||||
mm_token_type_ids[input_ids == self.image_token_id] = 1
|
||||
inputs_dict = {
|
||||
"pixel_values": pixel_values,
|
||||
"image_grid_thw": torch.tensor([[1, 1, 1]] * self.batch_size, device=torch_device),
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
"mm_token_type_ids": mm_token_type_ids,
|
||||
}
|
||||
return config, inputs_dict
|
||||
|
||||
|
||||
@require_torch
|
||||
class Qwen3_5MoeModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase):
|
||||
"""
|
||||
Model tester for `Qwen3_5MoeForConditionalGeneration`.
|
||||
"""
|
||||
|
||||
all_model_classes = (
|
||||
(
|
||||
Qwen3_5MoeModel,
|
||||
Qwen3_5MoeForConditionalGeneration,
|
||||
)
|
||||
if is_torch_available()
|
||||
else ()
|
||||
)
|
||||
|
||||
def setUp(self):
|
||||
self.model_tester = Qwen3_5MoeVisionText2TextModelTester(self)
|
||||
self.config_tester = ConfigTester(self, config_class=Qwen3_5MoeConfig, has_text_modality=False)
|
||||
|
||||
def test_config(self):
|
||||
self.config_tester.run_common_tests()
|
||||
class Qwen3_5MoeModelTest(VLMModelTest, unittest.TestCase):
|
||||
model_tester_class = Qwen3_5MoeVisionText2TextModelTester
|
||||
|
||||
@parameterized.expand([("from_pretrained",), ("from_config",)])
|
||||
def test_automodelforcausallm(self, loader: str) -> None:
|
||||
@@ -547,7 +483,7 @@ class Qwen3_5MoeModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.Test
|
||||
config, _ = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
|
||||
bsz = self.model_tester.batch_size
|
||||
channels = config.vision_config.in_chans
|
||||
channels = config.vision_config.in_channels
|
||||
temporal_patch = config.vision_config.temporal_patch_size
|
||||
patch_size = config.vision_config.patch_size
|
||||
num_images = 2
|
||||
@@ -598,7 +534,7 @@ class Qwen3_5MoeModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.Test
|
||||
config, _ = self.model_tester.prepare_config_and_inputs_for_common()
|
||||
|
||||
bsz = self.model_tester.batch_size
|
||||
channels = config.vision_config.in_chans
|
||||
channels = config.vision_config.in_channels
|
||||
temporal_patch = config.vision_config.temporal_patch_size
|
||||
patch_size = config.vision_config.patch_size
|
||||
|
||||
|
||||
@@ -3860,7 +3860,9 @@ class ModelTesterMixin(ExportTesterMixin):
|
||||
"audio_model",
|
||||
}
|
||||
language_model_names = {"language_model", "model", "text_model"}
|
||||
modality_tower_name = [name for name in modality_tower_names if hasattr(model_sdpa, name)]
|
||||
modality_tower_name = [
|
||||
name for name in modality_tower_names if getattr(model_sdpa, name, None) is not None
|
||||
]
|
||||
modality_tower_name = modality_tower_name[0] if len(modality_tower_name) > 0 else None
|
||||
language_model_name = [name for name in language_model_names if hasattr(model_sdpa, name)]
|
||||
language_model_name = language_model_name[0] if len(language_model_name) > 0 else None
|
||||
|
||||
Reference in new issue
Block a user