diff --git a/conversion/base.py b/conversion/base.py index f58d95e846..0ffa8eb642 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -177,6 +177,7 @@ class ModelBase: self._nvfp4_layer_algo: dict[str, str | None] = {} # per-layer quant_algo, keyed by HF module path self._prec_a4: dict[str, bool] = {} # gguf tensor name -> can use 4-bit (A4) activations self._fp8_as_q8 = fp8_as_q8 + self._preserve_fp8 = ftype == gguf.LlamaFileType.GUESSED and not fp8_as_q8 self._fp8_dequantized: set[str] = set() self._fp8_e4m3_preserved: set[str] = set() @@ -620,22 +621,30 @@ class ModelBase: for name, value in new_tensors.items(): self.model_tensors[name] = value - def _transform_fp8_scale(self, name: str, scale: Tensor) -> Tensor: - return scale - def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]: - return (self.map_tensor_name(name),) + new_name = self.tensor_map.get_name(name, try_suffixes=(".weight",)) + return (new_name,) if new_name is not None else () def _prepare_fp8_e4m3_tensors(self): - if self._fp8_as_q8: + if not self._preserve_fp8: return scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {} input_scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {} consumed: list[str] = [] - candidates: dict[str, tuple[str, Tensor]] = {} + candidates: dict[str, tuple[str, Tensor, tuple[str, ...], torch.Size]] = {} + weight_dtypes: dict[str, torch.dtype] = {} scaled_weights: set[str] = set() - expert_pattern = re.compile(r"\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$") + expert_pattern = re.compile(r"(\.[^.]*experts)\.(\d+)\.") + scale_types = { + gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V, + gguf.MODEL_TENSOR.ATTN_OUT, gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE, + gguf.MODEL_TENSOR.FFN_GATE, gguf.MODEL_TENSOR.FFN_UP, gguf.MODEL_TENSOR.FFN_DOWN, + gguf.MODEL_TENSOR.FFN_GATE_SHEXP, gguf.MODEL_TENSOR.FFN_UP_SHEXP, gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, + gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP, + gguf.MODEL_TENSOR.SSM_IN, gguf.MODEL_TENSOR.SSM_OUT, gguf.MODEL_TENSOR.SSM_ALPHA, gguf.MODEL_TENSOR.SSM_BETA, + gguf.MODEL_TENSOR.NEXTN_EH_PROJ, gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, gguf.MODEL_TENSOR.OUTPUT, + } for scale_name in list(self.model_tensors.keys()): weight_name = None @@ -651,6 +660,7 @@ class ModelBase: scaled_weights.add(weight_name) weight = self.model_tensors[weight_name]() + weight_dtypes[weight_name] = weight.dtype if weight.dtype != torch.float8_e4m3fn or weight.ndim < 2: continue @@ -658,28 +668,33 @@ class ModelBase: if prod(scale.shape) != 1: continue - merged_name = re.sub(r"(\.experts)\.\d+\.", r"\1.", weight_name) + merged_name = expert_pattern.sub(r"\1.", weight_name) new_names = self._map_fp8_weight_names(merged_name) if not new_names: continue + if any((mapped := self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",))) is None or mapped[0] not in scale_types for new_name in new_names): + continue bid = next((int(part) for part in new_names[0].split(".") if part.isdecimal()), None) self._fp8_e4m3_preserved.add(weight_name) if any(new_name.endswith("_norm.weight") or self.tensor_force_quant(weight_name, new_name, bid, weight.ndim) != gguf.GGMLQuantizationType.F8_E4M3 for new_name in new_names): self._fp8_e4m3_preserved.discard(weight_name) continue - candidates[weight_name] = (scale_name, scale) + candidates[weight_name] = (scale_name, scale, new_names, weight.shape) expert_groups: dict[str, list[str]] = {} for name in self.model_tensors: - if expert_pattern.search(name): - expert_groups.setdefault(name.split(".experts.")[0], []).append(name) + if (match := expert_pattern.search(name)) and name.endswith(".weight"): + expert_groups.setdefault(name[:match.end(1)], []).append(name) for names in expert_groups.values(): - dtypes = {self.model_tensors[name]().dtype for name in names} + for name in names: + if name not in weight_dtypes: + weight_dtypes[name] = self.model_tensors[name]().dtype + dtypes = {weight_dtypes[name] for name in names} if torch.float8_e4m3fn not in dtypes: continue if dtypes == {torch.float8_e4m3fn} and not any(name in scaled_weights for name in names): continue - # The converter emits all three projections when the last expert arrives. + # The converter merges expert projections together. if not all(name in candidates for name in names): for name in names: candidates.pop(name, None) @@ -688,10 +703,8 @@ class ModelBase: gen = self.model_tensors[name] self.model_tensors[name] = lambda gen=gen: gen().float() - for weight_name, (scale_name, scale) in candidates.items(): - weight = self.model_tensors[weight_name]() + for weight_name, (scale_name, scale, new_names, shape) in candidates.items(): scale = LazyTorchTensor.to_eager(scale).float().flatten() - scale = self._transform_fp8_scale(weight_name, scale) weight_prefix = weight_name.removesuffix(".weight") # Transformers fine-grained FP8 uses activation_scale while ModelOpt uses input_scale. @@ -714,28 +727,25 @@ class ModelBase: expert_match = expert_pattern.search(weight_name) if expert_match: - bid = int(expert_match.group(1)) expert_id = int(expert_match.group(2)) - proj_type = expert_match.group(3) - merged_name = f"model.layers.{bid}.mlp.experts.{proj_type}.weight" - new_name = self.map_tensor_name(merged_name) - target_name = new_name.replace(".weight", ".scale") - entries = scale_tensors.setdefault(target_name, []) - assert isinstance(entries, list) - cast(list[tuple[int, float]], entries).append((expert_id, float(scale[0]))) - if input_scale is not None: - target_name = new_name.replace(".weight", ".input_scale") - entries = input_scale_tensors.setdefault(target_name, []) + for new_name in new_names: + target_name = new_name.replace(".weight", ".scale") + entries = scale_tensors.setdefault(target_name, []) assert isinstance(entries, list) - cast(list[tuple[int, float]], entries).append((expert_id, float(input_scale[0]))) + cast(list[tuple[int, float]], entries).append((expert_id, float(scale[0]))) + if input_scale is not None: + target_name = new_name.replace(".weight", ".input_scale") + entries = input_scale_tensors.setdefault(target_name, []) + assert isinstance(entries, list) + cast(list[tuple[int, float]], entries).append((expert_id, float(input_scale[0]))) else: - for new_name in self._map_fp8_weight_names(weight_name): - mapped = self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",)) if weight.ndim == 3 else None + for new_name in new_names: + mapped = self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",)) if len(shape) == 3 else None is_packed_expert = mapped is not None and mapped[0] in ( gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP, ) # Packed expert weights need one scale value per expert. - n_scales = weight.shape[0] if is_packed_expert else 1 + n_scales = shape[0] if is_packed_expert else 1 scale_tensors[new_name.replace(".weight", ".scale")] = np.repeat(scale.numpy(), n_scales) if input_scale is not None: input_scale_tensors[new_name.replace(".weight", ".input_scale")] = np.repeat(input_scale.numpy(), n_scales) @@ -1432,7 +1442,8 @@ class ModelBase: if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")): continue - preserve_fp8_e4m3 = not self._fp8_as_q8 and data_torch.dtype == torch.float8_e4m3fn + # FP8 transforms must only reorder values; norms and vectors use numeric F32 transforms. + preserve_fp8_e4m3 = self._preserve_fp8 and data_torch.dtype == torch.float8_e4m3fn and data_torch.ndim >= 2 and not name.endswith(("norm.weight", "norm_f.weight")) if preserve_fp8_e4m3: self._fp8_e4m3_preserved.add(name) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index d76a45d828..371f5b616f 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -433,6 +433,11 @@ class NemotronHModel(GraniteHybridModel): _ATTN_LAYER_TYPES = {"attention", "full_attention"} _MLP_LAYER_TYPES = {"moe"} + def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]: + if ".mixer.experts." in name: + name = name.replace("backbone.", "model.").replace(".mixer.experts.", ".mlp.experts.") + return super()._map_fp8_weight_names(name) + def __init__(self, *args, **kwargs): # We have to determine the correct model architecture (MoE vs non-MoE) before # calling the parent __init__. This is because the parent constructor diff --git a/conversion/qwen.py b/conversion/qwen.py index 64a52017c5..e6456e5a54 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -519,24 +519,6 @@ class _LinearAttentionVReorderBase(Qwen3NextModel): perm[dim], perm[dim + 1] = perm[dim + 1], perm[dim] return tensor.permute(*perm).contiguous().reshape(*shape) - def _transform_fp8_scale(self, name: str, scale: Tensor) -> Tensor: - num_k_heads = self.hparams.get("linear_num_key_heads", 0) - num_v_heads = self.hparams.get("linear_num_value_heads", 0) - if scale.numel() == 1 or num_k_heads == 0 or num_v_heads == 0 or num_k_heads == num_v_heads: - return scale - - num_v_per_k = num_v_heads // num_k_heads - head_v_dim = self.hparams["linear_value_head_dim"] - if name.endswith(".linear_attn.in_proj_qkv.weight"): - qk_dim = 2 * self.hparams["linear_key_head_dim"] * num_k_heads - v_scale = self._reorder_v_heads(scale[qk_dim:], 0, num_k_heads, num_v_per_k, head_v_dim) - return torch.cat([scale[:qk_dim], v_scale]) - if name.endswith(".linear_attn.in_proj_z.weight"): - return self._reorder_v_heads(scale, 0, num_k_heads, num_v_per_k, head_v_dim) - if name.endswith((".linear_attn.in_proj_a.weight", ".linear_attn.in_proj_b.weight")): - return self._reorder_v_heads(scale, 0, num_k_heads, num_v_per_k, 1) - return scale - def _transform_nvfp4_weight(self, name: str, weight: Tensor, scale: Tensor) -> tuple[Tensor, Tensor]: if not name.endswith(( ".linear_attn.in_proj_qkv.weight", diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 5e263fd16f..45880b2d17 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -58,7 +58,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument( "--outtype", type=str, choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"], default="auto", - help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type", + help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto to preserve supported FP8 weights and otherwise select the highest-fidelity 16-bit float type", ) parser.add_argument( "--bigendian", action="store_true", diff --git a/tests/test_conversion_fp8.py b/tests/test_conversion_fp8.py index cf510ee6aa..e350746772 100644 --- a/tests/test_conversion_fp8.py +++ b/tests/test_conversion_fp8.py @@ -21,6 +21,9 @@ from safetensors.torch import save_file from conversion.base import LazyTorchTensor, gguf from conversion.deepseek import DeepseekV2Model +from conversion.llama import LlamaModel +from conversion.nemotron import NemotronHModel +from conversion.phi import PhiMoeModel from conversion.qwen import Qwen2MoeModel @@ -33,8 +36,17 @@ class ForcedF16Model(Qwen2MoeModel): return super().tensor_force_quant(name, new_name, bid, n_dims) +class OffsetNormModel(Qwen2MoeModel): + model_arch = gguf.MODEL_ARCH.QWEN2MOE + + def modify_tensors(self, data_torch, name, bid): + if name.endswith("input_layernorm.weight"): + data_torch = data_torch + 1.0 + yield from super().modify_tensors(data_torch, name, bid) + + class TestFP8Conversion(unittest.TestCase): - def convert(self, model_class, tensors, eager=False, check_lazy=False, scan_nvfp4=False, **kwargs): + def convert(self, model_class, tensors, eager=False, check_lazy=False, scan_nvfp4=False, ftype=gguf.LlamaFileType.GUESSED, hparams_override=None, **kwargs): with tempfile.TemporaryDirectory() as directory: path = Path(directory) hparams = { @@ -49,9 +61,10 @@ class TestFP8Conversion(unittest.TestCase): "v_head_dim": 2, "quantization_config": {"quant_method": "modelopt"}, } + hparams.update(hparams_override or {}) (path / "config.json").write_text(json.dumps(hparams)) save_file(tensors, str(path / "model.safetensors")) - model = model_class(path, gguf.LlamaFileType.MOSTLY_F16, path / "out.gguf", hparams=hparams, eager=eager, **kwargs) + model = model_class(path, ftype, path / "out.gguf", hparams=hparams, eager=eager, **kwargs) reads: list[str] = [] if check_lazy: self.assertFalse(eager) @@ -143,6 +156,97 @@ class TestFP8Conversion(unittest.TestCase): self.assertEqual(dtype, gguf.GGMLQuantizationType.F32) np.testing.assert_array_equal(data, [1., -1.]) + def test_fp8_norm_transform(self): + for shape in ((4,), (2, 2)): + with self.subTest(shape=shape): + weight = torch.ones(shape, dtype=torch.float32).to(torch.float8_e4m3fn) + result = self.convert(OffsetNormModel, {"model.layers.0.input_layernorm.weight": weight}, check_lazy=True) + dtype, data = result["blk.0.attn_norm.weight"] + self.assertEqual(dtype, gguf.GGMLQuantizationType.F32) + np.testing.assert_array_equal(data, np.full(shape, 2.)) + + def test_explicit_outtypes(self): + name = "model.layers.0.self_attn.q_proj.weight" + weight = torch.ones((4, 256), dtype=torch.float32).to(torch.float8_e4m3fn) + for ftype, qtype in ( + (gguf.LlamaFileType.ALL_F32, gguf.GGMLQuantizationType.F32), + (gguf.LlamaFileType.MOSTLY_F16, gguf.GGMLQuantizationType.F16), + (gguf.LlamaFileType.MOSTLY_BF16, gguf.GGMLQuantizationType.BF16), + (gguf.LlamaFileType.MOSTLY_Q8_0, gguf.GGMLQuantizationType.Q8_0), + (gguf.LlamaFileType.MOSTLY_TQ1_0, gguf.GGMLQuantizationType.TQ1_0), + (gguf.LlamaFileType.MOSTLY_TQ2_0, gguf.GGMLQuantizationType.TQ2_0), + ): + for eager in (False, True): + for scaled in (False, True): + with self.subTest(ftype=ftype, eager=eager, scaled=scaled): + tensors = {name: weight} + if scaled: + tensors[name + "_scale"] = torch.tensor([0.25]) + tensors[name.removesuffix("weight") + "input_scale"] = torch.tensor([0.5]) + result = self.convert(Qwen2MoeModel, tensors, ftype=ftype, eager=eager, check_lazy=not eager) + self.assertEqual(set(result), {"blk.0.attn_q.weight"}) + dtype, data = result["blk.0.attn_q.weight"] + self.assertEqual(dtype, qtype) + np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 256), 0.25 if scaled else 1.), atol=1e-4) + + def test_embedding_scale_fallback(self): + name = "model.embed_tokens.weight" + weight = torch.ones((4, 4), dtype=torch.float32).to(torch.float8_e4m3fn) + for eager in (False, True): + with self.subTest(eager=eager): + result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale": torch.tensor([0.25])}, eager=eager, check_lazy=not eager) + self.assertEqual(set(result), {"token_embd.weight"}) + dtype, data = result["token_embd.weight"] + self.assertEqual(dtype, gguf.GGMLQuantizationType.F16) + np.testing.assert_array_equal(data, np.full((4, 4), 0.25)) + + def test_fp8_scale_formats(self): + name = "model.layers.0.self_attn.q_proj.weight" + weight = torch.ones((4, 4), dtype=torch.float32).to(torch.float8_e4m3fn) + for suffix, config in ( + ("_scale_inv", {"quant_method": "fp8"}), + ("_scale", {"quant_method": "compressed-tensors", "format": "float-quantized", "config_groups": {"group_0": {"weights": {"type": "float", "num_bits": 8, "strategy": "tensor"}}}}), + ): + for ftype in (gguf.LlamaFileType.GUESSED, gguf.LlamaFileType.MOSTLY_F16): + with self.subTest(suffix=suffix, ftype=ftype): + result = self.convert(Qwen2MoeModel, {name: weight, name + suffix: torch.tensor([0.25])}, ftype=ftype, hparams_override={"quantization_config": config}, check_lazy=True) + dtype, data = result["blk.0.attn_q.weight"] + if ftype == gguf.LlamaFileType.GUESSED: + self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3) + np.testing.assert_array_equal(result["blk.0.attn_q.scale"][1], [0.25]) + else: + self.assertEqual(dtype, gguf.GGMLQuantizationType.F16) + self.assertEqual(set(result), {"blk.0.attn_q.weight"}) + np.testing.assert_array_equal(data, np.full((4, 4), 0.25)) + + def test_expert_naming(self): + for model_class, prefix, projections, hparams in ( + (LlamaModel, "model.layers.0.block_sparse_moe", (("w1", "gate"), ("w3", "up"), ("w2", "down")), {"num_local_experts": 2}), + (PhiMoeModel, "model.layers.0.block_sparse_moe", (("w1", "gate"), ("w3", "up"), ("w2", "down")), {"max_position_embeddings": 16, "rope_scaling": {"original_max_position_embeddings": 16}}), + (NemotronHModel, "backbone.layers.0.mixer", (("up_proj", "up"), ("down_proj", "down")), {"num_experts_per_tok": 1, "n_routed_experts": 2, "head_dim": 4, "num_heads": 1, "n_groups": 1, "hybrid_override_pattern": "E"}), + ): + for eager in (False, True): + for mixed in (False, True): + with self.subTest(model=model_class.__name__, eager=eager, mixed=mixed): + tensors = {} + for expert in (1, 0): + for proj, _ in projections: + name = f"{prefix}.experts.{expert}.{proj}.weight" + dtype = torch.float16 if mixed and expert == 1 else torch.float8_e4m3fn + tensors[name] = torch.ones((4, 4), dtype=torch.float32).to(dtype) + tensors[name + "_scale"] = torch.tensor([0.25 * (expert + 1)]) + result = self.convert(model_class, tensors, hparams_override=hparams, eager=eager, check_lazy=not eager) + for _, proj in projections: + dtype, data = result[f"blk.0.ffn_{proj}_exps.weight"] + if mixed: + self.assertEqual(dtype, gguf.GGMLQuantizationType.F16) + np.testing.assert_array_equal(data, np.array([np.full((4, 4), 0.25), np.full((4, 4), 0.5)])) + self.assertNotIn(f"blk.0.ffn_{proj}_exps.scale", result) + else: + self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3) + np.testing.assert_array_equal(gguf.dequantize(data, dtype), np.ones((2, 4, 4))) + np.testing.assert_array_equal(result[f"blk.0.ffn_{proj}_exps.scale"][1], [0.25, 0.5]) + def test_mixed_experts(self): for eager in (False, True): for other in ("f16", "channel", "unscaled"):