Conversion fixes

conversion/base.py

@@ -177,6 +177,7 @@ class ModelBase:
         self._nvfp4_layer_algo: dict[str, str | None] = {} # per-layer quant_algo, keyed by HF module path
         self._prec_a4: dict[str, bool] = {} # gguf tensor name -> can use 4-bit (A4) activations
         self._fp8_as_q8 = fp8_as_q8
+        self._preserve_fp8 = ftype == gguf.LlamaFileType.GUESSED and not fp8_as_q8
         self._fp8_dequantized: set[str] = set()
         self._fp8_e4m3_preserved: set[str] = set()
 
@@ -620,22 +621,30 @@ class ModelBase:
         for name, value in new_tensors.items():
             self.model_tensors[name] = value
 
-    def _transform_fp8_scale(self, name: str, scale: Tensor) -> Tensor:
-        return scale
-
     def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
-        return (self.map_tensor_name(name),)
+        new_name = self.tensor_map.get_name(name, try_suffixes=(".weight",))
+        return (new_name,) if new_name is not None else ()
 
     def _prepare_fp8_e4m3_tensors(self):
-        if self._fp8_as_q8:
+        if not self._preserve_fp8:
             return
 
         scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
         input_scale_tensors: dict[str, list[tuple[int, float]] | np.ndarray] = {}
         consumed: list[str] = []
-        candidates: dict[str, tuple[str, Tensor]] = {}
+        candidates: dict[str, tuple[str, Tensor, tuple[str, ...], torch.Size]] = {}
+        weight_dtypes: dict[str, torch.dtype] = {}
         scaled_weights: set[str] = set()
-        expert_pattern = re.compile(r"\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$")
+        expert_pattern = re.compile(r"(\.[^.]*experts)\.(\d+)\.")
+        scale_types = {
+            gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V,
+            gguf.MODEL_TENSOR.ATTN_OUT, gguf.MODEL_TENSOR.ATTN_QKV, gguf.MODEL_TENSOR.ATTN_GATE,
+            gguf.MODEL_TENSOR.FFN_GATE, gguf.MODEL_TENSOR.FFN_UP, gguf.MODEL_TENSOR.FFN_DOWN,
+            gguf.MODEL_TENSOR.FFN_GATE_SHEXP, gguf.MODEL_TENSOR.FFN_UP_SHEXP, gguf.MODEL_TENSOR.FFN_DOWN_SHEXP,
+            gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
+            gguf.MODEL_TENSOR.SSM_IN, gguf.MODEL_TENSOR.SSM_OUT, gguf.MODEL_TENSOR.SSM_ALPHA, gguf.MODEL_TENSOR.SSM_BETA,
+            gguf.MODEL_TENSOR.NEXTN_EH_PROJ, gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, gguf.MODEL_TENSOR.OUTPUT,
+        }
 
         for scale_name in list(self.model_tensors.keys()):
             weight_name = None
@@ -651,6 +660,7 @@ class ModelBase:
 
             scaled_weights.add(weight_name)
             weight = self.model_tensors[weight_name]()
+            weight_dtypes[weight_name] = weight.dtype
             if weight.dtype != torch.float8_e4m3fn or weight.ndim < 2:
                 continue
 
@@ -658,28 +668,33 @@ class ModelBase:
             if prod(scale.shape) != 1:
                 continue
 
-            merged_name = re.sub(r"(\.experts)\.\d+\.", r"\1.", weight_name)
+            merged_name = expert_pattern.sub(r"\1.", weight_name)
             new_names = self._map_fp8_weight_names(merged_name)
             if not new_names:
                 continue
+            if any((mapped := self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",))) is None or mapped[0] not in scale_types for new_name in new_names):
+                continue
             bid = next((int(part) for part in new_names[0].split(".") if part.isdecimal()), None)
             self._fp8_e4m3_preserved.add(weight_name)
             if any(new_name.endswith("_norm.weight") or self.tensor_force_quant(weight_name, new_name, bid, weight.ndim) != gguf.GGMLQuantizationType.F8_E4M3 for new_name in new_names):
                 self._fp8_e4m3_preserved.discard(weight_name)
                 continue
-            candidates[weight_name] = (scale_name, scale)
+            candidates[weight_name] = (scale_name, scale, new_names, weight.shape)
 
         expert_groups: dict[str, list[str]] = {}
         for name in self.model_tensors:
-            if expert_pattern.search(name):
-                expert_groups.setdefault(name.split(".experts.")[0], []).append(name)
+            if (match := expert_pattern.search(name)) and name.endswith(".weight"):
+                expert_groups.setdefault(name[:match.end(1)], []).append(name)
         for names in expert_groups.values():
-            dtypes = {self.model_tensors[name]().dtype for name in names}
+            for name in names:
+                if name not in weight_dtypes:
+                    weight_dtypes[name] = self.model_tensors[name]().dtype
+            dtypes = {weight_dtypes[name] for name in names}
             if torch.float8_e4m3fn not in dtypes:
                 continue
             if dtypes == {torch.float8_e4m3fn} and not any(name in scaled_weights for name in names):
                 continue
-            # The converter emits all three projections when the last expert arrives.
+            # The converter merges expert projections together.
             if not all(name in candidates for name in names):
                 for name in names:
                     candidates.pop(name, None)
@@ -688,10 +703,8 @@ class ModelBase:
                         gen = self.model_tensors[name]
                         self.model_tensors[name] = lambda gen=gen: gen().float()
 
-        for weight_name, (scale_name, scale) in candidates.items():
-            weight = self.model_tensors[weight_name]()
+        for weight_name, (scale_name, scale, new_names, shape) in candidates.items():
             scale = LazyTorchTensor.to_eager(scale).float().flatten()
-            scale = self._transform_fp8_scale(weight_name, scale)
 
             weight_prefix = weight_name.removesuffix(".weight")
             # Transformers fine-grained FP8 uses activation_scale while ModelOpt uses input_scale.
@@ -714,28 +727,25 @@ class ModelBase:
 
             expert_match = expert_pattern.search(weight_name)
             if expert_match:
-                bid = int(expert_match.group(1))
                 expert_id = int(expert_match.group(2))
-                proj_type = expert_match.group(3)
-                merged_name = f"model.layers.{bid}.mlp.experts.{proj_type}.weight"
-                new_name = self.map_tensor_name(merged_name)
-                target_name = new_name.replace(".weight", ".scale")
-                entries = scale_tensors.setdefault(target_name, [])
-                assert isinstance(entries, list)
-                cast(list[tuple[int, float]], entries).append((expert_id, float(scale[0])))
-                if input_scale is not None:
-                    target_name = new_name.replace(".weight", ".input_scale")
-                    entries = input_scale_tensors.setdefault(target_name, [])
+                for new_name in new_names:
+                    target_name = new_name.replace(".weight", ".scale")
+                    entries = scale_tensors.setdefault(target_name, [])
                     assert isinstance(entries, list)
-                    cast(list[tuple[int, float]], entries).append((expert_id, float(input_scale[0])))
+                    cast(list[tuple[int, float]], entries).append((expert_id, float(scale[0])))
+                    if input_scale is not None:
+                        target_name = new_name.replace(".weight", ".input_scale")
+                        entries = input_scale_tensors.setdefault(target_name, [])
+                        assert isinstance(entries, list)
+                        cast(list[tuple[int, float]], entries).append((expert_id, float(input_scale[0])))
             else:
-                for new_name in self._map_fp8_weight_names(weight_name):
-                    mapped = self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",)) if weight.ndim == 3 else None
+                for new_name in new_names:
+                    mapped = self.tensor_map.get_type_and_name(new_name, try_suffixes=(".weight",)) if len(shape) == 3 else None
                     is_packed_expert = mapped is not None and mapped[0] in (
                         gguf.MODEL_TENSOR.FFN_GATE_EXP, gguf.MODEL_TENSOR.FFN_UP_EXP, gguf.MODEL_TENSOR.FFN_DOWN_EXP,
                     )
                     # Packed expert weights need one scale value per expert.
-                    n_scales = weight.shape[0] if is_packed_expert else 1
+                    n_scales = shape[0] if is_packed_expert else 1
                     scale_tensors[new_name.replace(".weight", ".scale")] = np.repeat(scale.numpy(), n_scales)
                     if input_scale is not None:
                         input_scale_tensors[new_name.replace(".weight", ".input_scale")] = np.repeat(input_scale.numpy(), n_scales)
@@ -1432,7 +1442,8 @@ class ModelBase:
             if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
                 continue
 
-            preserve_fp8_e4m3 = not self._fp8_as_q8 and data_torch.dtype == torch.float8_e4m3fn
+            # FP8 transforms must only reorder values; norms and vectors use numeric F32 transforms.
+            preserve_fp8_e4m3 = self._preserve_fp8 and data_torch.dtype == torch.float8_e4m3fn and data_torch.ndim >= 2 and not name.endswith(("norm.weight", "norm_f.weight"))
             if preserve_fp8_e4m3:
                 self._fp8_e4m3_preserved.add(name)
 

conversion/nemotron.py

@@ -433,6 +433,11 @@ class NemotronHModel(GraniteHybridModel):
     _ATTN_LAYER_TYPES = {"attention", "full_attention"}
     _MLP_LAYER_TYPES = {"moe"}
 
+    def _map_fp8_weight_names(self, name: str) -> tuple[str, ...]:
+        if ".mixer.experts." in name:
+            name = name.replace("backbone.", "model.").replace(".mixer.experts.", ".mlp.experts.")
+        return super()._map_fp8_weight_names(name)
+
     def __init__(self, *args, **kwargs):
         # We have to determine the correct model architecture (MoE vs non-MoE) before
         # calling the parent __init__. This is because the parent constructor

conversion/qwen.py

@@ -519,24 +519,6 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
         perm[dim], perm[dim + 1] = perm[dim + 1], perm[dim]
         return tensor.permute(*perm).contiguous().reshape(*shape)
 
-    def _transform_fp8_scale(self, name: str, scale: Tensor) -> Tensor:
-        num_k_heads = self.hparams.get("linear_num_key_heads", 0)
-        num_v_heads = self.hparams.get("linear_num_value_heads", 0)
-        if scale.numel() == 1 or num_k_heads == 0 or num_v_heads == 0 or num_k_heads == num_v_heads:
-            return scale
-
-        num_v_per_k = num_v_heads // num_k_heads
-        head_v_dim = self.hparams["linear_value_head_dim"]
-        if name.endswith(".linear_attn.in_proj_qkv.weight"):
-            qk_dim = 2 * self.hparams["linear_key_head_dim"] * num_k_heads
-            v_scale = self._reorder_v_heads(scale[qk_dim:], 0, num_k_heads, num_v_per_k, head_v_dim)
-            return torch.cat([scale[:qk_dim], v_scale])
-        if name.endswith(".linear_attn.in_proj_z.weight"):
-            return self._reorder_v_heads(scale, 0, num_k_heads, num_v_per_k, head_v_dim)
-        if name.endswith((".linear_attn.in_proj_a.weight", ".linear_attn.in_proj_b.weight")):
-            return self._reorder_v_heads(scale, 0, num_k_heads, num_v_per_k, 1)
-        return scale
-
     def _transform_nvfp4_weight(self, name: str, weight: Tensor, scale: Tensor) -> tuple[Tensor, Tensor]:
         if not name.endswith((
             ".linear_attn.in_proj_qkv.weight",

convert_hf_to_gguf.py

@@ -58,7 +58,7 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--outtype", type=str, choices=["f32", "f16", "bf16", "q8_0", "tq1_0", "tq2_0", "auto"], default="auto",
-        help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto for the highest-fidelity 16-bit float type",
+        help="output format - use f32 for float32, f16 for float16, bf16 for bfloat16, q8_0 for Q8_0, tq1_0 or tq2_0 for ternary, and auto to preserve supported FP8 weights and otherwise select the highest-fidelity 16-bit float type",
     )
     parser.add_argument(
         "--bigendian", action="store_true",

tests/test_conversion_fp8.py

@@ -21,6 +21,9 @@ from safetensors.torch import save_file
 
 from conversion.base import LazyTorchTensor, gguf
 from conversion.deepseek import DeepseekV2Model
+from conversion.llama import LlamaModel
+from conversion.nemotron import NemotronHModel
+from conversion.phi import PhiMoeModel
 from conversion.qwen import Qwen2MoeModel
 
 
@@ -33,8 +36,17 @@ class ForcedF16Model(Qwen2MoeModel):
         return super().tensor_force_quant(name, new_name, bid, n_dims)
 
 
+class OffsetNormModel(Qwen2MoeModel):
+    model_arch = gguf.MODEL_ARCH.QWEN2MOE
+
+    def modify_tensors(self, data_torch, name, bid):
+        if name.endswith("input_layernorm.weight"):
+            data_torch = data_torch + 1.0
+        yield from super().modify_tensors(data_torch, name, bid)
+
+
 class TestFP8Conversion(unittest.TestCase):
-    def convert(self, model_class, tensors, eager=False, check_lazy=False, scan_nvfp4=False, **kwargs):
+    def convert(self, model_class, tensors, eager=False, check_lazy=False, scan_nvfp4=False, ftype=gguf.LlamaFileType.GUESSED, hparams_override=None, **kwargs):
         with tempfile.TemporaryDirectory() as directory:
             path = Path(directory)
             hparams = {
@@ -49,9 +61,10 @@ class TestFP8Conversion(unittest.TestCase):
                 "v_head_dim": 2,
                 "quantization_config": {"quant_method": "modelopt"},
             }
+            hparams.update(hparams_override or {})
             (path / "config.json").write_text(json.dumps(hparams))
             save_file(tensors, str(path / "model.safetensors"))
-            model = model_class(path, gguf.LlamaFileType.MOSTLY_F16, path / "out.gguf", hparams=hparams, eager=eager, **kwargs)
+            model = model_class(path, ftype, path / "out.gguf", hparams=hparams, eager=eager, **kwargs)
             reads: list[str] = []
             if check_lazy:
                 self.assertFalse(eager)
@@ -143,6 +156,97 @@ class TestFP8Conversion(unittest.TestCase):
         self.assertEqual(dtype, gguf.GGMLQuantizationType.F32)
         np.testing.assert_array_equal(data, [1., -1.])
 
+    def test_fp8_norm_transform(self):
+        for shape in ((4,), (2, 2)):
+            with self.subTest(shape=shape):
+                weight = torch.ones(shape, dtype=torch.float32).to(torch.float8_e4m3fn)
+                result = self.convert(OffsetNormModel, {"model.layers.0.input_layernorm.weight": weight}, check_lazy=True)
+                dtype, data = result["blk.0.attn_norm.weight"]
+                self.assertEqual(dtype, gguf.GGMLQuantizationType.F32)
+                np.testing.assert_array_equal(data, np.full(shape, 2.))
+
+    def test_explicit_outtypes(self):
+        name = "model.layers.0.self_attn.q_proj.weight"
+        weight = torch.ones((4, 256), dtype=torch.float32).to(torch.float8_e4m3fn)
+        for ftype, qtype in (
+            (gguf.LlamaFileType.ALL_F32, gguf.GGMLQuantizationType.F32),
+            (gguf.LlamaFileType.MOSTLY_F16, gguf.GGMLQuantizationType.F16),
+            (gguf.LlamaFileType.MOSTLY_BF16, gguf.GGMLQuantizationType.BF16),
+            (gguf.LlamaFileType.MOSTLY_Q8_0, gguf.GGMLQuantizationType.Q8_0),
+            (gguf.LlamaFileType.MOSTLY_TQ1_0, gguf.GGMLQuantizationType.TQ1_0),
+            (gguf.LlamaFileType.MOSTLY_TQ2_0, gguf.GGMLQuantizationType.TQ2_0),
+        ):
+            for eager in (False, True):
+                for scaled in (False, True):
+                    with self.subTest(ftype=ftype, eager=eager, scaled=scaled):
+                        tensors = {name: weight}
+                        if scaled:
+                            tensors[name + "_scale"] = torch.tensor([0.25])
+                            tensors[name.removesuffix("weight") + "input_scale"] = torch.tensor([0.5])
+                        result = self.convert(Qwen2MoeModel, tensors, ftype=ftype, eager=eager, check_lazy=not eager)
+                        self.assertEqual(set(result), {"blk.0.attn_q.weight"})
+                        dtype, data = result["blk.0.attn_q.weight"]
+                        self.assertEqual(dtype, qtype)
+                        np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 256), 0.25 if scaled else 1.), atol=1e-4)
+
+    def test_embedding_scale_fallback(self):
+        name = "model.embed_tokens.weight"
+        weight = torch.ones((4, 4), dtype=torch.float32).to(torch.float8_e4m3fn)
+        for eager in (False, True):
+            with self.subTest(eager=eager):
+                result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale": torch.tensor([0.25])}, eager=eager, check_lazy=not eager)
+                self.assertEqual(set(result), {"token_embd.weight"})
+                dtype, data = result["token_embd.weight"]
+                self.assertEqual(dtype, gguf.GGMLQuantizationType.F16)
+                np.testing.assert_array_equal(data, np.full((4, 4), 0.25))
+
+    def test_fp8_scale_formats(self):
+        name = "model.layers.0.self_attn.q_proj.weight"
+        weight = torch.ones((4, 4), dtype=torch.float32).to(torch.float8_e4m3fn)
+        for suffix, config in (
+            ("_scale_inv", {"quant_method": "fp8"}),
+            ("_scale", {"quant_method": "compressed-tensors", "format": "float-quantized", "config_groups": {"group_0": {"weights": {"type": "float", "num_bits": 8, "strategy": "tensor"}}}}),
+        ):
+            for ftype in (gguf.LlamaFileType.GUESSED, gguf.LlamaFileType.MOSTLY_F16):
+                with self.subTest(suffix=suffix, ftype=ftype):
+                    result = self.convert(Qwen2MoeModel, {name: weight, name + suffix: torch.tensor([0.25])}, ftype=ftype, hparams_override={"quantization_config": config}, check_lazy=True)
+                    dtype, data = result["blk.0.attn_q.weight"]
+                    if ftype == gguf.LlamaFileType.GUESSED:
+                        self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3)
+                        np.testing.assert_array_equal(result["blk.0.attn_q.scale"][1], [0.25])
+                    else:
+                        self.assertEqual(dtype, gguf.GGMLQuantizationType.F16)
+                        self.assertEqual(set(result), {"blk.0.attn_q.weight"})
+                        np.testing.assert_array_equal(data, np.full((4, 4), 0.25))
+
+    def test_expert_naming(self):
+        for model_class, prefix, projections, hparams in (
+            (LlamaModel, "model.layers.0.block_sparse_moe", (("w1", "gate"), ("w3", "up"), ("w2", "down")), {"num_local_experts": 2}),
+            (PhiMoeModel, "model.layers.0.block_sparse_moe", (("w1", "gate"), ("w3", "up"), ("w2", "down")), {"max_position_embeddings": 16, "rope_scaling": {"original_max_position_embeddings": 16}}),
+            (NemotronHModel, "backbone.layers.0.mixer", (("up_proj", "up"), ("down_proj", "down")), {"num_experts_per_tok": 1, "n_routed_experts": 2, "head_dim": 4, "num_heads": 1, "n_groups": 1, "hybrid_override_pattern": "E"}),
+        ):
+            for eager in (False, True):
+                for mixed in (False, True):
+                    with self.subTest(model=model_class.__name__, eager=eager, mixed=mixed):
+                        tensors = {}
+                        for expert in (1, 0):
+                            for proj, _ in projections:
+                                name = f"{prefix}.experts.{expert}.{proj}.weight"
+                                dtype = torch.float16 if mixed and expert == 1 else torch.float8_e4m3fn
+                                tensors[name] = torch.ones((4, 4), dtype=torch.float32).to(dtype)
+                                tensors[name + "_scale"] = torch.tensor([0.25 * (expert + 1)])
+                        result = self.convert(model_class, tensors, hparams_override=hparams, eager=eager, check_lazy=not eager)
+                        for _, proj in projections:
+                            dtype, data = result[f"blk.0.ffn_{proj}_exps.weight"]
+                            if mixed:
+                                self.assertEqual(dtype, gguf.GGMLQuantizationType.F16)
+                                np.testing.assert_array_equal(data, np.array([np.full((4, 4), 0.25), np.full((4, 4), 0.5)]))
+                                self.assertNotIn(f"blk.0.ffn_{proj}_exps.scale", result)
+                            else:
+                                self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3)
+                                np.testing.assert_array_equal(gguf.dequantize(data, dtype), np.ones((2, 4, 4)))
+                                np.testing.assert_array_equal(result[f"blk.0.ffn_{proj}_exps.scale"][1], [0.25, 0.5])
+
     def test_mixed_experts(self):
         for eager in (False, True):
             for other in ("f16", "channel", "unscaled"):