model : add LiquidAI/LFM3-1.7B and LiquidAI/LFM3-VL-2B (#31252)

* model : add LiquidAI/LFM3-1.7B and LiquidAI/LFM3-VL-2B

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: Xuan-Son Nguyen <son@huggingface.co>
Assisted-by: Claude Opus 5.5

* tests : set the SWA rope base of the LFM3 test model

Assisted-by: Claude Opus 5.5

* llama : enable recurrent state rollback for LFM3

Assisted-by: Claude Opus 5.5

* Apply batched suggestions from code review

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: Tarek Dakhran <github@dakhran.com>

* gguf-py : align the lfm3vl mapping comments

Assisted-by: Claude Opus 5.5

* llama : fix CRLF line endings from the review suggestions

Assisted-by: Claude Opus 5.5

* tests : add the LFM3 Metal fusion baseline

Assisted-by: Claude Opus 5.5

* llama : keep the LFM3-VL depthwise conv kernels in F32 when quantizing

Assisted-by: Claude Opus 5.5

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: Xuan-Son Nguyen <son@huggingface.co>

conversion/__init__.py

@@ -172,6 +172,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "D1OmniModel": "lfm2",
     "Lfm2Model": "lfm2",
     "Lfm2MoeForCausalLM": "lfm2",
+    "Lfm3ForCausalLM": "lfm3",
+    "Lfm3VlForConditionalGeneration": "lfm3",
     "Llama4ForCausalLM": "llama",
     "Llama4ForConditionalGeneration": "llama",
     "LlamaBidirectionalModel": "llama",
@@ -358,6 +360,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Lfm2AudioForConditionalGeneration": "lfm2",
     "D1OmniModel": "lfm2",
     "Lfm2VlForConditionalGeneration": "lfm2",
+    "Lfm3VlForConditionalGeneration": "lfm3",
     "LightOnOCRForConditionalGeneration": "lighton_ocr",
     "Llama4ForConditionalGeneration": "llama4",
     "LlavaForConditionalGeneration": "llava",

conversion/lfm3.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+from typing import Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import MmprojModel, ModelBase, TextModel, gguf
+
+
+@ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-1.7B", "LiquidAI/LFM3-1.7B-Base")
+class LFM3Model(TextModel):
+    model_arch = gguf.MODEL_ARCH.LFM3
+
+    def set_gguf_parameters(self):
+        layer_types = self.hparams["layer_types"]
+        n_kv_heads = self.hparams["num_key_value_heads"]
+
+        # set num_key_value_heads only for attention layers
+        self.hparams["num_key_value_heads"] = [
+            n_kv_heads if lt != "conv" else 0
+            for lt in layer_types
+        ]
+
+        super().set_gguf_parameters()
+
+        self.gguf_writer.add_rope_dimension_count(self.hparams["head_dim"])
+
+        self.gguf_writer.add_sliding_window_pattern([lt == "sliding_attention" for lt in layer_types])
+        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+
+        self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
+
+    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+        # Partial RoPE is represented by masking trailing frequency pairs.
+        # rope_parameters is keyed by layer type; only full_attention layers use partial RoPE.
+        rope_parameters = self.hparams["rope_parameters"]["full_attention"]
+        partial = rope_parameters.get("partial_rotary_factor", 1.0)
+        head_dim = self.hparams["head_dim"]
+        rope_angles = int(partial * head_dim) // 2
+        n_pairs = head_dim // 2
+        assert 0 < rope_angles <= n_pairs
+        values = [1.0] * rope_angles + [1e30] * (n_pairs - rope_angles)
+        rope_freqs = torch.tensor(values, dtype=torch.float32)
+        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # conv op requires 2d tensor
+        if "conv.conv" in name:
+            data_torch = data_torch.squeeze(1)
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("Lfm3VlForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-VL-2B")
+class LFM3VLMmprojModel(MmprojModel):
+    has_vision_encoder = True
+    has_audio_encoder = False
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        assert self.hparams_vision is not None
+        # dynamic resolution, image_size is only set for MmprojModel compatibility
+        self.hparams_vision["image_size"] = 512
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        assert self.hparams_vision is not None
+        # clip.cpp hardcodes the vision rope base, the RGB patchifier, the 3x3 spatial conv and the GELU projector
+        if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
+                or self.hparams_vision["conv_kernel_size"] != 3):
+            raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
+        if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
+                or self.global_config["projector_layer_norm_eps"] != 1e-6):
+            raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
+        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.LFM3VL)
+        self.gguf_writer.add_vision_head_count_kv(self.hparams_vision["num_key_value_heads"])
+        self.gguf_writer.add_vision_spatial_merge_size(self.global_config["downsample_factor"])
+        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["norm_eps"])
+        self.gguf_writer.add_vision_use_silu(True)
+        size = self.preprocessor_config["size"]
+        self.gguf_writer.add_vision_min_pixels(size["shortest_edge"])
+        self.gguf_writer.add_vision_max_pixels(size["longest_edge"])
+        # PIL resample enum, the torchvision backend has no lanczos
+        resample = self.preprocessor_config["resample"]
+        resize_algo = {2: "bilinear", 3: "bicubic"}.get(resample)
+        if resize_algo is None:
+            raise ValueError(f"unsupported resample: {resample}")
+        self.gguf_writer.add_vision_image_resize_algo(resize_algo)
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        if "patch_embd" in new_name:
+            return gguf.GGMLQuantizationType.F16
+        if "conv2d_spatial" in new_name:
+            # CUDA CONV_2D_DW requires an f32 kernel; the tensor is tiny
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if not name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
+            return
+
+        if name == "model.vision_tower.patchifier.weight":
+            # qwen2vl-style layout [channel, temporal, ph, pw], one conv kernel per temporal slice
+            assert self.hparams_vision is not None
+            t = self.hparams_vision["temporal_patch_size"]
+            p = self.hparams_vision["patch_size"]
+            n_embd = data_torch.shape[0]
+            w = data_torch.view(n_embd, 3, t, p, p)
+            yield ("v.patch_embd.weight", w[:, :, 0].contiguous())
+            if t == 2:
+                yield ("v.patch_embd.weight.1", w[:, :, 1].contiguous())
+            else:
+                assert t == 1, f"unsupported vision_temporal_patch_size {t}"
+            return
+        if name == "model.vision_tower.embedding_norm.weight":
+            yield ("v.embedding_norm.weight", data_torch)
+            return
+        if "conv.in_proj.weight" in name:
+            yield (f"v.blk.{bid}.conv2d_in_proj.weight", data_torch)
+            return
+        if "conv.spatial_conv.weight" in name:
+            yield (f"v.blk.{bid}.conv2d_spatial.weight", data_torch)
+            return
+        if "conv.out_proj.weight" in name:
+            yield (f"v.blk.{bid}.conv2d_out_proj.weight", data_torch)
+            return
+
+        # projector: input LayerNorm -> linear_1 -> GELU -> linear_2
+        if name.startswith("model.multi_modal_projector."):
+            layer, suffix = name.removeprefix("model.multi_modal_projector.").split(".")
+            yield ({"layer_norm": "mm.input_norm", "linear_1": "mm.0", "linear_2": "mm.2"}[layer] + "." + suffix, data_torch)
+            return
+
+        yield (self.map_tensor_name(name), data_torch)

gguf-py/gguf/constants.py

@@ -691,6 +691,7 @@ class MODEL_ARCH(IntEnum):
     GPT_OSS          = auto()
     LFM2             = auto()
     LFM2MOE          = auto()
+    LFM3             = auto()
     DREAM            = auto()
     SMALLTHINKER     = auto()
     LLADA            = auto()
@@ -1523,6 +1524,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.GPT_OSS:          "gpt-oss",
     MODEL_ARCH.LFM2:             "lfm2",
     MODEL_ARCH.LFM2MOE:          "lfm2moe",
+    MODEL_ARCH.LFM3:             "lfm3",
     MODEL_ARCH.DREAM:            "dream",
     MODEL_ARCH.SMALLTHINKER:     "smallthinker",
     MODEL_ARCH.LLADA:            "llada",
@@ -5522,6 +5524,29 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_UP_EXP,
         MODEL_TENSOR.FFN_EXP_PROBS_B,
     ],
+    MODEL_ARCH.LFM3: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ROPE_FREQS,
+        MODEL_TENSOR.SHORTCONV_CONV,
+        MODEL_TENSOR.SHORTCONV_INPROJ,
+        MODEL_TENSOR.SHORTCONV_OUTPROJ,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.ATTN_NORM, # operator_norm
+        MODEL_TENSOR.ATTN_POST_NORM,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_GATE,
+    ],
     MODEL_ARCH.SMALLTHINKER: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -6507,6 +6532,7 @@ class VisionProjectorType:
     VOXTRAL_RT = "voxtral_rt"
     MERALION = "meralion"  # audio: Whisper + gated MLP adaptor
     LFM2 = "lfm2"
+    LFM3VL = "lfm3vl"
     KIMIVL = "kimivl"
     PADDLEOCR = "paddleocr"
     KIMIK25 = "kimik25"

gguf-py/gguf/tensor_mapping.py

@@ -119,6 +119,7 @@ class TensorNameMap:
             "model.transformer.ln_f",                  # llada
             "final_norm",                              # modern-bert
             "model.norm",                              # cogvlm
+            "model.embedding_norm",                    # lfm3
         ),
 
         # Rope frequencies
@@ -418,6 +419,7 @@ class TensorNameMap:
             "model.layers.{bid}.post_self_attn_layernorm",       # glm-4-0414
             "model.layers.{bid}.post_attn_norm",                 # kolibri1
             "model.layers.layers.{bid}.post_mixer_norm.weight",  # plamo2
+            "model.layers.{bid}.post_operator_norm",             # lfm3
         ),
 
         # Rotary embeddings
@@ -504,6 +506,7 @@ class TensorNameMap:
             "model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2
             "model.layers.{bid}.feed_forward.up_proj",
             "model.layers.{bid}.post_moe_norm",               # grok-2
+            "model.layers.{bid}.post_ffn_norm",               # lfm3
         ),
 
         MODEL_TENSOR.FFN_POST_NORM_1: (
@@ -1855,6 +1858,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.q_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.q_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wq", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.q_proj", # lfm3vl
             "visual.blocks.{bid}.attn.q", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wq", # kimi-vl, generated
             "siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl
@@ -1870,6 +1874,7 @@ class TensorNameMap:
             "model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
             "visual.blocks.{bid}.attn.q_norm", # GLM-OCR
             "vision_model.model.layers.{bid}.self_attn.q_norm", # gemma4
+            "model.vision_tower.layers.{bid}.self_attn.q_layernorm", # lfm3vl
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_K: (
@@ -1884,6 +1889,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.k_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.k_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wk", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.k_proj", # lfm3vl
             "visual.blocks.{bid}.attn.k", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wk", # kimi-vl, generated
             "model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated
@@ -1899,6 +1905,7 @@ class TensorNameMap:
             "model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
             "visual.blocks.{bid}.attn.k_norm", # GLM-OCR
             "vision_model.model.layers.{bid}.self_attn.k_norm", # gemma4
+            "model.vision_tower.layers.{bid}.self_attn.k_layernorm", # lfm3vl
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_V: (
@@ -1913,6 +1920,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.v_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.v_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wv", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.v_proj", # lfm3vl
             "visual.blocks.{bid}.attn.v", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wv", # kimi-vl, generated
             "siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj",
@@ -1935,6 +1943,7 @@ class TensorNameMap:
             "vit.layers.{bid}.input_layernorm", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.attention_norm", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention_norm", # pixtral
+            "model.vision_tower.layers.{bid}.operator_norm", # lfm3vl
             "vision_model.model.layers.{bid}.input_layernorm", # llama4, gemma4
             "visual.blocks.{bid}.norm1", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm0", # kimi-vl (norm0/norm1)
@@ -1964,6 +1973,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.o_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.o_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wo", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.out_proj", # lfm3vl
             "visual.blocks.{bid}.attn.proj", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.wo", # kimi-vl
             "model.vision.transformer.layers.{bid}.attention.dense", # cogvlm
@@ -1996,6 +2006,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.post_attention_layernorm", # llama4
             "vision_tower.transformer.layers.{bid}.ffn_norm", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.ffn_norm", # pixtral
+            "model.vision_tower.layers.{bid}.ffn_norm", # lfm3vl
             "visual.blocks.{bid}.norm2", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm1", # kimi-vl (norm0/norm1)
             "model.vision.transformer.layers.{bid}.post_attention_layernorm", # cogvlm
@@ -2023,6 +2034,7 @@ class TensorNameMap:
             "vit.layers.{bid}.mlp.dense_h_to_4h", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.feed_forward.up_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w3", # pixtral
+            "model.vision_tower.layers.{bid}.feed_forward.w3", # lfm3vl
             "vision_model.model.layers.{bid}.mlp.fc1", # llama4
             "visual.blocks.{bid}.mlp.fc1", # qwen2vl
             "visual.blocks.{bid}.mlp.up_proj", # qwen2.5vl
@@ -2043,6 +2055,7 @@ class TensorNameMap:
             "vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
             "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
+            "model.vision_tower.layers.{bid}.feed_forward.w1", # lfm3vl
             "visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
             "vision_model.model.layers.{bid}.mlp.gate_proj", # gemma4
             "model.qwen2_model.model.model.layers.{bid}.mlp.gate_proj", # Deepseek-OCR-2 qwen2
@@ -2061,6 +2074,7 @@ class TensorNameMap:
             "vit.layers.{bid}.mlp.dense_4h_to_h", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.feed_forward.down_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w2", # pixtral
+            "model.vision_tower.layers.{bid}.feed_forward.w2", # lfm3vl
             "vision_model.model.layers.{bid}.mlp.fc2", # llama4
             "visual.blocks.{bid}.mlp.fc2", # qwen2vl
             "visual.blocks.{bid}.mlp.down_proj", # qwen2.5vl

models/templates/LFM3.jinja

@@ -0,0 +1,280 @@
+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+{%- set enable_thinking = enable_thinking | default(true) -%}
+
+{%- macro render_value(v) -%}
+    {%- if v is string -%}
+        {{- v -}}
+    {%- elif v is boolean -%}
+        {{- "true" if v else "false" -}}
+    {%- elif v is none -%}
+        {{- "null" -}}
+    {%- elif v is mapping or (v is sequence and v is not string) -%}
+        {{- v | tojson -}}
+    {%- else -%}
+        {{- v -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping and content.get("type") == "image" -%}
+        {{- "<|vision_start|><|image|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "video" -%}
+        {{- "<|vision_start|><|video|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "text" -%}
+        {{- (content.get("text") or "") | string -}}
+    {%- elif content is mapping -%}
+        {{- content | tojson -}}
+    {%- elif content is iterable -%}
+        {%- set _ns = namespace(result="") -%}
+        {%- for item in content -%}
+            {%- if item is string -%}
+                {%- set _ns.result = _ns.result + item -%}
+            {%- elif item is mapping and item.get("type") == "image" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|image|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "video" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|video|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "text" -%}
+                {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
+            {%- else -%}
+                {%- set _ns.result = _ns.result + (item | tojson) -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- _ns.result -}}
+    {%- elif content is not none -%}
+        {{- content | tojson -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+    {%- for tool_call in tool_calls -%}
+        {%- if not loop.first -%}{{- "\n\n" -}}{%- endif -%}
+        {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
+        {%- set func_args = func.get("arguments") -%}
+        {%- if func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
+            {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
+        {%- endif -%}
+        {{- "<tool_call>\n<function=" + func["name"] + ">\n" -}}
+        {%- if func_args is mapping -%}
+            {%- for arg_name, arg_value in func_args.items() -%}
+                {{- "<parameter=" + arg_name + ">\n" + render_value(arg_value) + "\n</parameter>\n" -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- "</function>\n</tool_call>" -}}
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro tool_to_json(tool) -%}
+    {%- set fn = tool.function if tool.function is defined else tool -%}
+    {%- set ns_t = namespace(first=true) -%}
+    {{- "{" -}}
+    {%- for k, v in fn.items() -%}
+        {%- if k != "defer_loading" -%}
+            {%- if not ns_t.first -%}{{- ", " -}}{%- endif -%}
+            {%- set ns_t.first = false -%}
+            {{- k | tojson -}}{{- ": " -}}{{- v | tojson -}}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- "}" -}}
+{%- endmacro -%}
+
+{%- macro is_tool_reference(item) -%}
+    {{- "1" if item is mapping and item.get("type") == "tool_reference" else "" -}}
+{%- endmacro -%}
+
+{%- macro referenced_tool_to_json(name) -%}
+    {%- set _ns = namespace(found=none) -%}
+    {%- for tool in tools or [] -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- if fn.name == name -%}
+            {%- set _ns.found = fn -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if _ns.found is none -%}
+        {{- raise_exception("Tool reference '" + name | string + "' not found in available tools. Every tool referenced by a tool_reference must be in `tools` on every request.") -}}
+    {%- endif -%}
+    {{- tool_to_json(_ns.found) -}}
+{%- endmacro -%}
+
+{%- macro render_tool_response(content) -%}
+    {%- if content is string and "<tool_response>" in content -%}
+        {{- content -}}
+    {%- else -%}
+    {{- "<tool_response>\n" -}}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping -%}
+        {{- parse_content(content) -}}
+    {%- elif content is iterable -%}
+        {%- for item in content -%}
+            {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+            {%- if is_tool_reference(item) -%}
+                {%- if loop.first or not is_tool_reference(loop.previtem) -%}
+                    {{- "<additional_tools>\n" -}}
+                {%- endif -%}
+                {{- referenced_tool_to_json(item.name) -}}
+                {%- if loop.last or not is_tool_reference(loop.nextitem) -%}
+                    {{- "\n</additional_tools>" -}}
+                {%- endif -%}
+            {%- elif item is mapping and item.result is defined and item.result is mapping and item.result.tools is defined -%}
+                {{- "<additional_tools>\n" -}}
+                {%- set _tl = item.result.tools -%}
+                {%- if _tl is string -%}
+                    {{- _tl -}}
+                {%- elif _tl is mapping -%}
+                    {{- tool_to_json(_tl) -}}
+                {%- else -%}
+                    {%- for t in _tl -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>" -}}
+            {%- else -%}
+                {{- parse_content(item) -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- elif content is not none -%}
+        {{- content | tojson -}}
+    {%- endif -%}
+    {{- "\n</tool_response>" -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages and messages[0]["role"] == "system" -%}
+    {%- if messages[0].get("content") -%}
+        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+    {%- endif -%}
+    {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set tools_ns = namespace(lines=[]) -%}
+    {%- for tool in tools -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- set _dyn = (fn.defer_loading is defined and fn.defer_loading) or (tool.defer_loading is defined and tool.defer_loading) -%}
+        {%- if not _dyn -%}
+            {%- set tools_ns.lines = tools_ns.lines + [tool_to_json(fn)] -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if tools_ns.lines -%}
+        {%- set ns.system_prompt = ns.system_prompt + ("\n\n" if ns.system_prompt else "") + "<tools>\n" + (tools_ns.lines | join("\n")) + "\n</tools>" -%}
+    {%- endif -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+    {%- if message["role"] == "user" -%}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- set carry = namespace(pending=[]) -%}
+{%- for message in messages -%}
+    {%- set _next = messages[loop.index0 + 1] if not loop.last else none -%}
+    {%- set _merge = message.role == "tool" and not message.get("additional_tools")
+        and _next is not none and _next.role == "tool" and not _next.get("additional_tools")
+        and message.get("tool_call_id") and _next.get("tool_call_id") == message.get("tool_call_id") -%}
+    {%- set _items = [] -%}
+    {%- if message.role == "tool" and not message.get("additional_tools") -%}
+        {%- set _c = message.get("content") -%}
+        {%- if _c is string -%}
+            {%- set _items = [{"type": "text", "text": _c}] if _c else [] -%}
+        {%- elif _c is mapping -%}
+            {%- set _items = [_c] -%}
+        {%- elif _c is iterable -%}
+            {%- set _items = _c | list -%}
+        {%- elif _c is not none -%}
+            {%- set _items = [_c] -%}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if _merge -%}
+        {%- set carry.pending = carry.pending + _items -%}
+    {%- else -%}
+    {{- "<|im_start|>" + message.role + "\n" -}}
+    {%- if message.role == "assistant" -%}
+        {%- generation -%}
+        {%- set thinking = message.thinking if message.thinking is defined else message.reasoning if message.reasoning is defined else message.reasoning_content -%}
+        {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
+        {%- if thinking is defined and keep_thinking -%}
+            {{- "<think>" + (thinking or "") + "</think>" -}}
+        {%- elif keep_thinking and "</think>" not in (message.get("content") if message.get("content") is string else "") -%}
+            {{- "<think></think>" -}}
+        {%- endif -%}
+        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+        {%- set _has_cfm = false -%}
+        {%- set content = "" -%}
+        {%- if message.get("content") -%}
+            {%- set content = parse_content(message.content) -%}
+        {%- endif -%}
+        {%- if not keep_thinking -%}
+            {%- if "</think>" in content -%}
+                {%- set content = content.split("</think>")[-1] | trim -%}
+            {%- endif -%}
+            {%- set content = "<think></think>" + content -%}
+        {%- endif -%}
+        {%- if content.endswith(_cfm_tag) -%}
+            {%- set _has_cfm = true -%}
+            {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+            {%- set content = content[:_trunc_len] -%}
+        {%- endif -%}
+        {{- content -}}
+        {%- if message.tool_calls -%}
+            {{- render_tool_calls(message.tool_calls) -}}
+        {%- endif -%}
+        {%- if _has_cfm -%}
+            {{- _cfm_tag -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+        {%- endgeneration -%}
+    {%- elif message.role == "tool" -%}
+        {%- if message.get("additional_tools") -%}
+            {%- set _at = message["additional_tools"] -%}
+            {%- if _at is mapping or (_at is sequence and _at is not string) -%}
+                {%- set _src = _at -%}
+            {%- else -%}
+                {%- set _src = message.get("content") -%}
+            {%- endif -%}
+            {%- if _src is string and "<tool_response>" in _src -%}
+                {{- _src -}}
+            {%- elif _src is string and "<additional_tools>" in _src -%}
+                {{- "<tool_response>\n" + _src + "\n</tool_response>" -}}
+            {%- elif _src -%}
+                {{- "<tool_response>\n<additional_tools>\n" -}}
+                {%- if _src is string -%}
+                    {{- _src -}}
+                {%- elif _src is mapping -%}
+                    {{- tool_to_json(_src) -}}
+                {%- else -%}
+                    {%- for t in _src -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>\n</tool_response>" -}}
+            {%- endif -%}
+        {%- elif carry.pending -%}
+            {{- render_tool_response(carry.pending + _items) -}}
+            {%- set carry.pending = [] -%}
+        {%- else -%}
+            {{- render_tool_response("" if message.get("content") is none else message.get("content")) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- else %}
+        {%- if message.get("content") -%}
+            {{- parse_content(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- endif %}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- "<|im_start|>assistant\n" -}}
+    {%- if enable_thinking -%}
+        {{- "<think>" -}}
+    {%- else -%}
+        {{- "<think></think>" -}}
+    {%- endif -%}
+{%- endif -%}

src/llama-arch.cpp

@@ -135,6 +135,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_OPENAI_MOE,       "gpt-oss"          },
     { LLM_ARCH_LFM2,             "lfm2"             },
     { LLM_ARCH_LFM2MOE,          "lfm2moe"          },
+    { LLM_ARCH_LFM3,             "lfm3"             },
     { LLM_ARCH_DREAM,            "dream"            },
     { LLM_ARCH_SMALLTHINKER,     "smallthinker"     },
     { LLM_ARCH_LLADA,            "llada"            },
@@ -1251,6 +1252,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
         case LLM_ARCH_GRANITE_HYBRID:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_QWEN3NEXT:
@@ -1295,6 +1297,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_BAILINGMOE3:
             return true;
         default:
@@ -1341,6 +1344,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_GRANITE_HYBRID:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_MINIMAX_01:
         case LLM_ARCH_MINIMAX_M2:
         case LLM_ARCH_MINIMAX_M3:

src/llama-arch.h

@@ -139,6 +139,7 @@ enum llm_arch {
     LLM_ARCH_OPENAI_MOE,
     LLM_ARCH_LFM2,
     LLM_ARCH_LFM2MOE,
+    LLM_ARCH_LFM3,
     LLM_ARCH_DREAM,
     LLM_ARCH_SMALLTHINKER,
     LLM_ARCH_LLADA,

src/llama-model.cpp

@@ -314,6 +314,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_lfm2(params);
         case LLM_ARCH_LFM2MOE:
             return new llama_model_lfm2moe(params);
+        case LLM_ARCH_LFM3:
+            return new llama_model_lfm3(params);
         case LLM_ARCH_SMALLTHINKER:
             return new llama_model_smallthinker(params);
         case LLM_ARCH_GROVEMOE:
@@ -3637,6 +3639,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_HY_V3:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_SMALLTHINKER:
         case LLM_ARCH_SEED_OSS:
         case LLM_ARCH_GROVEMOE:

src/llama-quant.cpp

@@ -342,6 +342,9 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
     quantize &= name.find("ssm_conv1d") == std::string::npos;
     quantize &= name.find("shortconv.conv.weight") == std::string::npos;
 
+    // do not quantize LFM3-VL's depthwise conv kernels, CONV_2D_DW takes F32 kernels
+    quantize &= name.find("conv2d_spatial.weight") == std::string::npos;
+
     // do not quantize MiniMax's indexer projection weights, they are tiny
     quantize &= name.find("indexer.k_proj.weight") == std::string::npos;
     quantize &= name.find("indexer.q_proj.weight") == std::string::npos;

src/models/lfm3.cpp

@@ -0,0 +1,197 @@
+#include "models.h"
+#include "../llama-memory-hybrid-iswa.h"
+
+void llama_model_lfm3::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_SHORTCONV_L_CACHE,           hparams.n_shortconv_l_cache);
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+    for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+        hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;
+    }
+
+    // LFM3 always mixes full and sliding window attention layers
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+    if (!hparams.is_swa_any()) {
+        throw std::runtime_error("LFM3 requires sliding window attention layers");
+    }
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+
+    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
+
+    switch (hparams.n_ff()) {
+        case 7680: type = LLM_TYPE_1_7B; break;
+        default:   type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_lfm3::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd    = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  "weight"), {n_embd, n_vocab}, 0);
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (output == NULL) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    int rope_freqs_flag = 0;
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, 0);
+        layer.ffn_norm       = create_tensor(tn(LLM_TENSOR_FFN_NORM,       "weight", i), {n_embd}, 0);
+        // absent when use_sandwich_norm is false
+        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
+        layer.ffn_post_norm  = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM,  "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
+        if ((layer.attn_post_norm == nullptr) != (layer.ffn_post_norm == nullptr)) {
+            throw std::runtime_error("LFM3 requires both post norms or neither");
+        }
+
+        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, 0);
+        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, 0);
+        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, 0);
+
+        if (!hparams.is_recr(i)) {
+            // note: n_embd_head_k * n_head may differ from n_embd
+            const int64_t n_embd_q = n_embd_head_k * n_head;
+            const int64_t n_embd_k = hparams.n_embd_k_gqa(i);
+            const int64_t n_embd_v = hparams.n_embd_v_gqa(i);
+            if (n_embd_k != n_embd_v) {
+                throw std::runtime_error("LFM3 requires equal key and value sizes");
+            }
+
+            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+            create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_k, n_embd_v, 0);
+
+            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);
+
+            layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0);
+
+            if (!hparams.is_swa(i)) {
+                layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_embd_head_k / 2}, rope_freqs_flag);
+                rope_freqs_flag = TENSOR_DUPLICATED;
+            }
+        } else {
+            layer.shortconv.conv     = create_tensor(tn(LLM_TENSOR_SHORTCONV_CONV,    "weight", i), {hparams.n_shortconv_l_cache, n_embd}, 0);
+            layer.shortconv.in_proj  = create_tensor(tn(LLM_TENSOR_SHORTCONV_INPROJ,  "weight", i), {n_embd, 3 * n_embd}, 0);
+            layer.shortconv.out_proj = create_tensor(tn(LLM_TENSOR_SHORTCONV_OUTPROJ, "weight", i), {n_embd, n_embd}, 0);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_lfm3::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params & params) :
+    llm_build_lfm_base(params) {
+    auto build_attn_block = [&model, this](ggml_tensor *                  cur_in,
+                                           ggml_tensor *                  inp_pos,
+                                           llm_graph_input_attn_kv_iswa * inp_attn,
+                                           int                            il) -> ggml_tensor * {
+        const int64_t n_embd_head = hparams.n_embd_head_k(il);
+        const int64_t n_head_il   = hparams.n_head(il);
+        const int64_t n_head_kv   = hparams.n_head_kv(il);
+
+        const int     n_rot_l      = hparams.n_rot(il);
+        const float   freq_base_l  = model.get_rope_freq_base (cparams, il);
+        const float   freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+        ggml_tensor * rope_factors = hparams.is_swa(il) ? nullptr : model.layers[il].rope_freqs;
+
+        auto [q, k, v] = build_qkv(model.layers[il], cur_in, n_embd_head, n_head_il, n_head_kv, il);
+
+        q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
+        cb(q, "model.layers.{}.self_attn.q_layernorm", il);
+        k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
+        cb(k, "model.layers.{}.self_attn.k_layernorm", il);
+
+        q = ggml_rope_ext(ctx0, q, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
+                          freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+        k = ggml_rope_ext(ctx0, k, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
+                          freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+
+        ggml_tensor * attn_out = build_attn(inp_attn,
+                NULL, NULL, NULL,
+                q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
+        cb(attn_out, "attn_out", il);
+
+        ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur_in);
+        gate = ggml_sigmoid(ctx0, gate);
+        cb(gate, "model.layers.{}.self_attn.gate", il);
+
+        ggml_tensor * attn_3d = ggml_reshape_3d(ctx0, attn_out, n_embd_head, n_head_il, n_tokens);
+        ggml_tensor * gate_3d = ggml_reshape_3d(ctx0, gate,     1,           n_head_il, n_tokens);
+        attn_3d = ggml_mul(ctx0, attn_3d, gate_3d);
+        attn_out = ggml_reshape_2d(ctx0, attn_3d, n_embd_head * n_head_il, n_tokens);
+
+        ggml_tensor * out = build_lora_mm(model.layers[il].wo, attn_out, model.layers[il].wo_s);
+        cb(out, "model.layers.{}.self_attn.out_proj", il);
+
+        return out;
+    };
+
+    ggml_tensor * cur = build_inp_embd(model.tok_embd);
+    cb(cur, "model.embed_tokens", -1);
+
+    ggml_build_forward_expand(gf, cur);
+
+    auto * inp_hybrid = build_inp_mem_hybrid_iswa();
+
+    ggml_tensor * inp_pos     = build_inp_pos();
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    for (int il = 0; il < n_layer; ++il) {
+        auto * prev_cur = cur;
+
+        cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "model.layers.{}.operator_norm", il);
+
+        if (hparams.is_recr(il)) {
+            const auto * mctx_recr = static_cast<const llama_memory_hybrid_iswa_context *>(mctx)->get_recr();
+            cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il);
+        } else {
+            cur = build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il);
+        }
+
+        if (model.layers[il].attn_post_norm) {
+            cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
+            cb(cur, "model.layers.{}.post_operator_norm", il);
+        }
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);
+            prev_cur = ggml_get_rows(ctx0, prev_cur, inp_out_ids);
+        }
+
+        cur = ggml_add(ctx0, prev_cur, cur);
+
+        auto * ffn_norm_out = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(ffn_norm_out, "model.layers.{}.ffn_norm", il);
+
+        ggml_tensor * ffn_out = build_lfm_dense_ffn(model, ffn_norm_out, il);
+
+        if (model.layers[il].ffn_post_norm) {
+            ffn_out = build_norm(ffn_out, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);
+            cb(ffn_out, "model.layers.{}.post_ffn_norm", il);
+        }
+
+        cur = ggml_add(ctx0, cur, ffn_out);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+    }
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+    cb(cur, "result_output", -1);
+
+    res->t_logits = cur;
+}

src/models/models.h

@@ -2433,6 +2433,19 @@ struct llama_model_lfm2moe : public llama_model_base {
 };
 
 
+struct llama_model_lfm3 : public llama_model_base {
+    llama_model_lfm3(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_build_lfm_base {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_smallthinker : public llama_model_base {
     llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

tests/fusion/MTL.csv

@@ -155,6 +155,7 @@ laguna              ,0   ,any     ,RMS_NORM+MUL                ,      9
 lfm2                ,0   ,any     ,RMS_NORM+MUL                ,      7
 lfm2moe             ,1   ,any     ,MUL+ADD                     ,      2
 lfm2moe             ,1   ,any     ,RMS_NORM+MUL                ,      7
+lfm3                ,0   ,any     ,RMS_NORM+MUL                ,     17
 llada               ,0   ,any     ,RMS_NORM+MUL                ,      5
 llada-moe           ,1   ,any     ,MUL+ADD                     ,      4
 llada-moe           ,1   ,any     ,RMS_NORM+MUL                ,      9

tests/test-chat.cpp

@@ -5589,6 +5589,47 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
         test_lfm2_parser(tmpl, detailed_debug);
     }
 
+    {
+        auto tst = peg_tester("models/templates/LFM3.jinja", detailed_debug);
+
+        tst.test("Hello, world!\nWhat's up?")
+            .enable_thinking(false)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .expect(message_assist)
+            .run();
+
+        tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")
+            .enable_thinking(true)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .expect(message_assist_thoughts)
+            .run();
+
+        tst.test(
+               "<tool_call>\n"
+               "<function=special_function>\n"
+               "<parameter=arg1>\n1\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .enable_thinking(false)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .tools({ special_function_tool })
+            .expect(message_assist_call)
+            .run();
+
+        tst.test(
+               "I'm\nthinking</think>"
+               "<tool_call>\n"
+               "<function=special_function>\n"
+               "<parameter=arg1>\n1\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .enable_thinking(true)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .tools({ special_function_tool })
+            .expect(message_assist_call_thoughts)
+            .run();
+    }
+
     // Thinking cases only apply to LFM2.5-8B-A1B, the one LFM2 template that emits <think>
     {
         auto tst = peg_tester("models/templates/LFM2.5-8B-A1B.jinja", detailed_debug);

tests/test-llama-archs.cpp

@@ -251,6 +251,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         n_head = 1;
         n_ff   = 96;
         n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
+    } else if (arch == LLM_ARCH_LFM3) {
+        n_layer = 3;
     } else if (arch == LLM_ARCH_DEEPSEEK4) {
         // head size 64 so that GPU flash attention kernels support the model
         n_embd  = 512;
@@ -336,8 +338,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
 
     if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
-            arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR ||
-            arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT) {
+            arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_LFM3 ||
+            arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT
+            ) {
         GGML_ASSERT(n_layer >= 2);
         std::vector<uint32_t> n_head_per_layer;
         n_head_per_layer.reserve(n_layer);
@@ -439,6 +442,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         if (arch == LLM_ARCH_FLINT) {
             ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern);
         }
+    } else if (arch == LLM_ARCH_LFM3) {
+        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>({0, 0, 1}));
+        ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA,               10000.0f);
     } else {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
     }

tools/mtmd/CMakeLists.txt

@@ -58,6 +58,7 @@ add_library(mtmd
             models/sensenova-u1.cpp
             models/minimax-m3.cpp
             models/qwen3vl.cpp
+            models/lfm3vl.cpp
             models/ling3vl.cpp
             models/mimovl.cpp
             models/qwen3a.cpp

tools/mtmd/clip-impl.h

@@ -173,6 +173,12 @@
 #define TN_DEEPSTACK_FC1   "v.deepstack.%d.fc1.%s"      // qwen3vl deepstack
 #define TN_DEEPSTACK_FC2   "v.deepstack.%d.fc2.%s"      // qwen3vl deepstack
 
+// lfm3vl
+#define TN_LFM3VL_EMBEDDING_NORM   "%s.embedding_norm.%s"
+#define TN_LFM3VL_CONV2D_IN_PROJ   "%s.blk.%d.conv2d_in_proj.%s"
+#define TN_LFM3VL_CONV2D_SPATIAL   "%s.blk.%d.conv2d_spatial.%s"
+#define TN_LFM3VL_CONV2D_OUT_PROJ  "%s.blk.%d.conv2d_out_proj.%s"
+
 // mimicpmv
 #define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"
 #define TN_MINICPMV_QUERY      "resampler.query"
@@ -517,6 +523,7 @@ enum projector_type {
     PROJECTOR_TYPE_MERALION,
     PROJECTOR_TYPE_MUSIC_FLAMINGO,
     PROJECTOR_TYPE_LFM2,
+    PROJECTOR_TYPE_LFM3VL,
     PROJECTOR_TYPE_D1OMNI_V,
     PROJECTOR_TYPE_KIMIVL,
     PROJECTOR_TYPE_PADDLEOCR,
@@ -595,6 +602,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_MERALION,          "meralion"},
     { PROJECTOR_TYPE_MUSIC_FLAMINGO,    "musicflamingo"},
     { PROJECTOR_TYPE_LFM2,              "lfm2"},
+    { PROJECTOR_TYPE_LFM3VL,            "lfm3vl"},
     { PROJECTOR_TYPE_D1OMNI_V,          "d1omni_v"},
     { PROJECTOR_TYPE_KIMIVL,            "kimivl"},
     { PROJECTOR_TYPE_PADDLEOCR,         "paddleocr"},

tools/mtmd/clip-model.h

@@ -325,6 +325,11 @@ struct clip_layer {
     ggml_tensor * deepstack_fc2_w = nullptr;
     ggml_tensor * deepstack_fc2_b = nullptr;
 
+    // lfm3vl conv layers
+    ggml_tensor * conv2d_in_proj_w  = nullptr;
+    ggml_tensor * conv2d_spatial_w  = nullptr; // may be nullptr
+    ggml_tensor * conv2d_out_proj_w = nullptr;
+
     // sam rel_pos
     ggml_tensor * rel_pos_w = nullptr;
     ggml_tensor * rel_pos_h = nullptr;
@@ -786,6 +791,9 @@ struct clip_model {
     ggml_tensor * yasa_vision_pos_embed = nullptr;
     std::vector<yasa2_stage> yasa_stages;
 
+    // lfm3vl
+    ggml_tensor * lfm3vl_embedding_norm_w = nullptr;
+
     // pixtral, glm4v
     ggml_tensor * token_embd_img_break = nullptr;
 

tools/mtmd/clip.cpp

@@ -1031,6 +1031,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
             {
                 builder = std::make_unique<clip_graph_sensenova_u1>(ctx, img);
             } break;
+        case PROJECTOR_TYPE_LFM3VL:
+            {
+                builder = std::make_unique<clip_graph_lfm3vl>(ctx, img);
+            } break;
         case PROJECTOR_TYPE_LING3VL:
             {
                 builder = std::make_unique<clip_graph_ling3vl>(ctx, img);
@@ -1876,6 +1880,39 @@ struct clip_model_loader {
                         }
                         hparams.set_warmup_n_tokens(16*16);
                     } break;
+                case PROJECTOR_TYPE_LFM3VL:
+                    {
+                        std::string resize_algo;
+                        get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo);
+                        // Lfm3VlImageProcessor resizes with torchvision
+                        if (resize_algo == "bilinear") {
+                            hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true);
+                        } else if (resize_algo == "bicubic") {
+                            hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true);
+                        } else {
+                            throw std::runtime_error(string_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str()));
+                        }
+                        // smart_resize stretches to the aligned size, no letterbox
+                        hparams.image_resize_pad = PAD_NONE;
+                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
+                        // token counts and the projector graph assume a 2x2 merge
+                        if (hparams.n_merge != 2) {
+                            throw std::runtime_error(string_format("%s: unsupported spatial_merge_size %d for lfm3vl, only 2 is supported\n", __func__, hparams.n_merge));
+                        }
+                        get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
+                        hparams.rope_theta = 1000000.0f; // vision_rope_theta
+                        // HF processor size; the max is also capped at 4096 tokens as an OOM guard
+                        int min_pixels = 0;
+                        int max_pixels = 0;
+                        get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels);
+                        get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels);
+                        if (min_pixels <= 0 || max_pixels < min_pixels) {
+                            throw std::runtime_error(string_format("%s: invalid image min/max pixels %d/%d\n", __func__, min_pixels, max_pixels));
+                        }
+                        const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
+                        hparams.set_limit_image_tokens(min_pixels / patch_area, std::min(4096, max_pixels / patch_area));
+                        hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
+                    } break;
                 case PROJECTOR_TYPE_MINIMAX_M3:
                     {
                         hparams.n_merge = 2; // spatial_merge_size
@@ -2515,6 +2552,9 @@ struct clip_model_loader {
 
         model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false);
 
+        // lfm3vl
+        model.lfm3vl_embedding_norm_w = get_tensor(string_format(TN_LFM3VL_EMBEDDING_NORM, prefix, "weight"), false);
+
         const bool has_standard_layers = (
             model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
             model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
@@ -2527,10 +2567,12 @@ struct clip_model_loader {
         model.layers.resize(n_layers_to_load);
         for (int il = 0; il < n_layers_to_load; ++il) {
             auto & layer = model.layers[il];
+            // lfm3vl conv layers have no attention tensors
+            const bool o_w_required = model.proj_type != PROJECTOR_TYPE_LFM3VL;
             layer.k_w    = get_tensor(string_format(TN_ATTN_K,      prefix, il, "weight"), false);
             layer.q_w    = get_tensor(string_format(TN_ATTN_Q,      prefix, il, "weight"), false);
             layer.v_w    = get_tensor(string_format(TN_ATTN_V,      prefix, il, "weight"), false);
-            layer.o_w    = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"));
+            layer.o_w    = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"), o_w_required);
             layer.qkv_w  = get_tensor(string_format(TN_ATTN_QKV,    prefix, il, "weight"), false);
             layer.k_norm = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "weight"), false);
             layer.q_norm = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false);
@@ -2580,6 +2622,11 @@ struct clip_model_loader {
                 model.n_deepstack_layers++;
             }
 
+            // lfm3vl conv layers
+            layer.conv2d_in_proj_w  = get_tensor(string_format(TN_LFM3VL_CONV2D_IN_PROJ,  prefix, il, "weight"), false);
+            layer.conv2d_spatial_w  = get_tensor(string_format(TN_LFM3VL_CONV2D_SPATIAL,  prefix, il, "weight"), false);
+            layer.conv2d_out_proj_w = get_tensor(string_format(TN_LFM3VL_CONV2D_OUT_PROJ, prefix, il, "weight"), false);
+
             // some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here
             // note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check!
             bool is_ffn_swapped = (
@@ -2764,6 +2811,24 @@ struct clip_model_loader {
                     model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
                     model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
                 } break;
+            case PROJECTOR_TYPE_LFM3VL:
+                {
+                    model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
+                    model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
+                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+                    // each layer must be either a conv block or a complete attention block
+                    for (size_t il = 0; il < model.layers.size(); ++il) {
+                        const auto & layer = model.layers[il];
+                        const bool is_conv = layer.conv2d_in_proj_w != nullptr;
+                        const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w;
+                        if (is_conv == is_attn) {
+                            throw std::runtime_error(string_format("%s: lfm3vl layer %zu is neither a complete conv nor a complete attention block\n", __func__, il));
+                        }
+                    }
+                } break;
             case PROJECTOR_TYPE_LING3VL:
                 {
                     model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);        // merger.norm
@@ -4452,6 +4517,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_HUNYUANVL:
         case PROJECTOR_TYPE_YOUTUVL:
         case PROJECTOR_TYPE_MUSE_GLIMMER:
+        case PROJECTOR_TYPE_LFM3VL:
             return (img->nx() / params.patch_size) / 2;
         case PROJECTOR_TYPE_STEP3VL:
         case PROJECTOR_TYPE_MINICPMV4_6:
@@ -4484,6 +4550,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_HUNYUANVL:
         case PROJECTOR_TYPE_YOUTUVL:
         case PROJECTOR_TYPE_MUSE_GLIMMER:
+        case PROJECTOR_TYPE_LFM3VL:
             return (img->ny() / params.patch_size) / 2;
         case PROJECTOR_TYPE_STEP3VL:
         case PROJECTOR_TYPE_MINICPMV4_6:
@@ -4571,6 +4638,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_GLM5V:
         case PROJECTOR_TYPE_YOUTUVL:
         case PROJECTOR_TYPE_MUSE_GLIMMER:
+        case PROJECTOR_TYPE_LFM3VL:
             {
                 // dynamic size (2 conv, so double patch size)
                 int x_patch = img->nx() / (params.patch_size * 2);
@@ -5377,6 +5445,24 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
                 set_input_i32("merger_ds_idx_2", m_ds_2);
                 set_input_i32("merger_ds_idx_3", m_ds_3);
             } break;
+        case PROJECTOR_TYPE_LFM3VL:
+            {
+                // patches in raster-scan order (no interleave before encoder layers)
+                const int pw = image_size_width  / patch_size;
+                const int ph = image_size_height / patch_size;
+                std::vector<int> positions(n_pos * 4);
+                int ptr = 0;
+                for (int y = 0; y < ph; y++) {
+                    for (int x = 0; x < pw; x++) {
+                        positions[                  ptr] = y;
+                        positions[    num_patches + ptr] = x;
+                        positions[2 * num_patches + ptr] = y;
+                        positions[3 * num_patches + ptr] = x;
+                        ptr++;
+                    }
+                }
+                set_input_i32("positions", positions);
+            } break;
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN3VL:
         case PROJECTOR_TYPE_LING3VL:
@@ -6603,6 +6689,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
             return ctx->model.mm_2_w->ne[1];
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
+        case PROJECTOR_TYPE_LFM3VL:
         case PROJECTOR_TYPE_EXAONE4_5:
         case PROJECTOR_TYPE_JANUS_PRO:
         case PROJECTOR_TYPE_YOUTUVL:

tools/mtmd/models/lfm3vl.cpp

@@ -0,0 +1,182 @@
+#include "models.h"
+
+ggml_cgraph * clip_graph_lfm3vl::build() {
+    GGML_ASSERT(model.patch_embeddings_0 != nullptr);
+    GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr);
+    GGML_ASSERT(model.class_embedding == nullptr);
+    GGML_ASSERT(n_batch == 1);
+
+    const int batch_size       = 1;
+    const int n_pos            = n_patches;
+    const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
+
+    const norm_type norm_t = NORM_TYPE_RMS;
+
+    // 2D rope: pairs (j, j+n_dims); first d_head/4 pairs rotate by y, next d_head/4 by x
+    int mrope_sections[4] = {d_head/4, d_head/4, 0, 0};
+
+    ggml_tensor * inp_raw = build_inp_raw();
+
+    // one patchifier kernel per temporal slice: a still image fills both slices, so both kernels apply to it
+    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+    if (model.patch_embeddings_1) {
+        inp = ggml_add(ctx0, inp, ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+    }
+    inp = ggml_reshape_2d(ctx0, inp, n_pos, n_embd);
+    inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
+    cb(inp, "patchifier_out", -1);
+
+    // the 2x2 interleave waits for the projector, the conv layers need raster-scan order
+    inp = build_norm(inp, model.lfm3vl_embedding_norm_w, nullptr, norm_t, eps, -1);
+    cb(inp, "embedding_norm", -1);
+
+    ggml_tensor * inpL = inp;
+
+    // position IDs for 2D RoPE (same format as Qwen3VL)
+    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
+    ggml_set_name(positions, "positions");
+    ggml_set_input(positions);
+
+    const float rope_theta = hparams.rope_theta;
+
+    for (int il = 0; il < n_layer; il++) {
+        auto & layer = model.layers[il];
+
+        const bool is_conv_layer = (layer.conv2d_in_proj_w != nullptr);
+
+        ggml_tensor * cur = inpL;
+
+        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
+        cb(cur, "ln1", il);
+
+        if (is_conv_layer) {
+            ggml_tensor * proj = build_mm(layer.conv2d_in_proj_w, cur);
+            cb(proj, "conv_in_proj", il);
+
+            ggml_tensor * first  = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+                proj->nb[1], 0);
+            ggml_tensor * middle = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+                proj->nb[1], ggml_row_size(proj->type, n_embd));
+            ggml_tensor * third  = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+                proj->nb[1], ggml_row_size(proj->type, 2 * n_embd));
+
+            ggml_tensor * gated = ggml_mul(ctx0, first, third);
+            cb(gated, "conv_gated", il);
+
+            ggml_tensor * conv_out;
+            if (layer.conv2d_spatial_w) {
+                ggml_tensor * gated_3d = ggml_reshape_4d(ctx0, gated,
+                    n_embd, n_patches_x, n_patches_y, batch_size);
+                ggml_tensor * gated_2d = ggml_cont(ctx0,
+                    ggml_permute(ctx0, gated_3d, 2, 0, 1, 3));
+                cb(gated_2d, "conv_gated_2d", il);
+
+                conv_out = ggml_conv_2d_dw_direct(ctx0, layer.conv2d_spatial_w, gated_2d,
+                    1, 1, 1, 1, 1, 1);
+                cb(conv_out, "conv_spatial", il);
+
+                conv_out = ggml_cont(ctx0,
+                    ggml_permute(ctx0, conv_out, 1, 2, 0, 3));
+                conv_out = ggml_reshape_2d(ctx0, conv_out, n_embd, n_pos);
+            } else {
+                conv_out = gated;
+            }
+
+            cur = ggml_mul(ctx0, middle, conv_out);
+            cb(cur, "conv_mul", il);
+
+            cur = build_mm(layer.conv2d_out_proj_w, cur);
+            cb(cur, "conv_out_proj", il);
+        } else {
+            ggml_tensor * Qcur = build_mm(layer.q_w, cur);
+            ggml_tensor * Kcur = build_mm(layer.k_w, cur);
+            ggml_tensor * Vcur = build_mm(layer.v_w, cur);
+
+            cb(Qcur, "Qcur", il);
+            cb(Kcur, "Kcur", il);
+            cb(Vcur, "Vcur", il);
+
+            Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head,    n_pos);
+            Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head_kv, n_pos);
+            Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head_kv, n_pos);
+
+            if (layer.q_norm) {
+                Qcur = build_norm(Qcur, layer.q_norm, nullptr, norm_t, eps, il);
+            }
+            if (layer.k_norm) {
+                Kcur = build_norm(Kcur, layer.k_norm, nullptr, norm_t, eps, il);
+            }
+
+            Qcur = ggml_rope_multi(
+                ctx0, Qcur, positions, nullptr,
+                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
+                32768, rope_theta, 1, 0, 1, 32, 1);
+            Kcur = ggml_rope_multi(
+                ctx0, Kcur, positions, nullptr,
+                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
+                32768, rope_theta, 1, 0, 1, 32, 1);
+
+            cb(Qcur, "Qcur_rope", il);
+            cb(Kcur, "Kcur_rope", il);
+
+            cur = build_attn(layer.o_w, layer.o_b,
+                Qcur, Kcur, Vcur, nullptr, kq_scale, il);
+            cb(cur, "attn_out", il);
+        }
+
+        cur = ggml_add(ctx0, cur, inpL);
+        inpL = cur;
+        cb(cur, "ffn_inp", il);
+
+        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
+        cb(cur, "ffn_inp_normed", il);
+
+        cur = build_ffn(cur,
+            layer.ff_up_w, layer.ff_up_b,
+            layer.ff_gate_w, layer.ff_gate_b,
+            layer.ff_down_w, layer.ff_down_b,
+            hparams.ffn_op, il);
+        cb(cur, "ffn_out", il);
+
+        cur = ggml_add(ctx0, inpL, cur);
+        cb(cur, "layer_out", il);
+
+        inpL = cur;
+    }
+
+    // no post-layernorm for LFM3-VL
+
+    ggml_tensor * embeddings = inpL;
+
+    // projector input LayerNorm, eps is projector_layer_norm_eps (not the encoder eps)
+    embeddings = build_norm(embeddings, model.mm_input_norm_w, model.mm_input_norm_b, NORM_TYPE_NORMAL, 1e-6f, -1);
+    cb(embeddings, "proj_input_norm", -1);
+
+    // 2x2 patch interleave (same as Qwen3VL) so the reshape below merges spatial 2x2 blocks
+    {
+        embeddings = ggml_cont_4d(
+            ctx0, embeddings,
+            n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
+        embeddings = ggml_reshape_4d(
+            ctx0, embeddings,
+            n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
+        embeddings = ggml_permute(ctx0, embeddings, 0, 2, 1, 3);
+        embeddings = ggml_cont_3d(
+            ctx0, embeddings,
+            n_embd, n_patches_x * n_patches_y, batch_size);
+    }
+    cb(embeddings, "proj_interleaved", -1);
+
+    embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
+
+    // projector MLP: linear_1 -> GELU (erf, as HF "gelu") -> linear_2
+    embeddings = build_ffn(embeddings,
+        model.mm_0_w, model.mm_0_b,
+        nullptr, nullptr,
+        model.mm_1_w, model.mm_1_b,
+        ffn_op_type::FFN_GELU_ERF, -1);
+
+    ggml_build_forward_expand(gf, embeddings);
+
+    return gf;
+}

tools/mtmd/models/models.h

@@ -54,6 +54,11 @@ struct clip_graph_sensenova_u1 : clip_graph {
     ggml_cgraph * build() override;
 };
 
+struct clip_graph_lfm3vl : clip_graph {
+    clip_graph_lfm3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+    ggml_cgraph * build() override;
+};
+
 struct clip_graph_ling3vl : clip_graph_qwen3vl {
     clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}
     ggml_cgraph * build() override;

tools/mtmd/mtmd-image.cpp

@@ -809,8 +809,7 @@ clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hpa
 
 mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
     clip_image_u8 resized_image;
-    const clip_image_size target_size = get_target_size(hparams, img.get_size());
-    img_tool::resize(img, resized_image, target_size,
+    img_tool::resize(img, resized_image, target_size(img.get_size()),
                         hparams.image_resize_algo,
                         hparams.image_resize_pad,
                         hparams.image_pad_color);
@@ -819,6 +818,34 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
     return output;
 }
 
+//
+// mtmd_image_preprocessor_lfm3vl
+//
+
+// rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does
+clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels) {
+    const double h = height;
+    const double w = width;
+    int h_bar = static_cast<int>(std::nearbyint(h / factor)) * factor;
+    int w_bar = static_cast<int>(std::nearbyint(w / factor)) * factor;
+    if (static_cast<int64_t>(h_bar) * w_bar > max_pixels) {
+        const double beta = std::sqrt(h * w / max_pixels);
+        h_bar = static_cast<int>(std::floor(h / beta / factor)) * factor;
+        w_bar = static_cast<int>(std::floor(w / beta / factor)) * factor;
+    } else if (static_cast<int64_t>(h_bar) * w_bar < min_pixels) {
+        const double beta = std::sqrt(min_pixels / (h * w));
+        h_bar = static_cast<int>(std::ceil(h * beta / factor)) * factor;
+        w_bar = static_cast<int>(std::ceil(w * beta / factor)) * factor;
+    }
+
+    return {std::max(factor, w_bar), std::max(factor, h_bar)};
+}
+
+clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const {
+    return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge,
+                        hparams.image_min_pixels, hparams.image_max_pixels);
+}
+
 //
 // mtmd_image_preprocessor_glm5v
 //

tools/mtmd/mtmd-image.h

@@ -135,6 +135,15 @@ struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
     mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
     static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
     mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+    virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); }
+};
+
+// LFM3-VL, same as dyn_size but sized exactly as the transformers Qwen2-VL smart_resize
+struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size {
+    mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {}
+    clip_image_size target_size(const clip_image_size & original_size) const override;
+
+    static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels);
 };
 
 // GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally

tools/mtmd/mtmd.cpp

@@ -772,6 +772,13 @@ struct mtmd_context {
                     img_end = "<|image_end|>";
                     image_preproc = std::make_unique<mtmd_image_preprocessor_muse_glimmer>(ctx_v);
                 } break;
+            case PROJECTOR_TYPE_LFM3VL:
+                {
+                    // <|vision_start|> ... (image embeddings) ... <|vision_end|>
+                    img_beg = "<|vision_start|>";
+                    img_end = "<|vision_end|>";
+                    image_preproc = std::make_unique<mtmd_image_preprocessor_lfm3vl>(ctx_v);
+                } break;
             case PROJECTOR_TYPE_YOUTUVL:
                 {
                     // <|vision_start|> ... (image embeddings) ... <|vision_end|>