From 36140dbee26704d517889db16caf5318039beaeb Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Tue, 20 Oct 2026 16:05:30 +0200 Subject: [PATCH] mtmd : read LFM2 tiling params from GGUF metadata (#30840) * mtmd : read LFM2 tiling params from GGUF metadata Write min_tiles, max_tiles, tile_size and max_pixels_tolerance from processor_config.json when converting LFM2-VL mmproj, read them in clip.cpp with the previous constants as defaults. do_image_splitting=false maps to min_tiles=max_tiles=1 (no tiling). Rebased version of #25524, original implementation by Michael DM (@michael-dm). Co-authored-by: Michael DM * fix test --------- Co-authored-by: Michael DM --- conversion/lfm2.py | 10 ++++++++++ gguf-py/gguf/constants.py | 2 ++ gguf-py/gguf/gguf_writer.py | 6 ++++++ tests/test-mtmd-impl.cpp | 10 ++++++++++ tools/mtmd/clip-impl.h | 2 ++ tools/mtmd/clip-model.h | 3 ++- tools/mtmd/clip.cpp | 10 ++++++++++ tools/mtmd/mtmd-image.cpp | 9 +++++++-- tools/mtmd/mtmd-image.h | 7 +------ 9 files changed, 50 insertions(+), 9 deletions(-) diff --git a/conversion/lfm2.py b/conversion/lfm2.py index 113c911168..b1f0d1d325 100644 --- a/conversion/lfm2.py +++ b/conversion/lfm2.py @@ -374,6 +374,16 @@ class LFM2VLModel(MmprojModel): if resize_algo is None: raise ValueError(f"unsupported resample: {resample}") self.gguf_writer.add_vision_image_resize_algo(resize_algo) + # tiling params from processor_config.json (do_image_splitting=False -> single tile, i.e. no tiling) + preproc = self.preprocessor_config + if preproc.get("do_image_splitting", True): + self.gguf_writer.add_vision_preproc_min_tiles(preproc.get("min_tiles", 2)) + self.gguf_writer.add_vision_preproc_max_tiles(preproc.get("max_tiles", 10)) + else: + self.gguf_writer.add_vision_preproc_min_tiles(1) + self.gguf_writer.add_vision_preproc_max_tiles(1) + self.gguf_writer.add_vision_preproc_tile_size(preproc.get("tile_size", 512)) + self.gguf_writer.add_vision_preproc_max_pixels_tolerance(preproc.get("max_pixels_tolerance", 2.0)) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 8c19e22dab..b2d8d59e04 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -419,6 +419,8 @@ class Keys: PREPROC_MIN_TILES = "clip.vision.preproc_min_tiles" PREPROC_MAX_TILES = "clip.vision.preproc_max_tiles" PREPROC_IMAGE_SIZE = "clip.vision.preproc_image_size" + PREPROC_TILE_SIZE = "clip.vision.preproc_tile_size" + PREPROC_MAX_PIXELS_TOLERANCE = "clip.vision.preproc_max_pixels_tolerance" PATCH_SIZE = "clip.vision.patch_size" EMBEDDING_LENGTH = "clip.vision.embedding_length" FEED_FORWARD_LENGTH = "clip.vision.feed_forward_length" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 7becd74af9..2943e01869 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1470,6 +1470,12 @@ class GGUFWriter: def add_vision_preproc_image_size(self, value: int) -> None: self.add_uint32(Keys.ClipVision.PREPROC_IMAGE_SIZE, value) + def add_vision_preproc_tile_size(self, value: int) -> None: + self.add_uint32(Keys.ClipVision.PREPROC_TILE_SIZE, value) + + def add_vision_preproc_max_pixels_tolerance(self, value: float) -> None: + self.add_float32(Keys.ClipVision.PREPROC_MAX_PIXELS_TOLERANCE, value) + def add_vision_projector_query_side(self, value: int) -> None: self.add_uint32(Keys.ClipVision.Projector.QUERY_SIDE, value) diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 2d0f248741..1b7796d956 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -96,6 +96,10 @@ MAKE_TEST(test_image_preprocessor_lfm2) { hparams.patch_size = 16; hparams.n_merge = 2; hparams.set_limit_image_tokens(64, 256); + hparams.preproc_min_tiles = 2; + hparams.preproc_max_tiles = 10; + hparams.preproc_tile_size = 512; + hparams.preproc_max_pixels_tolerance = 2.0f; // { image size, expected tiling } const std::vector> cases = { @@ -116,6 +120,12 @@ MAKE_TEST(test_image_preprocessor_lfm2) { std::string(expected ? "tiled" : "single"), std::string(actual ? "tiled" : "single")); } + + // do_image_splitting=false + hparams.preproc_min_tiles = 1; + hparams.preproc_max_tiles = 1; + t.assert_equal("tiling for 1024x977 with max_tiles=1", std::string("single"), + std::string(mtmd_image_preprocessor_lfm2::should_tile(hparams, { 1024, 977 }) ? "tiled" : "single")); } // diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 10eea6f746..8c4ffc9bb2 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -56,6 +56,8 @@ #define KEY_PREPROC_MIN_TILES "clip.vision.preproc_min_tiles" #define KEY_PREPROC_MAX_TILES "clip.vision.preproc_max_tiles" #define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size" +#define KEY_PREPROC_TILE_SIZE "clip.vision.preproc_tile_size" +#define KEY_PREPROC_MAX_PIXELS_TOLERANCE "clip.vision.preproc_max_pixels_tolerance" #define KEY_PATCH_SIZE "clip.vision.patch_size" #define KEY_IMAGE_MEAN "clip.vision.image_mean" #define KEY_IMAGE_STD "clip.vision.image_std" diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index fefbd4f460..64e006e8d6 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -73,7 +73,8 @@ struct clip_hparams { int32_t preproc_min_tiles = 0; int32_t preproc_max_tiles = 0; int32_t max_slice_nums = 9; // llava-uhd slice cap; per-model, carried in the GGUF - int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr) + int32_t preproc_tile_size = 0; // local tile size (deepseek-ocr, lfm2) + float preproc_max_pixels_tolerance = 1.0f; resize_algo image_resize_algo_rf = RESIZE_ALGO_BICUBIC; resize_algo image_resize_algo_ov = RESIZE_ALGO_BICUBIC; pad_style image_pad_rf = PAD_CEIL; // padding style for the refined image (e.g. llava-1.6) diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index faeb377027..3a4dd682ab 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1598,7 +1598,17 @@ struct clip_model_loader { hparams.image_pad_rf = PAD_NONE; } get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + // defaults for GGUFs converted before tiling params were written // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json + hparams.preproc_min_tiles = 2; + hparams.preproc_max_tiles = 10; + hparams.preproc_tile_size = 512; + hparams.preproc_max_pixels_tolerance = 2.0f; + get_u32(KEY_PREPROC_MIN_TILES, hparams.preproc_min_tiles, false); + get_u32(KEY_PREPROC_MAX_TILES, hparams.preproc_max_tiles, false); + get_u32(KEY_PREPROC_TILE_SIZE, hparams.preproc_tile_size, false); + get_f32(KEY_PREPROC_MAX_PIXELS_TOLERANCE, hparams.preproc_max_pixels_tolerance, false); + GGML_ASSERT(hparams.preproc_min_tiles >= 1 && hparams.preproc_min_tiles <= hparams.preproc_max_tiles); hparams.set_limit_image_tokens(64, 256); } break; case PROJECTOR_TYPE_PHI4: diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index e56f33e77c..86feea6bd4 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -947,8 +947,9 @@ bool mtmd_image_preprocessor_lfm2::should_tile( const int h_bar = std::max(hparams.patch_size, round_by_factor(original_size.height)); const int w_bar = std::max(hparams.patch_size, round_by_factor(original_size.width)); - return static_cast(h_bar) * static_cast(w_bar) > - static_cast(hparams.image_max_pixels) * max_pixels_tolerance; + return hparams.preproc_max_tiles > 1 && + static_cast(h_bar) * static_cast(w_bar) > + static_cast(hparams.image_max_pixels) * hparams.preproc_max_pixels_tolerance; } mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) const { @@ -959,6 +960,7 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lf { align_size, hparams.image_min_pixels, hparams.image_max_pixels, 0 }); const bool needs_tiling = should_tile(hparams, original_size); + const int tile_size = hparams.preproc_tile_size; if (!needs_tiling) { inst.refined_size = clip_image_size{0, 0}; @@ -1009,6 +1011,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( best_ratio_diff = ratio_diff; best_ratio = ratio; } else if (ratio_diff == best_ratio_diff) { + const int tile_size = hparams.preproc_tile_size; const float target_area = static_cast(tile_size * tile_size * ratio.width * ratio.height); if (area > 0.5f * target_area) { best_ratio = ratio; @@ -1019,6 +1022,8 @@ clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio( } std::vector mtmd_image_preprocessor_lfm2::get_target_ratios() const { + const int min_tiles = hparams.preproc_min_tiles; + const int max_tiles = hparams.preproc_max_tiles; std::vector ratios; for (int n = min_tiles; n <= max_tiles; n++) { for (int w = 1; w <= n; w++) { diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 26193e4e4d..a9d16f1c9c 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -191,14 +191,9 @@ protected: }; // custom llava-uhd slicing logic for LFM2 +// tiling params (min/max tiles, tile size, pixels tolerance) come from hparams // ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { - // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json - static constexpr int min_tiles = 2; - static constexpr int max_tiles = 10; - static constexpr float max_pixels_tolerance = 2.0f; - static constexpr int tile_size = 512; - using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; slice_instructions get_slice_instructions(const clip_image_size & original_size) const override;