From 6bbbf8d77df4cc2e6c816d46a5356bdfe3798ee3 Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Wed, 21 Oct 2026 14:13:11 +0200 Subject: [PATCH] handle bitnet scales too, disable in backends --- ggml/include/ggml.h | 2 +- ggml/src/ggml-blas/ggml-blas.cpp | 4 +++ ggml/src/ggml-cann/ggml-cann.cpp | 4 +++ ggml/src/ggml-cpu/ggml-cpu.cpp | 3 +++ ggml/src/ggml-et/ggml-et.cpp | 4 +++ ggml/src/ggml-hexagon/ggml-hexagon.cpp | 4 +++ ggml/src/ggml-impl.h | 10 +++++++ ggml/src/ggml-metal/ggml-metal.cpp | 4 +++ ggml/src/ggml-opencl/ggml-opencl.cpp | 4 +++ ggml/src/ggml-openvino/ggml-openvino.cpp | 4 +++ ggml/src/ggml-sycl/ggml-sycl.cpp | 4 +++ ggml/src/ggml-vulkan/ggml-vulkan.cpp | 4 +++ ggml/src/ggml-webgpu/ggml-webgpu.cpp | 4 +++ ggml/src/ggml-zdnn/ggml-zdnn.cpp | 4 +++ ggml/src/ggml-zendnn/ggml-zendnn.cpp | 4 +++ src/llama-graph.cpp | 34 +++--------------------- src/llama-model-loader.cpp | 2 +- 17 files changed, 67 insertions(+), 32 deletions(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index e54c17f9db..76991c494a 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -3037,7 +3037,7 @@ extern "C" { int64_t blck_size; int64_t blck_size_interleave; // interleave elements in blocks size_t type_size; - bool is_quantized; + bool is_quantized; // whether it's a block-quantized data-type bool needs_scale; // whether the quantization type needs a scale factor for valid dequantization ggml_to_float_t to_float; ggml_from_float_t from_float_ref; diff --git a/ggml/src/ggml-blas/ggml-blas.cpp b/ggml/src/ggml-blas/ggml-blas.cpp index 6feb370bb5..be2144d409 100644 --- a/ggml/src/ggml-blas/ggml-blas.cpp +++ b/ggml/src/ggml-blas/ggml-blas.cpp @@ -397,6 +397,10 @@ static bool ggml_backend_blas_device_supports_op(ggml_backend_dev_t dev, const s const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * src1 = op->src[1]; + if (ggml_get_src_scale(op, 0)) { + return false; + } + switch (op->op) { case GGML_OP_NONE: case GGML_OP_RESHAPE: diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp index 2c8154300a..d5aad3f044 100644 --- a/ggml/src/ggml-cann/ggml-cann.cpp +++ b/ggml/src/ggml-cann/ggml-cann.cpp @@ -2405,6 +2405,10 @@ static enum ggml_status ggml_backend_cann_graph_compute(ggml_backend_t backend, * otherwise false. */ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + if (ggml_get_src_scale(op, 0)) { + return false; + } + switch (op->op) { case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 81ff5d79fa..a95c664f42 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -434,6 +434,9 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st for (int i = 0; i < 4; i++) { if (op->src[i] && op->src[i]->buffer && ggml_backend_cpu_is_extra_buffer_type(op->src[i]->buffer->buft)) { + if (ggml_get_src_scale(op, 0)) { + return false; + } auto * buf_extra = (ggml::cpu::extra_buffer_type *) op->src[i]->buffer->buft->context; return buf_extra->supports_op(dev, op); } diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cpp index 54a6d166dc..7746c37fbf 100644 --- a/ggml/src/ggml-et/ggml-et.cpp +++ b/ggml/src/ggml-et/ggml-et.cpp @@ -865,6 +865,10 @@ static bool et_ggml_is_row_contiguous(const ggml_tensor * t) { static bool ggml_backend_et_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { GGML_UNUSED(dev); + if (ggml_get_src_scale(op, 0)) { + return false; + } + bool supported = false; switch (op->op) { case GGML_OP_CUMSUM: diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index cf2e37aa6e..57099a3716 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -2994,6 +2994,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons return false; } + if (ggml_get_src_scale(op, 0)) { + return false; + } + bool supp = false; switch (op->op) { case GGML_OP_NONE: diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 4dc55e78e2..4226fef025 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -104,6 +104,16 @@ static inline bool ggml_impl_is_view(const struct ggml_tensor * t) { return t->view_src != NULL; } +// scale of src[i] set by ggml_mul_mat_ext / ggml_mul_mat_id_ext +static inline const struct ggml_tensor * ggml_get_src_scale(const struct ggml_tensor * op, int i) { + GGML_ASSERT(i == 0 || i == 1); + switch (op->op) { + case GGML_OP_MUL_MAT: return op->src[2 + i]; + case GGML_OP_MUL_MAT_ID: return op->src[3 + i]; + default: return NULL; + } +} + static inline float ggml_compute_softplus_f32(float input) { return (input > 20.0f) ? input : logf(1 + expf(input)); } diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp index d0771eb493..067c10ca8f 100644 --- a/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp @@ -756,6 +756,10 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen static bool ggml_backend_metal_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context; + if (ggml_get_src_scale(op, 0)) { + return false; + } + return ggml_metal_device_supports_op(ctx_dev, op); } diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 67248264e1..f9464dc882 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -9849,6 +9849,10 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return false; } + if (ggml_get_src_scale(op, 0)) { + return false; + } + switch (op->op) { case GGML_OP_NONE: return true; diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 22b2f45e24..9618fc1ff7 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1890,6 +1890,10 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg } static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + if (ggml_get_src_scale(op, 0)) { + return false; + } + auto res = ggml_backend_openvino_device_supports_op_impl(dev, op); if (!res.is_supported) { static const bool log_unsupported = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_UNSUPPORTED_OPS") != 0; diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 97ad492e59..48ff5434d2 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -7180,6 +7180,10 @@ static ggml_backend_buffer_t ggml_backend_sycl_device_buffer_from_host_ptr(ggml_ } static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + if (ggml_get_src_scale(op, 0)) { + return false; + } + ggml_backend_sycl_device_context *sycl_ctx = (ggml_backend_sycl_device_context *)dev->context; int device = sycl_ctx->device; diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index c0943689f3..1fbde587ec 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -15506,6 +15506,10 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context; const vk_device& device = ggml_vk_get_device(ctx->device); + if (ggml_get_src_scale(op, 0)) { + return false; + } + const bool uses_bda = (op->op == GGML_OP_IM2COL || op->op == GGML_OP_IM2COL_3D) && device->shader_int64 && device->buffer_device_address; diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 65d6d456a8..3656b82da4 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -4436,6 +4436,10 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * src1 = op->src[1]; ggml_tensor * src2 = op->src[2]; + if (ggml_get_src_scale(op, 0)) { + return false; + } + // on smaller devices (or CI), tensors may be larger than the max storage buffer size if (ggml_nbytes(op) > ctx->webgpu_global_ctx->capabilities.limits.maxStorageBufferBindingSize || (src0 != nullptr && diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cpp index 40c6516a82..59e3c3b1f9 100644 --- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp +++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp @@ -86,6 +86,10 @@ static enum ggml_status ggml_zdnn_graph_compute(ggml_backend_t backend, ggml_cgr } static bool ggml_zdnn_supports_op(const ggml_backend_zdnn_device_context * ctx_dev, const ggml_tensor * op) { + if (ggml_get_src_scale(op, 0)) { + return false; + } + switch (op->op) { case GGML_OP_NONE: case GGML_OP_RESHAPE: diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cpp index 357cfcd99a..412ec42646 100644 --- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp +++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp @@ -697,6 +697,10 @@ static bool ggml_zendnn_adaptive_fallback_enabled() { } static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + if (ggml_get_src_scale(op, 0)) { + return false; + } + switch (op->op) { case GGML_OP_NONE: case GGML_OP_RESHAPE: diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index d5d8c9177f..186d73a3c1 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1943,18 +1943,7 @@ ggml_tensor * llm_graph_context::build_ffn( llm_ffn_op_type type_op, llm_ffn_gate_type type_gate, int il) const { - // Low-precision weights (NVFP4, F8_E4M3) carry a per-tensor dequant scale that must be applied - // in dequant space. Route it into the matmul via the ext API so it lands before bias/LoRA. - // Architectural *_s (on full-precision weights) stays a post-matmul multiply. - auto is_scaled_low_precision = [](ggml_tensor * w) { - return w && (w->type == GGML_TYPE_NVFP4 || w->type == GGML_TYPE_F8_E4M3); - }; - - const bool up_derived = is_scaled_low_precision(up); - const bool gate_derived = is_scaled_low_precision(gate); - const bool down_derived = is_scaled_low_precision(down); - - ggml_tensor * tmp = up ? build_lora_mm(up, cur, up_derived ? up_s : nullptr) : cur; + ggml_tensor * tmp = up ? build_lora_mm(up, cur, up_s) : cur; cb(tmp, "ffn_up", il); if (up_b) { @@ -1962,21 +1951,16 @@ ggml_tensor * llm_graph_context::build_ffn( cb(tmp, "ffn_up_b", il); } - if (up_s && !up_derived) { - tmp = ggml_mul(ctx0, tmp, up_s); - cb(tmp, "ffn_up_s", il); - } - if (gate) { switch (type_gate) { case LLM_FFN_SEQ: { - cur = build_lora_mm(gate, tmp, gate_derived ? gate_s : nullptr); + cur = build_lora_mm(gate, tmp, gate_s); cb(cur, "ffn_gate", il); } break; case LLM_FFN_PAR: { - cur = build_lora_mm(gate, cur, gate_derived ? gate_s : nullptr); + cur = build_lora_mm(gate, cur, gate_s); cb(cur, "ffn_gate", il); } break; } @@ -1986,11 +1970,6 @@ ggml_tensor * llm_graph_context::build_ffn( cb(cur, "ffn_gate_b", il); } - if (gate_s && !gate_derived) { - cur = ggml_mul(ctx0, cur, gate_s); - cb(cur, "ffn_gate_s", il); - } - } else { cur = tmp; } @@ -2112,7 +2091,7 @@ ggml_tensor * llm_graph_context::build_ffn( } if (down) { - cur = build_lora_mm(down, cur, down_derived ? down_s : nullptr); + cur = build_lora_mm(down, cur, down_s); if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) { // GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators ggml_prec_set_acc(cur, GGML_PREC_F32); @@ -2127,11 +2106,6 @@ ggml_tensor * llm_graph_context::build_ffn( cur = ggml_add(ctx0, cur, down_b); } - if (down_s && !down_derived) { - cur = ggml_mul(ctx0, cur, down_s); - cb(cur, "ffn_down_s", il); - } - return cur; } diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 410e20d542..404b439d5f 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1227,7 +1227,7 @@ struct ggml_tensor * llama_model_loader::create_tensor( } const ggml_tensor * scale_meta = nullptr; - if ((op == GGML_OP_MUL_MAT || op == GGML_OP_MUL_MAT_ID) && ggml_needs_scale_quantized(t_meta->type)) { + if (op == GGML_OP_MUL_MAT || op == GGML_OP_MUL_MAT_ID) { const std::string scale_name = LLM_TN_IMPL(tn.arch, tn.tensor, "scale", tn.bid, tn.xid).str(); scale_meta = get_tensor_meta(scale_name.c_str()); }