From 1b4554eff124a8f0033bc09b3f708a5e51cbda15 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 27 Oct 2026 22:39:37 +0100 Subject: [PATCH] server: expose the reasoning effort levels of the chat template (#31271) * server: expose the reasoning effort levels of the chat template The capability probe renders a user turn for each candidate level (minimal, low, medium, high, xhigh, max) and for an unknown level. A level is honored when it renders without raising and differently from the unknown level, which shows the template fallback. Levels sharing a render are aliases and the highest one is kept. The caps serialize to JSON, and chat_template_caps in /props reports the list as reasoning_effort_levels, from lowest to highest. * jinja: keep the caps free of the json dependency The caps keep reporting their flags as a map, and the effort levels stay a plain member. common_chat_templates_get_caps() builds the JSON the server reports from both, so jinja depends on json only for the probe inputs. --- common/chat.cpp | 13 ++++---- common/chat.h | 2 +- common/jinja/caps.cpp | 54 +++++++++++++++++++++++++++++++++ common/jinja/caps.h | 4 +++ tests/test-chat.cpp | 22 ++++++++++++-- tools/server/README.md | 2 +- tools/server/server-common.cpp | 2 +- tools/server/server-context.cpp | 2 +- tools/server/server-context.h | 2 +- 9 files changed, 90 insertions(+), 13 deletions(-) diff --git a/common/chat.cpp b/common/chat.cpp index ad05717905..96019cc382 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1940,12 +1940,13 @@ const common_chat_msg & common_chat_session::finish(const common_chat_input & ch return result; } -std::map common_chat_templates_get_caps(const common_chat_templates * chat_templates) { +common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) { GGML_ASSERT(chat_templates != nullptr); GGML_ASSERT(chat_templates->template_default != nullptr); - if (chat_templates->template_tool_use != nullptr) { - // take the more expressive template when available - return chat_templates->template_tool_use->caps.to_map(); - } - return chat_templates->template_default->caps.to_map(); + // take the more expressive template when available + const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps + : chat_templates->template_default->caps; + common_json result = caps.to_map(); + result["reasoning_effort_levels"] = caps.reasoning_effort_levels; + return result; } diff --git a/common/chat.h b/common/chat.h index 86ecc1315c..ebfc80f512 100644 --- a/common/chat.h +++ b/common/chat.h @@ -472,7 +472,7 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector common_chat_templates_get_caps(const common_chat_templates * chat_templates); +common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates); common_chat_prompt common_chat_template_direct_apply( const common_chat_template & tmpl, diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index ac295abd3b..de7780bdf7 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -6,7 +6,9 @@ // we can remove it in the future when we figure out a better way to define inputs using jinja::value #include "json.h" +#include #include +#include #include #define FILENAME "jinja-caps" @@ -588,6 +590,58 @@ caps caps_get(jinja::program & prog) { } ); + if (result.supports_reasoning_effort) { + JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort levels"); + + static const std::vector candidates = { "minimal", "low", "medium", "high", "xhigh", "max" }; + + // renders a single user turn with the given effort, nothing when the template raises + const auto render = [&](const std::string & effort) { + std::optional rendered; + caps_try_execute( + prog, + [&]() { + // messages + return json::array({ + { + {"role", "user"}, + {"content", "User message"} + }, + }); + }, + [&](context & ctx) { + ctx.set_val("enable_thinking", mk_val(true)); + caps_apply_reasoning_effort(ctx, effort); + }, + nullptr, // tools_fn + [&](context &, bool success, value &, value &, const std::string & output) { + if (success) { + rendered = output; + } + } + ); + return rendered; + }; + + // an unknown level shows the render of the template fallback, a level that renders the same is not honored + // levels sharing a render are aliases and the highest one is kept + const auto unknown = render("REASONING_EFFORT_UNKNOWN"); + std::vector renders; + for (const auto & level : candidates) { + const auto rendered = render(level); + if (!rendered || rendered == unknown) { + continue; + } + const auto it = std::find(renders.begin(), renders.end(), *rendered); + if (it != renders.end()) { + result.reasoning_effort_levels[it - renders.begin()] = level; + } else { + renders.push_back(*rendered); + result.reasoning_effort_levels.push_back(level); + } + } + } + JJ_DEBUG("%s\n", result.to_string().c_str()); return result; diff --git a/common/jinja/caps.h b/common/jinja/caps.h index b81dd95f2e..108501e06b 100644 --- a/common/jinja/caps.h +++ b/common/jinja/caps.h @@ -4,6 +4,7 @@ #include #include +#include namespace jinja { @@ -19,6 +20,9 @@ struct caps { // supports reasoning effort levels bool supports_reasoning_effort = false; + // reasoning effort levels the template renders distinctly, from lowest to highest + std::vector reasoning_effort_levels; + // one of the 2 content capabilities must be true bool supports_string_content = true; bool supports_typed_content = false; diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 41a8d4cfed..38a3fbe2e2 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -7738,7 +7738,7 @@ static void test_template_generation_prompt() { preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "true"; auto preserve_reasoning_params = common_chat_templates_apply(tmpls.get(), preserve_reasoning_inputs); assert_contains(preserve_reasoning_params.prompt.str(), "<｜Assistant｜>Previous reasoningPrevious answer"); - assert_equals(true, common_chat_templates_get_caps(tmpls.get()).at("supports_preserve_reasoning")); + assert_equals(true, common_chat_templates_get_caps(tmpls.get()).at("supports_preserve_reasoning").get()); auto no_preserve_reasoning_inputs = default_history_inputs; no_preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "false"; @@ -8311,7 +8311,7 @@ static void test_reasoning_effort_caps() { auto assert_supports_effort = [](const std::string & path, bool expected) { auto tmpls = read_templates(path); - assert_equals(expected, common_chat_templates_get_caps(tmpls.get()).at("supports_reasoning_effort")); + assert_equals(expected, common_chat_templates_get_caps(tmpls.get()).at("supports_reasoning_effort").get()); }; assert_supports_effort("models/templates/deepseek-ai-DeepSeek-V4.jinja", true); @@ -8322,6 +8322,24 @@ static void test_reasoning_effort_caps() { assert_supports_effort("models/templates/Cohere2MoE.jinja", true); assert_supports_effort("models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja", false); assert_supports_effort("models/templates/Qwen-Qwen3-0.6B.jinja", false); + + auto assert_effort_levels = [](const std::string & path, const std::vector & expected) { + auto tmpls = read_templates(path); + assert_equals(json(expected).dump(), common_chat_templates_get_caps(tmpls.get()).at("reasoning_effort_levels").dump()); + }; + + const std::vector all_levels = { "minimal", "low", "medium", "high", "xhigh", "max" }; + + assert_effort_levels("models/templates/deepseek-ai-DeepSeek-V4.jinja", { "max" }); + assert_effort_levels("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja", { "high", "max" }); + assert_effort_levels("models/templates/IFM-K2-Horizon.jinja", { "low", "medium", "high" }); + assert_effort_levels("models/templates/tencent-Hy3.jinja", { "low", "high" }); + assert_effort_levels("models/templates/muse-spark.jinja", { "minimal", "low", "medium", "high" }); + assert_effort_levels("models/templates/upstage-Solar-Open-100B.jinja", { "low" }); + assert_effort_levels("models/templates/muse-glimmer.jinja", all_levels); + assert_effort_levels("models/templates/openai-gpt-oss-120b.jinja", all_levels); + assert_effort_levels("models/templates/Cohere2MoE.jinja", {}); + assert_effort_levels("models/templates/Qwen-Qwen3-0.6B.jinja", {}); } static void test_msg_diffs_compute() { diff --git a/tools/server/README.md b/tools/server/README.md index b6202bd36b..de01eaeabc 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1019,7 +1019,7 @@ By default, it is read-only. To make POST request to change global properties, y - `total_slots` - the total number of slots for process requests (defined by `--parallel` option) - `model_path` - the path to model file (same with `-m` argument) - `chat_template` - the model's original Jinja2 prompt template -- `chat_template_caps` - capabilities of the chat template (see `common/jinja/caps.h` for more info) +- `chat_template_caps` - capabilities of the chat template (see `common/jinja/caps.h` for more info), `reasoning_effort_levels` lists the `reasoning_effort` values the template renders distinctly, from lowest to highest - `modalities` - the list of supported modalities - `is_sleeping` - sleeping status, see [Sleeping on idle](#sleeping-on-idle) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index f0ebf513e8..ade5668bb4 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1442,7 +1442,7 @@ json oaicompat_chat_params_parse( inputs.json_schema = json_schema.is_null() ? "" : json_schema.dump(); inputs.grammar = grammar; inputs.use_jinja = opt.use_jinja; - inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", caps["supports_parallel_tool_calls"]); + inputs.parallel_tool_calls = json_value(body, "parallel_tool_calls", caps.at("supports_parallel_tool_calls").get()); inputs.add_generation_prompt = json_value(body, "add_generation_prompt", true); inputs.continue_final_message = body.contains("continue_final_message") ? common_chat_continuation_parse(body.at("continue_final_message")) : diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 0ec9b548fd..3f8645706e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1721,7 +1721,7 @@ private: { auto caps = common_chat_templates_get_caps(chat_params.tmpls.get()); auto it = params_base.default_template_kwargs.find("preserve_reasoning"); - bool supported = caps.at("supports_preserve_reasoning"); + bool supported = caps.at("supports_preserve_reasoning").get(); bool specified = params_base.preserve_reasoning_specified; // note: the kwarg is enabled by default if not specified explicitly, so check the value bool enabled = it != params_base.default_template_kwargs.end() && it->second == "true"; diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 614ffad99f..d283c91217 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -33,7 +33,7 @@ struct server_context_meta { // chat params server_chat_params & chat_params; - std::map chat_template_caps; + json chat_template_caps; // tokens std::string bos_token_str;