[Rust Frontend] Match reasoning delimiters by token identity (#62892)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: vllm-agent <claw@inferact.ai>

rust/src/chat/src/parser/reasoning/tests.rs

@@ -128,7 +128,8 @@ fn factory_distinguishes_glm_reasoning_framing() {
 
 #[test]
 fn step5_preserves_reasoning_and_content_whitespace_across_chunks() {
-    use vllm_tokenizer::{DecodedText, Tokenizer as _};
+    use vllm_tokenizer::Tokenizer as _;
+    use vllm_tokenizer::test_utils::decode_chunks;
 
     let tokenizer = Arc::new(
         TestTokenizer::new()
@@ -154,9 +155,12 @@ fn step5_preserves_reasoning_and_content_whitespace_across_chunks() {
             parser.initialize(&prompt).unwrap();
             let mut reasoning = String::new();
             let mut content = String::new();
+            let [head, rest] = decode_chunks(tokenizer.as_ref(), &[&wire[..split], &wire[split..]])
+                .try_into()
+                .unwrap();
             for delta in [
-                parser.push(DecodedText::unattributed(&wire[..split])).unwrap(),
-                parser.push(DecodedText::unattributed(&wire[split..])).unwrap(),
+                parser.push(head).unwrap(),
+                parser.push(rest).unwrap(),
                 parser.finish().unwrap(),
             ] {
                 if let Some(text) = delta.reasoning {

rust/src/chat/tests/chat.rs

@@ -30,6 +30,7 @@ use vllm_text::{
     DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, Prompt,
     TextBackend,
 };
+use vllm_tokenizer::Tokenizer as _;
 use vllm_tokenizer::test_utils::TestTokenizer;
 use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -113,6 +114,12 @@ fn bytes_to_token_ids(bytes: &[u8]) -> Vec<u32> {
     bytes.iter().map(|byte| u32::from(*byte)).collect()
 }
 
+/// Encode `text` with the fake chat tokenizer, so markers spelled in full are
+/// their dedicated tokens, as the model emits them.
+fn token_ids(text: &str) -> Vec<u32> {
+    fake_chat_tokenizer().encode(text, false).unwrap()
+}
+
 fn bytes_with_special_stop_token(bytes: &[u8]) -> Vec<u32> {
     let mut token_ids = bytes_to_token_ids(bytes);
     token_ids.push(SPECIAL_STOP_TOKEN_ID);
@@ -826,24 +833,14 @@ async fn chat_stream_separates_reasoning_blocks_automatically() {
                     push,
                     RequestBatchOutputs {
                         outputs: vec![
-                            request_output(
-                                "chat-reasoning",
-                                bytes_to_token_ids(b"<think>"),
-                                None,
-                                None,
-                            ),
+                            request_output("chat-reasoning", token_ids("<think>"), None, None),
                             request_output(
                                 "chat-reasoning",
                                 bytes_to_token_ids(b"reason "),
                                 None,
                                 None,
                             ),
-                            request_output(
-                                "chat-reasoning",
-                                bytes_to_token_ids(b"more</think>"),
-                                None,
-                                None,
-                            ),
+                            request_output("chat-reasoning", token_ids("more</think>"), None, None),
                             request_output(
                                 "chat-reasoning",
                                 bytes_to_token_ids(b"answer"),
@@ -977,7 +974,7 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
                     RequestBatchOutputs {
                         outputs: vec![request_output(
                             "chat-collect",
-                            bytes_to_token_ids(b"<think>inner</think>outer"),
+                            token_ids("<think>inner</think>outer"),
                             Some(EngineCoreFinishReason::Length),
                             None,
                         )],
@@ -1016,7 +1013,7 @@ async fn chat_collectors_return_structured_message_and_visible_text() {
     );
     assert_eq!(
         message.usage.output_token_count,
-        "<think>inner</think>outer".len()
+        token_ids("<think>inner</think>outer").len()
     );
     // 5 tokens for "inner"; markers are excluded.
     assert_eq!(message.usage.reasoning_tokens, 5);
@@ -1044,7 +1041,7 @@ async fn chat_explicitly_disables_reasoning_parser() {
                         outputs: vec![
                             request_output(
                                 "chat-reasoning-disabled",
-                                bytes_to_token_ids(b"<think>"),
+                                token_ids("<think>"),
                                 None,
                                 None,
                             ),
@@ -1056,7 +1053,7 @@ async fn chat_explicitly_disables_reasoning_parser() {
                             ),
                             request_output(
                                 "chat-reasoning-disabled",
-                                bytes_to_token_ids(b"more</think>"),
+                                token_ids("more</think>"),
                                 None,
                                 None,
                             ),
@@ -1124,7 +1121,7 @@ async fn chat_stream_parses_tool_calls_automatically() {
                         outputs: vec![
                             request_output(
                                 "chat-tool",
-                                bytes_to_token_ids(b"<think>Need tool.</think>"),
+                                token_ids("<think>Need tool.</think>"),
                                 None,
                                 None,
                             ),
@@ -1233,7 +1230,7 @@ async fn chat_collect_message_preserves_tool_call_arguments_in_final_only_mode()
                         outputs: vec![
                             request_output(
                                 "chat-final-only-tool",
-                                bytes_to_token_ids(b"<think>Need tool.</think>"),
+                                token_ids("<think>Need tool.</think>"),
                                 None,
                                 None,
                             ),

rust/src/parser/src/reasoning/delimited.rs

@@ -2,10 +2,14 @@
 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer};
+use winnow::Parser;
+use winnow::combinator::alt;
+use winnow::error::ErrMode;
+use winnow::stream::StreamIsPartial;
 use xgrammar_structural_tag::format::Format;
 
 use super::{ReasoningDelta, ReasoningError, Result};
-use crate::utils::partial_prefix_len;
+use crate::utils::{Marker, SpecialToken, attributed_with_markers, safe_text_len_mul};
 
 /// Build a delimited reasoning parser with fixed text at each boundary.
 pub(crate) struct DelimitedReasoningParserBuilder {
@@ -73,12 +77,22 @@ impl DelimitedReasoningParserBuilder {
             }
         })?;
 
+        let delimiter = |token: String, id: u32, before: &str| {
+            let token = SpecialToken { text: token, id };
+            (
+                Marker::special(&token),
+                Marker::text(before).then_special(&token),
+            )
+        };
+        let (start, framed_start) = delimiter(self.start_token, start_token_id, self.before_start);
+        let (end, framed_end) = delimiter(self.end_token, end_token_id, self.before_end);
+
         Ok(DelimitedReasoningParser {
             tokenizer: self.tokenizer,
-            framed_start_token: format!("{}{}", self.before_start, self.start_token),
-            start_token: self.start_token,
-            framed_end_token: format!("{}{}", self.before_end, self.end_token),
-            end_token: self.end_token,
+            start,
+            framed_start,
+            end,
+            framed_end,
             start_token_id,
             end_token_id,
             after_start: self.after_start,
@@ -101,16 +115,21 @@ impl DelimitedReasoningParserBuilder {
 /// hardcoding model-family conventions. That means families with the same
 /// delimiters can often reuse this implementation even if their chat templates
 /// prefill different prompts.
+///
+/// A delimiter counts only where its dedicated token produced it: the same
+/// bytes spelled by ordinary tokens (the model quoting `</think>`) stay body
+/// text, as in the Python parsers' streaming path. The parser therefore needs
+/// attributed input; fed unattributed text, it never crosses a delimiter.
 pub(crate) struct DelimitedReasoningParser {
     tokenizer: DynTokenizer,
-    start_token: String,
-    framed_start_token: String,
-    end_token: String,
+    start: Marker,
+    framed_start: Marker,
+    end: Marker,
+    framed_end: Marker,
     start_token_id: u32,
     end_token_id: u32,
     after_start: &'static str,
     after_end: &'static str,
-    framed_end_token: String,
 
     // Mutable state.
     current_in_reasoning: bool,
@@ -149,14 +168,19 @@ impl DelimitedReasoningParser {
         self.current_in_reasoning
     }
 
+    /// The end delimiter, guarded by its token.
+    pub(crate) fn end_marker(&self) -> &Marker {
+        &self.end
+    }
+
     /// Wrap visible output from the initialized reasoning and framing state.
     pub(crate) fn wrap_visible_format(&self, visible: &Format) -> Format {
         let begin = if self.current_in_reasoning {
             "".to_string()
         } else {
-            self.framed_start_token.clone()
+            self.framed_start.as_str().to_string()
         };
-        let mut reasoning = Format::tag(begin, Format::any_text(), self.end_token.clone());
+        let mut reasoning = Format::tag(begin, Format::any_text(), self.end.as_str());
         if !self.after_end.is_empty() {
             reasoning = Format::sequence(vec![reasoning, Format::const_string(self.after_end)]);
         }
@@ -202,39 +226,41 @@ impl DelimitedReasoningParser {
             }
 
             let (marker, framed_marker) = if self.current_in_reasoning {
-                (&self.end_token, &self.framed_end_token)
+                (&self.end, &self.framed_end)
             } else {
-                (&self.start_token, &self.framed_start_token)
+                (&self.start, &self.framed_start)
             };
-            if let Some(index) = self.buffer.text.find(marker) {
-                let before = &framed_marker[..framed_marker.len() - marker.len()];
-                let body_len = if self.buffer.text[..index].ends_with(before) {
-                    index - before.len()
-                } else {
-                    index
-                };
-                let boundary_len = index + marker.len() - body_len;
-                let body = self.buffer.drain_prefix(body_len);
-                self.push_body(&mut delta, body);
-                let _ = self.buffer.drain_prefix(boundary_len);
-                self.current_in_reasoning = !self.current_in_reasoning;
-                self.pending_after = if self.current_in_reasoning {
-                    self.after_start
-                } else {
-                    self.after_end
-                };
-                continue;
+            let mut input = attributed_with_markers(&self.buffer, &());
+            if finishing {
+                // Partial delimiters at the end of the stream are body text.
+                let _ = input.complete();
             }
-
-            let keep_len = if finishing {
-                0
-            } else {
-                partial_prefix_len(&self.buffer.text, marker)
-                    .max(partial_prefix_len(&self.buffer.text, framed_marker))
+            let boundary = alt((framed_marker, marker)).map(str::len).parse_next(&mut input);
+            let body_len = match boundary {
+                Ok(boundary_len) => {
+                    // Zero-width tokens before the delimiter stay in the current state.
+                    let body = self.buffer.drain_prefix(0);
+                    self.push_body(&mut delta, body);
+                    let _ = self.buffer.drain_prefix(boundary_len);
+                    self.current_in_reasoning = !self.current_in_reasoning;
+                    self.pending_after = if self.current_in_reasoning {
+                        self.after_start
+                    } else {
+                        self.after_end
+                    };
+                    continue;
+                }
+                // A delimiter may start here once more input arrives.
+                Err(ErrMode::Incomplete(_)) => None,
+                // Body text runs up to the next delimiter that may start; the
+                // rest is held back. Fails only on an empty input.
+                Err(_) => safe_text_len_mul(&mut input, &[framed_marker, marker]).ok(),
             };
-            let body = self.buffer.drain_prefix(self.buffer.text.len() - keep_len);
+            let body = self.buffer.drain_prefix(body_len.unwrap_or(0));
             self.push_body(&mut delta, body);
-            break;
+            if body_len.is_none() {
+                break;
+            }
         }
         delta
     }

rust/src/parser/src/reasoning/minimax_m3.rs

@@ -2,11 +2,13 @@
 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 use vllm_tokenizer::{DecodedText, DynTokenizer};
+use winnow::error::ErrMode;
 
 use super::{
     DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
     Result,
 };
+use crate::utils::{MarkerLike as _, attributed_with_markers};
 
 const M3_THINK_START: &str = "<mm:think>";
 const M3_THINK_END: &str = "</mm:think>";
@@ -37,26 +39,36 @@ impl MiniMaxM3ReasoningParser {
         })
     }
 
-    /// Drop a response-leading `</mm:think>` while preserving later unmatched
-    /// closers as ordinary content.
+    /// Drop a response-leading dedicated `</mm:think>` while preserving later
+    /// unmatched closers as ordinary content.
     fn push_inner(&mut self, delta: DecodedText) -> ReasoningDelta {
         if self.at_response_start && !self.inner.in_reasoning() {
             self.leading_end_buffer.append(delta);
             let mut buffered = self.leading_end_buffer.take();
 
-            if buffered.text.starts_with(M3_THINK_END) {
-                self.at_response_start = false;
-                // The dropped marker span keeps its tokens out of any count.
-                let _ = buffered.drain_prefix(M3_THINK_END.len());
-                return self.inner.push(buffered);
+            let leading_end = self
+                .inner
+                .end_marker()
+                .as_marker()
+                .parse_next(&mut attributed_with_markers(&buffered, &()))
+                .map(str::len);
+            match leading_end {
+                Ok(end_len) => {
+                    self.at_response_start = false;
+                    // The dropped marker span keeps its tokens out of any count.
+                    let _ = buffered.drain_prefix(end_len);
+                    return self.inner.push(buffered);
+                }
+                // The dedicated closer may still be arriving.
+                Err(ErrMode::Incomplete(_)) => {
+                    self.leading_end_buffer = buffered;
+                    return ReasoningDelta::default();
+                }
+                Err(_) => {
+                    self.at_response_start = false;
+                    return self.inner.push(buffered);
+                }
             }
-            if M3_THINK_END.starts_with(buffered.text.as_str()) {
-                self.leading_end_buffer = buffered;
-                return ReasoningDelta::default();
-            }
-
-            self.at_response_start = false;
-            return self.inner.push(buffered);
         }
 
         self.inner.push(delta)
@@ -107,10 +119,14 @@ impl ReasoningParser for MiniMaxM3ReasoningParser {
 mod tests {
     use std::sync::Arc;
 
+    use vllm_tokenizer::Tokenizer as _;
+    use vllm_tokenizer::test_utils::decode_ids;
+
     use super::MiniMaxM3ReasoningParser;
     use crate::reasoning::ReasoningParser;
     use crate::reasoning::tests::{
-        MM_THINK_END_ID, MM_THINK_START_ID, content_str, fake_tokenizer, push_str, reasoning_str,
+        MM_THINK_END_ID, MM_THINK_START_ID, content_str, decode, fake_tokenizer, push_str,
+        reasoning_str,
     };
 
     #[test]
@@ -133,6 +149,21 @@ mod tests {
         assert_eq!(content_str(&delta), Some("answer"));
     }
 
+    #[test]
+    fn minimax_m3_keeps_ordinary_leading_end_marker() {
+        let tokenizer = fake_tokenizer();
+        let mut parser = MiniMaxM3ReasoningParser::new(Arc::new(fake_tokenizer())).unwrap();
+
+        // Ordinary bytes spelling `</mm:think>` are content, and a partial one
+        // is not held back.
+        let ordinary =
+            |text: &str| decode_ids(&tokenizer, &tokenizer.encode_ordinary(text).unwrap());
+        let head = parser.push(ordinary("</mm")).unwrap();
+        assert_eq!(content_str(&head), Some("</mm"));
+        let rest = parser.push(ordinary(":think>answer")).unwrap();
+        assert_eq!(content_str(&rest), Some(":think>answer"));
+    }
+
     #[test]
     fn minimax_m3_preserves_non_leading_end_marker() {
         let tokenizer = Arc::new(fake_tokenizer());
@@ -148,8 +179,9 @@ mod tests {
         let tokenizer = Arc::new(fake_tokenizer());
         let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
 
-        assert!(push_str(&mut parser, "</mm").is_empty());
-        let delta = push_str(&mut parser, ":think>answer");
+        let [head, rest] = decode(["</mm", ":think>answer"]);
+        assert!(parser.push(head).unwrap().is_empty());
+        let delta = parser.push(rest).unwrap();
         assert_eq!(delta.reasoning, None);
         assert_eq!(content_str(&delta), Some("answer"));
     }

rust/src/parser/src/reasoning/seed_oss.rs

@@ -53,7 +53,7 @@ mod tests {
     use super::SeedOssReasoningParser;
     use crate::reasoning::ReasoningParser;
     use crate::reasoning::tests::{
-        SEED_THINK_END_ID, SEED_THINK_START_ID, content_str, fake_tokenizer, push_str,
+        SEED_THINK_END_ID, SEED_THINK_START_ID, content_str, decode, fake_tokenizer, push_str,
         reasoning_str,
     };
 
@@ -142,11 +142,12 @@ mod tests {
         parser.initialize(&[SEED_THINK_START_ID]).unwrap();
 
         // Closing delimiter `</seed:think>` arrives in two halves.
-        let first = push_str(&mut parser, "reason</seed:");
+        let [first, second] = decode(["reason</seed:", "think>answer"]);
+        let first = parser.push(first).unwrap();
         assert_eq!(reasoning_str(&first), Some("reason"));
         assert_eq!(first.content, None);
 
-        let second = push_str(&mut parser, "think>answer");
+        let second = parser.push(second).unwrap();
         assert_eq!(second.reasoning, None);
         assert_eq!(content_str(&second), Some("answer"));
     }

rust/src/parser/src/reasoning/tests.rs

@@ -3,14 +3,14 @@
 
 use std::sync::Arc;
 
-use vllm_tokenizer::test_utils::TestTokenizer;
-use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution};
+use vllm_tokenizer::test_utils::{TestTokenizer, decode_chunks, decode_ids};
+use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution, Tokenizer as _};
 
 use super::{
-    CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DelimitedReasoningParser,
-    DelimitedReasoningParserBuilder, KimiReasoningParser, MiniMaxM3ReasoningParser,
-    Qwen3ReasoningParser, ReasoningDelta, ReasoningParser, Result, SeedOssReasoningParser,
-    Step3p5ReasoningParser,
+    CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
+    DelimitedReasoningParser, DelimitedReasoningParserBuilder, Glm45ReasoningParser,
+    KimiReasoningParser, MiniMaxM3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta,
+    ReasoningParser, Result, SeedOssReasoningParser, Step3p5ReasoningParser,
 };
 
 pub(crate) const THINK_START_ID: u32 = 256;
@@ -40,9 +40,17 @@ pub(crate) fn fake_tokenizer() -> TestTokenizer {
         .with_regular_token("</seed:think>", SEED_THINK_END_ID)
 }
 
-/// Feed an unattributed text delta into a reasoning parser.
+/// Decode `chunks` with [`fake_tokenizer`] as the frontend streams them:
+/// delimiters spelled in full are their dedicated tokens, and a delimiter split
+/// across chunks keeps its token anchor.
+pub(crate) fn decode<const N: usize>(chunks: [&str; N]) -> [DecodedText; N] {
+    decode_chunks(&fake_tokenizer(), &chunks).try_into().unwrap()
+}
+
+/// Feed one text delta into a reasoning parser as the frontend delivers it.
 pub(crate) fn push_str<P: ReasoningParser + ?Sized>(parser: &mut P, delta: &str) -> ReasoningDelta {
-    parser.push(DecodedText::unattributed(delta)).unwrap()
+    let [delta] = decode([delta]);
+    parser.push(delta).unwrap()
 }
 
 /// Return the reasoning text of a delta, if any.
@@ -62,7 +70,8 @@ fn delimited_content_only_stream() {
         .build()
         .unwrap();
 
-    let delta = parser.push(DecodedText::unattributed("plain content"));
+    let [delta] = decode(["plain content"]);
+    let delta = parser.push(delta);
     assert_eq!(content_str(&delta), Some("plain content"));
 }
 
@@ -73,7 +82,8 @@ fn delimited_single_chunk_with_reasoning_and_content() {
         .build()
         .unwrap();
 
-    let delta = parser.push(DecodedText::unattributed("<think>reason</think>answer"));
+    let [delta] = decode(["<think>reason</think>answer"]);
+    let delta = parser.push(delta);
     assert_eq!(reasoning_str(&delta), Some("reason"));
     assert_eq!(content_str(&delta), Some("answer"));
 }
@@ -85,8 +95,9 @@ fn delimited_partial_tokens_across_chunks() {
         .build()
         .unwrap();
 
-    assert!(parser.push(DecodedText::unattributed("<thi")).is_empty());
-    let delta = parser.push(DecodedText::unattributed("nk>reason</think>answer"));
+    let [head, rest] = decode(["<thi", "nk>reason</think>answer"]);
+    assert!(parser.push(head).is_empty());
+    let delta = parser.push(rest);
     assert_eq!(reasoning_str(&delta), Some("reason"));
     assert_eq!(content_str(&delta), Some("answer"));
 }
@@ -99,7 +110,9 @@ fn delimited_finish_flushes_buffer() {
         .unwrap();
     parser.initialize(&[THINK_START_ID]).unwrap();
 
-    let delta = parser.push(DecodedText::unattributed("unfinished</thi"));
+    // The stream ends inside a `</think>` token that output holdback split.
+    let [unfinished, _] = decode(["unfinished</thi", "nk>"]);
+    let delta = parser.push(unfinished);
     assert_eq!(reasoning_str(&delta), Some("unfinished"));
     let final_delta = parser.finish();
     assert_eq!(reasoning_str(&final_delta), Some("</thi"));
@@ -158,62 +171,93 @@ fn delimited_marker_tokens_are_dropped_from_attributions() {
 
     assert_eq!(collected.reasoning_text, "reason");
     assert_eq!(collected.content_text, "answer");
-    // The marker tokens (1 and 3) are dropped with their spans; only the
+    // The marker tokens (pieces 1 and 3) are dropped with their spans; only the
     // reasoning and content tokens remain attributed.
     assert_eq!(collected.reasoning_ids, [2]);
     assert_eq!(collected.content_ids, [4]);
 }
 
-#[test]
-fn reasoning_parsers_conserve_token_attributions() {
-    type ParserCase = (
-        fn(DynTokenizer) -> Result<Box<dyn ReasoningParser>>,
-        u32,
-        &'static str,
-    );
+/// One delimited reasoning family: name, constructor, start token ID, and end
+/// delimiter in [`fake_tokenizer`].
+type FamilyCase = (
+    &'static str,
+    fn(DynTokenizer) -> Result<Box<dyn ReasoningParser>>,
+    u32,
+    &'static str,
+);
 
-    let parsers: Vec<ParserCase> = vec![
-        (Qwen3ReasoningParser::create, THINK_START_ID, "qwen3"),
+/// The reasoning families built on [`DelimitedReasoningParser`]. DeepSeek V4,
+/// GLM-4.7, Kimi K2, and MiMo alias the DeepSeek V3 parser.
+fn delimited_families() -> Vec<FamilyCase> {
+    vec![
         (
+            "qwen3",
+            Qwen3ReasoningParser::create,
+            THINK_START_ID,
+            "</think>",
+        ),
+        (
+            "deepseek_r1",
             DeepSeekR1ReasoningParser::create,
             THINK_START_ID,
-            "deepseek_r1",
+            "</think>",
         ),
-        (KimiReasoningParser::create, MINIMAX_THINK_START_ID, "kimi"),
         (
+            "deepseek_v3",
+            DeepSeekV3ReasoningParser::create,
+            THINK_START_ID,
+            "</think>",
+        ),
+        (
+            "glm45",
+            Glm45ReasoningParser::create,
+            THINK_START_ID,
+            "</think>",
+        ),
+        (
+            "kimi",
+            KimiReasoningParser::create,
+            MINIMAX_THINK_START_ID,
+            "◁/think▷",
+        ),
+        (
+            "cohere_cmd",
             CohereCmdReasoningParser::create,
             START_THINKING_ID,
-            "cohere_cmd",
+            "<|END_THINKING|>",
         ),
         (
+            "seed_oss",
             SeedOssReasoningParser::create,
             SEED_THINK_START_ID,
-            "seed_oss",
+            "</seed:think>",
         ),
         (
+            "minimax_m3",
             MiniMaxM3ReasoningParser::create,
             MM_THINK_START_ID,
-            "minimax_m3",
+            "</mm:think>",
         ),
-        (Step3p5ReasoningParser::create, THINK_START_ID, "step3p5"),
-    ];
+        (
+            "step3p5",
+            Step3p5ReasoningParser::create,
+            THINK_START_ID,
+            "</think>",
+        ),
+    ]
+}
 
-    for (create, think_start_id, name) in parsers {
+#[test]
+fn reasoning_parsers_conserve_token_attributions() {
+    for (name, create, think_start_id, end_marker) in delimited_families() {
         let mut parser = create(Arc::new(fake_tokenizer())).unwrap();
         // Prompt prefills the start marker, opening reasoning before the stream.
         parser.initialize(&[think_start_id]).unwrap();
 
         // Script: reasoning text (1), a zero-width token inside reasoning (2),
-        // the end marker (3), then visible content (4). The zero-width token
+        // the end marker token, then visible content (4). The zero-width token
         // is attributed to reasoning, matching Python's current-state rule;
         // the marker token is dropped with its span.
-        let end_marker = match name {
-            "kimi" => "◁/think▷",
-            "cohere_cmd" => "<|END_THINKING|>",
-            "seed_oss" => "</seed:think>",
-            "minimax_m3" => "</mm:think>",
-            _ => "</think>",
-        };
         let chunks = attributed_chunks(&[Some("reason"), None, Some(end_marker), Some("answer")]);
 
         let collected = collect_attributed(parser.as_mut(), chunks);
@@ -225,17 +269,85 @@ fn reasoning_parsers_conserve_token_attributions() {
     }
 }
 
+#[test]
+fn delimited_families_end_reasoning_only_at_the_dedicated_end_token() {
+    let tokenizer = fake_tokenizer();
+    let dedicated = |text: &str| tokenizer.encode(text, false).unwrap();
+    for (name, create, think_start_id, end_marker) in delimited_families() {
+        let mut parser = create(Arc::new(fake_tokenizer())).unwrap();
+        parser.initialize(&[think_start_id]).unwrap();
+
+        // The model quotes the end delimiter with ordinary tokens, then ends
+        // reasoning with the dedicated token.
+        let ids = [
+            dedicated("quote "),
+            tokenizer.encode_ordinary(end_marker).unwrap(),
+            dedicated(" then"),
+            dedicated(end_marker),
+            dedicated("answer"),
+        ]
+        .concat();
+        let collected = collect_attributed(parser.as_mut(), vec![decode_ids(&tokenizer, &ids)]);
+
+        assert_eq!(
+            collected.reasoning_text,
+            format!("quote {end_marker} then"),
+            "{name}"
+        );
+        assert_eq!(collected.content_text, "answer", "{name}");
+    }
+}
+
+#[test]
+fn delimited_ordinary_start_delimiter_stays_content() {
+    let tokenizer = fake_tokenizer();
+    let mut parser =
+        DelimitedReasoningParserBuilder::new(Arc::new(fake_tokenizer()), "<think>", "</think>")
+            .build()
+            .unwrap();
+    parser.initialize(&[THINK_END_ID]).unwrap();
+
+    let ids = [
+        tokenizer.encode_ordinary("<think>").unwrap(),
+        tokenizer.encode("x", false).unwrap(),
+    ]
+    .concat();
+    let delta = parser.push(decode_ids(&tokenizer, &ids));
+    assert_eq!(delta.reasoning, None);
+    assert_eq!(content_str(&delta), Some("<think>x"));
+}
+
+#[test]
+fn delimited_ordinary_partial_end_delimiter_streams_immediately() {
+    let tokenizer = fake_tokenizer();
+    let mut parser =
+        DelimitedReasoningParserBuilder::new(Arc::new(fake_tokenizer()), "<think>", "</think>")
+            .build()
+            .unwrap();
+    parser.initialize(&[THINK_START_ID]).unwrap();
+
+    // Not held back: ordinary bytes cannot grow into the dedicated token.
+    let delta = parser.push(decode_ids(
+        &tokenizer,
+        &tokenizer.encode_ordinary("reason</thi").unwrap(),
+    ));
+    assert_eq!(reasoning_str(&delta), Some("reason</thi"));
+}
+
 /// Build one chunk per piece, each carrying a single token attributed to it.
 ///
 /// A `Some(text)` piece produces one token visibly anchored at the piece start;
-/// a `None` piece produces one zero-width token. Token IDs are 1-based
-/// positions in the script.
+/// a `None` piece produces one zero-width token. A piece spelling a delimiter
+/// of [`fake_tokenizer`] carries that delimiter's token ID; every other token
+/// ID is the 1-based position in the script.
 pub(crate) fn attributed_chunks(pieces: &[Option<&str>]) -> Vec<DecodedText> {
+    let tokenizer = fake_tokenizer();
     pieces
         .iter()
         .enumerate()
         .map(|(index, piece)| {
-            let token_id = index as u32 + 1;
+            let token_id =
+                piece.and_then(|text| tokenizer.token_to_id(text)).unwrap_or(index as u32 + 1);
             match piece {
                 Some(text) => DecodedText {
                     text: (*text).to_string(),
@@ -308,9 +420,10 @@ fn framed_parser() -> DelimitedReasoningParser {
 
 fn collect_framed(parser: &mut DelimitedReasoningParser, chunks: &[&str]) -> (String, String) {
     let mut result = (String::new(), String::new());
-    for chunk in chunks.iter().copied().map(Some).chain(std::iter::once(None)) {
+    let chunks = decode_chunks(&fake_tokenizer(), chunks);
+    for chunk in chunks.into_iter().map(Some).chain(std::iter::once(None)) {
         let delta = match chunk {
-            Some(chunk) => parser.push(DecodedText::unattributed(chunk)),
+            Some(chunk) => parser.push(chunk),
             None => parser.finish(),
         };
         if let Some(reasoning) = delta.reasoning {
@@ -403,12 +516,14 @@ fn delimited_prompt_initialization_consumes_only_remaining_framing() {
 #[test]
 fn delimited_framing_holds_only_boundary_candidates() {
     let mut parser = framed_parser();
-    let first = parser.push(DecodedText::unattributed("<think>\nreason\n"));
+    let [first, more, end, answer] =
+        decode(["<think>\nreason\n", "more\n", "</think>\n", "\n    answer"]);
+    let first = parser.push(first);
     assert_eq!(reasoning_str(&first), Some("reason"));
-    let more = parser.push(DecodedText::unattributed("more\n"));
+    let more = parser.push(more);
     assert_eq!(reasoning_str(&more), Some("\nmore"));
-    assert!(parser.push(DecodedText::unattributed("</think>\n")).is_empty());
-    let answer = parser.push(DecodedText::unattributed("\n    answer"));
+    assert!(parser.push(end).is_empty());
+    let answer = parser.push(answer);
     assert_eq!(content_str(&answer), Some("    answer"));
 }
 

rust/src/parser/src/unified/combined.rs

@@ -170,7 +170,7 @@ impl UnifiedParser for CombinedParser {
 mod tests {
     use std::sync::Arc;
 
-    use vllm_tokenizer::test_utils::TestTokenizer;
+    use vllm_tokenizer::test_utils::{TestTokenizer, decode_chunks};
     use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution, Tokenizer};
     use xgrammar_structural_tag::ToolChoice;
     use xgrammar_structural_tag::builders::ReasoningMode;
@@ -218,8 +218,8 @@ mod tests {
 
     fn collect(parser: &mut dyn UnifiedParser, chunks: &[&str]) -> UnifiedParserOutput {
         let mut output = UnifiedParserOutput::default();
-        for chunk in chunks {
-            parser.parse_into(DecodedText::unattributed(*chunk), &mut output).unwrap();
+        for chunk in decode_chunks(&tokenizer(), chunks) {
+            parser.parse_into(chunk, &mut output).unwrap();
         }
         output.append(parser.finish().unwrap());
         output
@@ -402,7 +402,7 @@ mod tests {
     #[test]
     fn combined_parser_emits_reasoning_and_text() {
         let tokenizer = Arc::new(tokenizer());
-        let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
+        let reasoning = Qwen3ReasoningParser::create(tokenizer.clone()).unwrap();
         let mut parser = CombinedParser::new(Some(reasoning), None);
 
         let output = collect(&mut parser, &["<think>work</think>answer"]);
@@ -410,7 +410,9 @@ mod tests {
         assert_eq!(
             output.events,
             vec![
-                UnifiedParserEvent::Reasoning(DecodedText::unattributed("work")),
+                UnifiedParserEvent::Reasoning(
+                    decode_chunks(tokenizer.as_ref(), &["work"]).remove(0)
+                ),
                 UnifiedParserEvent::Text("answer".to_string()),
             ]
         );
@@ -434,9 +436,9 @@ mod tests {
 
         let mut output = UnifiedParserOutput::default();
         for (token_id, text) in [
-            (1, "<think>"),
+            (256, "<think>"),
             (2, "reason"),
-            (3, "</think>"),
+            (257, "</think>"),
             (4, "answer"),
         ] {
             parser.parse_into(chunk(token_id, text), &mut output).unwrap();
@@ -444,7 +446,7 @@ mod tests {
         output.append(parser.finish().unwrap());
 
         // The reasoning tokens keep their attributions through the combined
-        // parser; marker tokens (1 and 3) are dropped with their spans.
+        // parser; marker tokens (256 and 257) are dropped with their spans.
         let reasoning_ids: Vec<u32> = output
             .events
             .iter()

rust/src/parser/src/unified/hy/hy_v3.rs

@@ -76,7 +76,8 @@ mod tests {
     use std::sync::Arc;
 
     use serde_json::json;
-    use vllm_tokenizer::{DecodedText, Tokenizer, test_utils::TestTokenizer};
+    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::{TestTokenizer, decode_chunks};
     use xgrammar_structural_tag::{StructuralTag, ToolChoice};
 
     use super::{HyV3UnifiedParser, UnifiedParser};
@@ -122,7 +123,7 @@ mod tests {
     fn parses_suffixed_reasoning_and_tool_call_as_one_stream() {
         let tokenizer = Arc::new(tokenizer());
         let think_start_id = tokenizer.token_to_id("<think:opensource>").unwrap();
-        let mut parser = HyV3UnifiedParser::new(&tools(), tokenizer).unwrap();
+        let mut parser = HyV3UnifiedParser::new(&tools(), tokenizer.clone()).unwrap();
         parser.initialize(&[think_start_id]).unwrap();
 
         let chunks = [
@@ -133,15 +134,17 @@ mod tests {
             "</tool_call:opensource></tool_calls:opensource>",
         ];
         let mut output = UnifiedParserOutput::default();
-        for chunk in chunks {
-            parser.parse_into(DecodedText::unattributed(chunk), &mut output).unwrap();
+        for chunk in decode_chunks(tokenizer.as_ref(), &chunks) {
+            parser.parse_into(chunk, &mut output).unwrap();
         }
         output.append(parser.finish().unwrap());
 
         assert_eq!(
             output.events,
             vec![
-                UnifiedParserEvent::Reasoning(DecodedText::unattributed("reasoning")),
+                UnifiedParserEvent::Reasoning(
+                    decode_chunks(tokenizer.as_ref(), &["reasoning"]).remove(0),
+                ),
                 UnifiedParserEvent::Text("answer".to_string()),
                 UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
                     tool_index: 0,

rust/src/parser/src/unified/hy/hy_v4.rs

@@ -76,7 +76,8 @@ mod tests {
     use std::sync::Arc;
 
     use serde_json::json;
-    use vllm_tokenizer::{DecodedText, Tokenizer, test_utils::TestTokenizer};
+    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::{TestTokenizer, decode_chunks};
     use xgrammar_structural_tag::{StructuralTag, ToolChoice};
 
     use super::{HyV4UnifiedParser, UnifiedParser};
@@ -130,7 +131,7 @@ mod tests {
     fn parses_suffixed_reasoning_and_compact_tool_calls_as_one_stream() {
         let tokenizer = Arc::new(tokenizer());
         let think_start_id = tokenizer.token_to_id("<think:opensource>").unwrap();
-        let mut parser = HyV4UnifiedParser::new(&tools(), tokenizer).unwrap();
+        let mut parser = HyV4UnifiedParser::new(&tools(), tokenizer.clone()).unwrap();
         parser.initialize(&[think_start_id]).unwrap();
 
         let chunks = [
@@ -142,15 +143,17 @@ mod tests {
             "</tool_call:opensource></tool_calls:opensource>",
         ];
         let mut output = UnifiedParserOutput::default();
-        for chunk in chunks {
-            parser.parse_into(DecodedText::unattributed(chunk), &mut output).unwrap();
+        for chunk in decode_chunks(tokenizer.as_ref(), &chunks) {
+            parser.parse_into(chunk, &mut output).unwrap();
         }
         output.append(parser.finish().unwrap());
 
         assert_eq!(
             output.events,
             vec![
-                UnifiedParserEvent::Reasoning(DecodedText::unattributed("reasoning")),
+                UnifiedParserEvent::Reasoning(
+                    decode_chunks(tokenizer.as_ref(), &["reasoning"]).remove(0),
+                ),
                 UnifiedParserEvent::Text("answer".to_string()),
                 UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
                     tool_index: 0,

rust/src/server/src/routes/tests.rs

@@ -50,6 +50,7 @@ use vllm_llm::Llm;
 use vllm_metrics::METRICS;
 use vllm_text::tokenizer::DynTokenizer;
 use vllm_text::{Prompt, TextBackend, TextRequestProcessor};
+use vllm_tokenizer::Tokenizer as _;
 use vllm_tokenizer::test_utils::TestTokenizer;
 use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -132,6 +133,12 @@ fn bytes_to_token_ids(bytes: &[u8]) -> Vec<u32> {
     bytes.iter().map(|byte| u32::from(*byte)).collect()
 }
 
+/// Encode `text` with the fake chat tokenizer, so markers spelled in full are
+/// their dedicated tokens, as the model emits them.
+fn token_ids(text: &str) -> Vec<u32> {
+    fake_chat_tokenizer().encode(text, false).unwrap()
+}
+
 fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
     vec![
         (vec![b'h' as u32], None),
@@ -142,7 +149,7 @@ fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason
 
 fn weather_tool_call_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
     vec![
-        (bytes_to_token_ids(b"<think>Need tool.</think>"), None),
+        (token_ids("<think>Need tool.</think>"), None),
         (
             bytes_to_token_ids(b"<tool_call>\n{\"name\":\"get_weather\", "),
             None,
@@ -5898,9 +5905,9 @@ async fn reasoning_blocks_are_mapped_to_reasoning_sse_chunks() {
     let (app, engine_task) = test_app_with_backend_and_stream_output_specs(
         Arc::new(FakeChatBackend::with_model_id("Qwen/Qwen3-0.6B")),
         vec![
-            (bytes_to_token_ids(b"<think>"), None),
+            (token_ids("<think>"), None),
             (bytes_to_token_ids(b"think "), None),
-            (bytes_to_token_ids(b"more</think>"), None),
+            (token_ids("more</think>"), None),
             (
                 bytes_to_token_ids(b"answer"),
                 Some(EngineCoreFinishReason::Length),
@@ -5945,7 +5952,7 @@ async fn include_reasoning_false_suppresses_reasoning_in_non_stream_chat() {
     let (app, engine_task) = test_app_with_backend_and_stream_output_specs(
         Arc::new(FakeChatBackend::with_model_id("Qwen/Qwen3-0.6B")),
         vec![
-            (bytes_to_token_ids(b"<think>think</think>"), None),
+            (token_ids("<think>think</think>"), None),
             (
                 bytes_to_token_ids(b"answer"),
                 Some(EngineCoreFinishReason::Length),
@@ -6005,7 +6012,7 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
                 let add = recv_engine_message(dealer).await;
                 let request: EngineCoreRequest =
                     rmp_serde::from_slice(&add[1]).expect("decode request");
-                let reasoning_token_ids = bytes_to_token_ids(b"<think>think</think>");
+                let reasoning_token_ids = token_ids("<think>think</think>");
                 let answer_token_ids = bytes_to_token_ids(b"answer");
 
                 send_outputs(
@@ -6230,11 +6237,11 @@ async fn tool_call_sse_chunks_can_carry_logprobs() {
                     RequestBatchOutputs {
                         outputs: vec![request_output_with_logprobs(
                             &request.request_id,
-                            bytes_to_token_ids(b"<think>Need tool.</think>"),
+                            token_ids("<think>Need tool.</think>"),
                             None,
                             None,
-                            Some(sample_logprobs_for_tokens(&bytes_to_token_ids(
-                                b"<think>Need tool.</think>",
+                            Some(sample_logprobs_for_tokens(&token_ids(
+                                "<think>Need tool.</think>",
                             ))),
                             None,
                         )],

rust/src/tokenizer/src/test_utils.rs

@@ -3,7 +3,7 @@
 
 use std::collections::BTreeMap;
 
-use crate::{Result, Tokenizer, TokenizerError};
+use crate::{DecodedText, Result, Tokenizer, TokenizerError};
 
 const FIRST_CONFIGURED_TOKEN_ID: u32 = 256;
 
@@ -281,6 +281,32 @@ impl Tokenizer for TestTokenizer {
     }
 }
 
+/// Decode the concatenated `chunks` as the frontend streams them, split back
+/// at the chunk boundaries.
+///
+/// The concatenation is encoded with `tokenizer`'s configured tokens (so a
+/// marker spelled in full is its dedicated token) and run through the
+/// incremental decoder once, so every chunk carries the attributions of the
+/// tokens that produced it, and a token split across chunks keeps its anchor in
+/// the chunk holding its first byte.
+pub fn decode_chunks(tokenizer: &dyn Tokenizer, chunks: &[&str]) -> Vec<DecodedText> {
+    let mut rest = decode_ids(
+        tokenizer,
+        &tokenizer.encode(&chunks.concat(), false).unwrap(),
+    );
+    chunks.iter().map(|chunk| rest.drain_prefix(chunk.len())).collect()
+}
+
+/// Decode generated `token_ids` through the incremental decoder, as the
+/// frontend does, keeping each token's attribution.
+pub fn decode_ids(tokenizer: &dyn Tokenizer, token_ids: &[u32]) -> DecodedText {
+    let mut decoder = tokenizer.create_decode_stream(&[], false, 0);
+    for &token_id in token_ids {
+        decoder.push_token(token_id).unwrap();
+    }
+    decoder.flush(None).unwrap().1
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;