From 2fb5111bbac7a2e31c0c19c1eb0aa01df5ecb664 Mon Sep 17 00:00:00 2001 From: Alec <35311602+alec-flowers@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:50:59 -0700 Subject: [PATCH] [Perf][Rust Frontend] Skip detokenization for gRPC requests that read no text (#63789) Signed-off-by: Alec Flowers Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Co-authored-by: Bugen Zhao Co-authored-by: vllm-agent --- rust/proto/inference.proto | 3 +- rust/src/server/src/grpc/convert.rs | 1 + rust/src/server/src/grpc/tests.rs | 162 ++++++++++++++-- .../routes/openai/chat_completions/convert.rs | 3 + .../src/routes/openai/completions/convert.rs | 1 + rust/src/text/src/error.rs | 6 + rust/src/text/src/output/decoded.rs | 174 +++++++++++++++--- rust/src/text/src/output/logprobs.rs | 43 +++-- rust/src/text/src/request.rs | 7 + rust/src/tokenizer/src/test_utils.rs | 42 ++++- 10 files changed, 387 insertions(+), 55 deletions(-) diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index 2daacbfc35b..47d5176e8dd 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -148,7 +148,8 @@ message ResponseOptions { bool prompt_logprobs = 2; optional CandidateTokens prompt_candidates = 3; - // Output options; output_text defaults to true + // Output options; output_text defaults to true. When false, the server does + // not detokenize, and stop_strings must be empty. optional bool output_text = 4; bool output_token_ids = 5; bool output_logprobs = 6; diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index cba6afe64ab..ce4f172f9ee 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -122,6 +122,7 @@ pub fn to_text_request( include_stop_str_in_output: stopping.is_some_and(|s| s.include_stop_strings), stop_strings: stopping.map(|s| &s.stop_strings).filter(|ss| !ss.is_empty()).cloned(), min_tokens: stopping.map_or(0, |s| s.min_new_tokens), + detokenize: response.and_then(|options| options.output_text).unwrap_or(true), }; Ok(TextRequest { diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 98d6579bce7..d1590c2129e 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -53,7 +53,7 @@ use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId use vllm_llm::Llm; use vllm_text::tokenizer::DynTokenizer; use vllm_text::{Prompt, TextBackend}; -use vllm_tokenizer::test_utils::TestTokenizer; +use vllm_tokenizer::test_utils::{PanickingTokenizer, TestTokenizer}; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; @@ -217,12 +217,22 @@ async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec { dealer.recv().await.expect("recv engine message").into_vec() } -#[derive(Clone, Debug)] -struct FakeTextBackend; +#[derive(Clone)] +struct FakeTextBackend { + tokenizer: DynTokenizer, +} + +impl Default for FakeTextBackend { + fn default() -> Self { + Self { + tokenizer: Arc::new(TestTokenizer::new()), + } + } +} impl TextBackend for FakeTextBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(TestTokenizer::new()) + Arc::clone(&self.tokenizer) } fn model_id(&self) -> &str { @@ -277,7 +287,7 @@ impl TextBackend for FakeMultimodalBackend { impl ChatBackend for FakeMultimodalBackend { fn chat_renderer(&self) -> DynChatRenderer { - Arc::new(FakeTextBackend) + Arc::new(FakeTextBackend::default()) } fn multimodal_model_info(&self) -> Option<&vllm_chat::multimodal::MultimodalModelInfo> { @@ -346,8 +356,13 @@ async fn setup_grpc_service( tokio::sync::watch::Receiver, MockEngineTask, ) { - setup_grpc_service_with_backend(engine_id, output_specs, Arc::new(FakeTextBackend), |_| {}) - .await + setup_grpc_service_with_backend( + engine_id, + output_specs, + Arc::new(FakeTextBackend::default()), + |_| {}, + ) + .await } async fn setup_grpc_service_with_backend( @@ -794,6 +809,90 @@ async fn unary_generate_with_token_ids_prompt() { server_task.abort(); } +/// `output_text=false` turns detokenization off, so neither RPC may decode. +/// Token IDs and finish info stay the same. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn generate_without_output_text_skips_detokenization() { + for streaming in [false, true] { + let (inference_service, control_service, engine_health, engine_task) = + setup_grpc_service_with_backend( + b"engine-grpc-token-only", + default_stream_output_specs(), + Arc::new(FakeTextBackend { + tokenizer: Arc::new(PanickingTokenizer), + }), + |_| {}, + ) + .await; + let (channel, server_task) = start_grpc_test_server( + inference_service, + control_service, + engine_health, + tokio_util::sync::CancellationToken::new(), + ) + .await; + let mut client = InferenceClient::new(channel); + let request = pb::GenerateRequest { + request_id: "test-token-only".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { + ids: vec![1, 2, 3], + })), + stopping: Some(pb::StoppingCriteria { + max_new_tokens: 10, + ..Default::default() + }), + response: Some(pb::ResponseOptions { + output_text: Some(false), + output_token_ids: true, + ..Default::default() + }), + ..Default::default() + }; + + let outputs: Vec<_> = if streaming { + let mut stream = + client.generate_stream(request).await.expect("start stream").into_inner(); + let mut outputs = Vec::new(); + while let Some(response) = stream.message().await.expect("stream message") { + outputs.extend(response.outputs); + } + outputs + } else { + client + .generate(request) + .await + .expect("unary generate") + .into_inner() + .outputs + .into_iter() + .collect() + }; + + assert!(outputs.iter().all(|output| output.text.is_empty())); + assert_eq!( + outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(), + [b'h' as u32, b'i' as u32, b'!' as u32] + ); + let finish = outputs + .last() + .and_then(|output| output.finish_info.as_ref()) + .expect("finish_info present"); + assert_eq!( + finish.finish_reason, + pb::finish_info::FinishReason::Stop as i32 + ); + assert_eq!(finish.num_output_tokens, 3); + + engine_task.await.expect("mock engine task"); + server_task.abort(); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn unary_generate_prepares_multimodal_input_for_engine_core() { @@ -1047,7 +1146,7 @@ async fn unary_generate_rejects_preprocessed_features_for_text_only_model() { setup_grpc_service_with_backend( b"engine-grpc-preprocessed-text-only", default_stream_output_specs(), - Arc::new(FakeTextBackend), + Arc::new(FakeTextBackend::default()), |_| {}, ) .await; @@ -1357,6 +1456,39 @@ async fn unary_generate_empty_stop_string_returns_invalid_argument() { server_task.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn unary_generate_stop_strings_without_output_text_returns_invalid_argument() { + let (mut client, server_task, _engine_task) = grpc_test_server( + b"engine-grpc-stop-without-text", + default_stream_output_specs(), + ) + .await; + + let status = client + .generate(pb::GenerateRequest { + request_id: "test-stop-without-text".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())), + stopping: Some(pb::StoppingCriteria { + stop_strings: vec!["\n".to_string()], + ..Default::default() + }), + response: Some(pb::ResponseOptions { + output_text: Some(false), + ..Default::default() + }), + ..Default::default() + }) + .await + .expect_err("stop strings need detokenized output"); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + assert!(status.message().contains("stop strings require detokenization")); + + server_task.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn unary_generate_invalid_sampling_params_returns_invalid_argument() { @@ -1886,7 +2018,7 @@ async fn dropping_generate_stream_aborts_before_engine_output() { setup_grpc_service_with_engine_script( b"engine-grpc-cancel-pending-output".to_vec(), default_ready_response(), - Arc::new(FakeTextBackend), + Arc::new(FakeTextBackend::default()), None, move |dealer, _push| { boxed_test_future(async move { @@ -2042,7 +2174,7 @@ async fn control_reports_server_and_model_info() { setup_grpc_service_with_engine_script( b"engine-grpc-info".to_vec(), ready, - Arc::new(FakeTextBackend), + Arc::new(FakeTextBackend::default()), None, |_, _| boxed_test_future(async {}), ) @@ -2111,7 +2243,7 @@ async fn control_lora_lifecycle_selects_adapter_for_generation() { setup_grpc_service_with_engine_script( b"engine-grpc-lora".to_vec(), ready, - Arc::new(FakeTextBackend), + Arc::new(FakeTextBackend::default()), None, |dealer, push| { boxed_test_future(async move { @@ -2236,7 +2368,7 @@ async fn control_list_loras_requires_lora_enabled_engine() { setup_grpc_service_with_engine_script( b"engine-grpc-lora-disabled".to_vec(), default_ready_response(), - Arc::new(FakeTextBackend), + Arc::new(FakeTextBackend::default()), None, |_, _| boxed_test_future(async move {}), ) @@ -2269,7 +2401,7 @@ async fn control_forwards_weight_update_without_pause_guard() { setup_grpc_service_with_engine_script( b"engine-grpc-rl".to_vec(), ready, - Arc::new(FakeTextBackend), + Arc::new(FakeTextBackend::default()), None, |dealer, push| { boxed_test_future(async move { @@ -2391,7 +2523,7 @@ async fn control_aggregates_multi_engine_capacity() { let client = client_task.await.unwrap().expect("connect multi-engine client"); let chat = ChatLlm::from_shared_backend( Llm::new(client), - Arc::new(FakeTextBackend) as Arc, + Arc::new(FakeTextBackend::default()) as Arc, ); let state = AppState::new(vec!["test-model".to_string()], chat); let service = ControlServiceImpl::new(Arc::new(state)); @@ -2566,7 +2698,7 @@ async fn grpc_shutdown_closes_health_watch_and_server() { setup_grpc_service_with_engine_script( b"engine-grpc-health-shutdown", default_ready_response(), - Arc::new(FakeTextBackend), + Arc::new(FakeTextBackend::default()), Some(shutdown.clone()), |_, _| boxed_test_future(async {}), ) diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 57c6b3acdff..da8b01ac2f3 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -203,6 +203,7 @@ pub(super) fn prepare_chat_request( include_stop_str_in_output: request.include_stop_str_in_output, stop_strings: request.stop.map(|stop| stop.into_vec()), min_tokens: request.min_tokens.unwrap_or(0), + detokenize: true, }, intermediate: request.stream, prompt_truncation, @@ -958,6 +959,7 @@ mod tests { include_stop_str_in_output: false, stop_strings: None, min_tokens: 0, + detokenize: true, } ); assert!(prepared.chat_request.initial_tools().is_empty()); @@ -1033,6 +1035,7 @@ mod tests { include_stop_str_in_output: false, stop_strings: None, min_tokens: 0, + detokenize: true, } ); assert!(prepared.chat_request.initial_tools().is_empty()); diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 75792bf1cdd..790f006f398 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -159,6 +159,7 @@ pub(super) fn prepare_completion_request( include_stop_str_in_output: request.include_stop_str_in_output, stop_strings: request.stop.map(|stop| stop.into_vec()), min_tokens: request.min_tokens.unwrap_or(0), + detokenize: true, }, intermediate: request.stream, prompt_truncation, diff --git a/rust/src/text/src/error.rs b/rust/src/text/src/error.rs index 9d27b497e8e..e6087cc9d04 100644 --- a/rust/src/text/src/error.rs +++ b/rust/src/text/src/error.rs @@ -17,6 +17,11 @@ pub enum Error { EmptyPromptTokenIds { request_id: String }, #[error("text request `{request_id}` stop strings cannot be empty")] EmptyStopString { request_id: String }, + #[error( + "text request `{request_id}` stop strings require detokenization; enable it or use stop \ + token IDs" + )] + StopStringsRequireDetokenize { request_id: String }, #[error( "Parameter 'cache_salt' must be a non-empty string of at most 128 characters \ and must not contain '@', '/', '\\', or NUL." @@ -76,6 +81,7 @@ impl Error { | Self::PromptFillsContext { .. } | Self::EmptyPromptTokenIds { .. } | Self::EmptyStopString { .. } + | Self::StopStringsRequireDetokenize { .. } | Self::InvalidCacheSalt | Self::Logprobs(_) | Self::TokenIds(_) diff --git a/rust/src/text/src/output/decoded.rs b/rust/src/text/src/output/decoded.rs index 6308bdf6e11..5074f35100c 100644 --- a/rust/src/text/src/output/decoded.rs +++ b/rust/src/text/src/output/decoded.rs @@ -28,6 +28,10 @@ pub struct TextDecodeOptions { /// in. Stop strings found within the first `min_tokens` tokens are /// ignored. pub min_tokens: u32, + /// Whether to detokenize the output. When false, the output is never + /// decoded: text and logprob token strings stay empty. + /// `TextRequest::validate` rejects stop strings in that case. + pub detokenize: bool, } impl Default for TextDecodeOptions { @@ -37,6 +41,24 @@ impl Default for TextDecodeOptions { include_stop_str_in_output: false, stop_strings: None, min_tokens: 0, + detokenize: true, + } + } +} + +impl TextDecodeOptions { + /// If we are excluding stop strings from output, we need to buffer + /// the output so that we don't return the beginning of a stop string + /// when streaming the outputs. + fn min_bytes_to_buffer(&self) -> usize { + match self.include_stop_str_in_output { + true => 0, + false => self + .stop_strings + .as_ref() + .and_then(|stops| stops.iter().map(|ss| ss.len()).max()) + .unwrap_or(1) + .saturating_sub(1), } } } @@ -109,6 +131,31 @@ pub enum DecodedTextEvent { }, } +/// Decoder for requests with detokenization disabled. Like Python's base +/// `IncrementalDetokenizer`, it never calls the tokenizer. +struct TokenOnlyDecoder; + +impl IncrementalDecoder for TokenOnlyDecoder { + fn push_token(&mut self, _token_id: u32) -> vllm_tokenizer::Result { + Ok(0) + } + + fn next_chunk(&mut self) -> Option { + None + } + + fn flush( + &mut self, + _truncate_output_to: Option, + ) -> vllm_tokenizer::Result<(Option, DecodedText)> { + Ok((None, DecodedText::default())) + } + + fn output(&self) -> &str { + "" + } +} + /// Convert the output token stream from the `vllm_llm` layer into incrementally /// decoded text. #[try_stream] @@ -127,6 +174,7 @@ pub async fn decoded_text_event_stream( let mut output_token_count: usize = 0; let mut sampled_logprobs: Option = None; let mut sampling_mask: Option = None; + let text_tokenizer = decode_options.detokenize.then_some(tokenizer.as_ref()); while let Some(next) = raw_stream.next().await { let output = next?; @@ -138,22 +186,14 @@ pub async fn decoded_text_event_stream( output.prompt_token_ids().expect("first llm output must carry prompt token ids"); prompt_token_count = prompt_token_ids.len(); - let dec = tokenizer.create_decode_stream( - prompt_token_ids, - decode_options.skip_special_tokens, - // If we are excluding stop strings from output, we need to buffer - // the output so that we don't return the beginning of a stop string - // when streaming the outputs. - match decode_options.include_stop_str_in_output { - true => 0, - false => decode_options - .stop_strings - .as_ref() - .and_then(|stops| stops.iter().map(|ss| ss.len()).max()) - .unwrap_or(1) - .saturating_sub(1), - }, - ); + let dec = match text_tokenizer { + Some(tokenizer) => tokenizer.create_decode_stream( + prompt_token_ids, + decode_options.skip_special_tokens, + decode_options.min_bytes_to_buffer(), + ), + None => Box::new(TokenOnlyDecoder), + }; decoder = Some(dec); y.yield_ok(DecodedTextEvent::Start { @@ -162,7 +202,7 @@ pub async fn decoded_text_event_stream( .prompt_logprobs() .map(|logprobs| { decode_prompt_logprobs( - tokenizer.as_ref(), + text_tokenizer, prompt_token_ids, logprobs, decode_options.skip_special_tokens, @@ -238,24 +278,20 @@ pub async fn decoded_text_event_stream( sampling_mask.get_or_insert_default().rows.append(&mut mask.rows); } - let decoded_logprobs = new_logprobs + let mut decoded_logprobs = new_logprobs .as_ref() .map(|logprobs| { - decode_logprobs( - tokenizer.as_ref(), - logprobs, - decode_options.skip_special_tokens, - ) + decode_logprobs(text_tokenizer, logprobs, decode_options.skip_special_tokens) }) .transpose()?; if !intermediate { sampled_token_ids.extend(&new_token_ids); - if let Some(dlp) = decoded_logprobs.as_ref() { + if let Some(dlp) = decoded_logprobs.take() { sampled_logprobs .get_or_insert_with(|| DecodedLogprobs { positions: vec![] }) .positions - .extend_from_slice(&dlp.positions); + .extend(dlp.positions); } } @@ -382,7 +418,8 @@ mod tests { use vllm_engine_core_client::AbortCause; use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob}; use vllm_llm::GenerateOutput; - use vllm_tokenizer::{TokenAnchor, TokenAttribution, test_utils::TestTokenizer}; + use vllm_tokenizer::test_utils::{PanickingTokenizer, TestTokenizer}; + use vllm_tokenizer::{TokenAnchor, TokenAttribution}; use super::*; use crate::output::TextOutputStreamExt as _; @@ -630,6 +667,91 @@ mod tests { } } + /// Without detokenization, a request gets token IDs, logprob values, usage + /// and finish info without any tokenizer call. + #[tokio::test] + async fn detokenize_false_keeps_sampled_metadata_without_tokenizer() { + let logprobs = |token_ids: &[u32]| Logprobs { + positions: token_ids + .iter() + .map(|&token_id| PositionLogprobs { + entries: vec![TokenLogprob { + token_id, + logprob: -0.5, + rank: 1, + }], + }) + .collect(), + }; + let prompt = ascii_tokens("hi"); + let mut output = GenerateOutput::for_test( + Some(Arc::from(prompt.clone())), + ascii_tokens("a!"), + Some(FinishReason::Stop(Some(StopReason::TokenId(b'!' as u32)))), + ); + output.prompt_info.as_mut().unwrap().prompt_logprobs = Some(logprobs(&prompt[1..])); + output.logprobs = Some(logprobs(&ascii_tokens("a!"))); + + let collected = decoded_text_event_stream( + "test".into(), + Arc::new(PanickingTokenizer), + stream::iter([Ok(output)]), + TextDecodeOptions { + detokenize: false, + ..Default::default() + }, + false, + ) + .collect_output() + .await + .unwrap(); + + let logprob_tokens: Vec<_> = collected + .prompt_logprobs + .iter() + .flat_map(|prompt| &prompt.scored_positions) + .chain(collected.logprobs.iter().flat_map(|logprobs| &logprobs.positions)) + .flat_map(|position| &position.entries) + .map(|entry| (entry.token_id, entry.token.as_str())) + .collect(); + expect_test::expect![[r#" + ( + "", + [ + 97, + 33, + ], + Stop( + Some( + TokenId( + 33, + ), + ), + ), + [ + ( + 105, + "", + ), + ( + 97, + "", + ), + ( + 33, + "", + ), + ], + ) + "#]] + .assert_debug_eq(&( + &collected.text, + &collected.token_ids, + &collected.finish_reason, + logprob_tokens, + )); + } + struct DropRecordingStream { next: Option>, dropped_cause: Arc>>, diff --git a/rust/src/text/src/output/logprobs.rs b/rust/src/text/src/output/logprobs.rs index 9979a618783..7867e33ab62 100644 --- a/rust/src/text/src/output/logprobs.rs +++ b/rust/src/text/src/output/logprobs.rs @@ -13,7 +13,8 @@ use crate::error::Error; pub struct DecodedTokenLogprob { /// Original vocabulary token ID for this candidate. pub token_id: u32, - /// Best-effort decoded token string for this candidate. + /// Best-effort decoded token string for this candidate; empty when the + /// request outputs no text. pub token: String, /// Log probability of this token candidate. pub logprob: f32, @@ -40,7 +41,8 @@ pub struct DecodedLogprobs { pub struct DecodedPromptLogprobs { /// Original vocabulary token ID for the first prompt token. pub first_token_id: u32, - /// Best-effort decoded string for the first prompt token. + /// Best-effort decoded string for the first prompt token; empty when the + /// request outputs no text. /// /// The first prompt token has no left context to score against, so it is /// stored separately instead of appearing in `scored_positions`. @@ -58,7 +60,7 @@ pub struct DecodedPromptLogprobs { /// Each returned position corresponds to one generated token position from the /// same `llm` update. pub(super) fn decode_logprobs( - tokenizer: &T, + tokenizer: Option<&T>, logprobs: &Logprobs, skip_special_tokens: bool, ) -> Result { @@ -78,7 +80,7 @@ pub(super) fn decode_logprobs( /// the remaining scored prompt positions into `scored_positions`, matching /// vLLM's prompt-logprobs semantics. pub(super) fn decode_prompt_logprobs( - tokenizer: &T, + tokenizer: Option<&T>, prompt_token_ids: &[u32], logprobs: &Logprobs, skip_special_tokens: bool, @@ -87,7 +89,7 @@ pub(super) fn decode_prompt_logprobs( .first() .copied() .expect("prompt logprobs require at least one prompt token"); - let first_token = tokenizer.decode(&[first_token_id], skip_special_tokens)?; + let first_token = decode_token(tokenizer, first_token_id, skip_special_tokens)?; let scored_positions = logprobs .positions .iter() @@ -104,10 +106,10 @@ pub(super) fn decode_prompt_logprobs( /// Decode one token position's raw candidate set into decoded token strings /// plus logprob metadata. /// -/// This decodes every candidate token ID independently through the active text -/// backend. +/// With a tokenizer, every candidate token ID is decoded independently through +/// the active text backend. fn decode_position_logprobs( - tokenizer: &T, + tokenizer: Option<&T>, position: &PositionLogprobs, skip_special_tokens: bool, ) -> Result { @@ -116,7 +118,7 @@ fn decode_position_logprobs( .entries .iter() .map(|entry| { - tokenizer.decode(&[entry.token_id], skip_special_tokens).map(|token| { + decode_token(tokenizer, entry.token_id, skip_special_tokens).map(|token| { DecodedTokenLogprob { token_id: entry.token_id, token, @@ -129,6 +131,18 @@ fn decode_position_logprobs( }) } +/// Without a tokenizer the request outputs no text, so the token string stays +/// empty. +fn decode_token( + tokenizer: Option<&T>, + token_id: u32, + skip_special_tokens: bool, +) -> vllm_tokenizer::Result { + tokenizer.map_or(Ok(String::new()), |tokenizer| { + tokenizer.decode(&[token_id], skip_special_tokens) + }) +} + #[cfg(test)] mod tests { use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob}; @@ -157,7 +171,7 @@ mod tests { }; assert_eq!( - decode_logprobs(&tokenizer, &logprobs, false).unwrap(), + decode_logprobs(Some(&tokenizer), &logprobs, false).unwrap(), DecodedLogprobs { positions: vec![DecodedPositionLogprobs { entries: vec![ @@ -193,8 +207,13 @@ mod tests { }; assert_eq!( - decode_prompt_logprobs(&tokenizer, &[b'p' as u32, b'x' as u32], &logprobs, false) - .unwrap(), + decode_prompt_logprobs( + Some(&tokenizer), + &[b'p' as u32, b'x' as u32], + &logprobs, + false + ) + .unwrap(), DecodedPromptLogprobs { first_token_id: b'p' as u32, first_token: "p".to_string(), diff --git a/rust/src/text/src/request.rs b/rust/src/text/src/request.rs index 88c9ad47bb9..3e1e2968429 100644 --- a/rust/src/text/src/request.rs +++ b/rust/src/text/src/request.rs @@ -307,6 +307,13 @@ impl TextRequest { request_id: self.request_id.clone(), }); } + if !self.decode_options.detokenize + && self.decode_options.stop_strings.as_ref().is_some_and(|stops| !stops.is_empty()) + { + return Err(Error::StopStringsRequireDetokenize { + request_id: self.request_id.clone(), + }); + } if self.cache_salt.as_deref().is_some_and(|salt| !is_valid_cache_salt(salt)) { return Err(Error::InvalidCacheSalt); } diff --git a/rust/src/tokenizer/src/test_utils.rs b/rust/src/tokenizer/src/test_utils.rs index c693f2f3b9c..3af57053858 100644 --- a/rust/src/tokenizer/src/test_utils.rs +++ b/rust/src/tokenizer/src/test_utils.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; -use crate::{DecodedText, Result, Tokenizer, TokenizerError}; +use crate::{DecodedText, IncrementalDecoder, Result, Tokenizer, TokenizerError}; const FIRST_CONFIGURED_TOKEN_ID: u32 = 256; @@ -307,6 +307,46 @@ pub fn decode_ids(tokenizer: &dyn Tokenizer, token_ids: &[u32]) -> DecodedText { decoder.flush(None).unwrap().1 } +/// Tokenizer that panics on any encode, decode or token lookup, for tests that +/// prove a path never tokenizes or detokenizes. +#[derive(Debug)] +pub struct PanickingTokenizer; + +impl Tokenizer for PanickingTokenizer { + fn encode(&self, _text: &str, _add_special_tokens: bool) -> Result> { + panic!("unexpected tokenizer call: encode") + } + + fn encode_ordinary(&self, _text: &str) -> Result> { + panic!("unexpected tokenizer call: encode_ordinary") + } + + fn decode(&self, _token_ids: &[u32], _skip_special_tokens: bool) -> Result { + panic!("unexpected tokenizer call: decode") + } + + fn token_to_id(&self, _token: &str) -> Option { + panic!("unexpected tokenizer call: token_to_id") + } + + fn id_to_token(&self, _id: u32) -> Option { + panic!("unexpected tokenizer call: id_to_token") + } + + fn is_special_id(&self, _token_id: u32) -> bool { + panic!("unexpected tokenizer call: is_special_id") + } + + fn create_decode_stream( + &self, + _prompt_token_ids: &[u32], + _skip_special_tokens: bool, + _min_bytes_to_buffer: usize, + ) -> Box { + panic!("unexpected tokenizer call: create_decode_stream") + } +} + #[cfg(test)] mod tests { use super::*;