[ty] Highlight PEP 723 script metadata as TOML (#27746)

## Summary

This PR adds support for TOML syntax highlighting in PEP 723 inline
metadata blocks by emitting TOML specific tokens during semantic
highlighting. I know, this wasn't really something we planned on adding
but I got annoyed by starring at large gray blocks and it was too easy.

This approach is relatively straightforward, and has the advantage that
it works for all editors. However, it also comes with limitations:

* Parentheses matching and completions don't work
* It can't easily support inserting `# ` on enter (when adding a new
line)
* No support for go to def, hover etc (we could add them, but it becomes
a bit more questionable whether this is in ty's scope)

To support these features, we'd have to add an embedded grammar to
ty-vscode (and other editors). I think this is still something worth
considering, and the approach taken here doesn't prevent us from doing
this. That's why I decided to go with this easier and cross-editor
implementation for now.

The emitted tokens are inspired by the grammars of VS Code toml
extensions.

Note: I'm only aware of sublime supporting semantic highlighting of
inline metadata. I don't think pyrefly or pylance do.

## Test Plan

<img width="1245" height="603" alt="Screenshot 2026-08-14 at 11 07 41"
src="https://github.com/user-attachments/assets/120785d5-18a9-478e-af65-77819446b2d2"
/>

Cargo.lock

@@ -4751,6 +4751,7 @@ dependencies = [
  "smallvec",
  "strum",
  "strum_macros",
+ "toml_parser",
  "tracing",
  "ty_module_resolver",
  "ty_project",

crates/ty_ide/Cargo.toml

@@ -44,6 +44,7 @@ salsa = { workspace = true, features = ["compact_str"] }
 smallvec = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
+toml_parser = { workspace = true }
 tracing = { workspace = true }
 
 [dev-dependencies]

crates/ty_ide/src/semantic_tokens.rs

@@ -29,6 +29,7 @@
 use crate::Db;
 use bitflags::bitflags;
 use ruff_db::parsed::parsed_module;
+use ruff_python_ast::script::ScriptTag;
 use ruff_python_ast::visitor::source_order::{
     SourceOrderVisitor, TraversalSignal, walk_arguments, walk_expr,
     walk_interpolated_string_element, walk_stmt,
@@ -39,6 +40,9 @@ use ruff_python_ast::{
 };
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 use std::ops::Deref;
+use toml_parser::decoder::ScalarKind;
+use toml_parser::parser::{Event, EventKind};
+use ty_project::script_tag;
 use ty_python_core::ProgramFile;
 use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind};
 use ty_python_semantic::{
@@ -70,11 +74,13 @@ pub enum SemanticTokenType {
     Decorator,
     BuiltinConstant,
     TypeParameter,
+    Operator,
+    Regexp,
 }
 
 impl SemanticTokenType {
     /// Returns all supported semantic token types as enum variants.
-    pub const fn all() -> [SemanticTokenType; 15] {
+    pub const fn all() -> [SemanticTokenType; 17] {
         [
             SemanticTokenType::Namespace,
             SemanticTokenType::Class,
@@ -91,6 +97,8 @@ impl SemanticTokenType {
             SemanticTokenType::Decorator,
             SemanticTokenType::BuiltinConstant,
             SemanticTokenType::TypeParameter,
+            SemanticTokenType::Operator,
+            SemanticTokenType::Regexp,
         ]
     }
 
@@ -118,6 +126,8 @@ impl SemanticTokenType {
             SemanticTokenType::Decorator => "decorator",
             SemanticTokenType::BuiltinConstant => "builtinConstant",
             SemanticTokenType::TypeParameter => "typeParameter",
+            SemanticTokenType::Operator => "operator",
+            SemanticTokenType::Regexp => "regexp",
         }
     }
 }
@@ -195,9 +205,104 @@ pub fn semantic_tokens(
     visitor.expecting_docstring = true;
     visitor.visit_body(parsed.suite());
 
+    if let Some(tag) = script_tag(db, file.file(db)) {
+        let insertion = visitor
+            .tokens
+            .partition_point(|token| token.start() < tag.start());
+        visitor
+            .tokens
+            .splice(insertion..insertion, script_metadata_tokens(tag, range));
+    }
+
     SemanticTokens::new(visitor.tokens)
 }
 
+/// Highlights embedded TOML without treating its Python comment prefixes as TOML content.
+fn script_metadata_tokens(tag: &ScriptTag, range: Option<TextRange>) -> Vec<SemanticToken> {
+    let metadata = tag.metadata();
+    let tokens = toml_parser::Source::new(metadata).lex().collect::<Vec<_>>();
+    let mut semantic_tokens = Vec::new();
+    let mut in_table_header = false;
+
+    toml_parser::parser::parse_document(
+        &tokens,
+        &mut |event: Event| {
+            let span = event.span();
+            let Some(text) = metadata.get(span.start()..span.end()) else {
+                return;
+            };
+
+            let token_type = match event.kind() {
+                EventKind::StdTableOpen | EventKind::ArrayTableOpen => {
+                    in_table_header = true;
+                    SemanticTokenType::Operator
+                }
+                EventKind::StdTableClose | EventKind::ArrayTableClose => {
+                    in_table_header = false;
+                    SemanticTokenType::Operator
+                }
+                EventKind::InlineTableOpen
+                | EventKind::InlineTableClose
+                | EventKind::ArrayOpen
+                | EventKind::ArrayClose
+                | EventKind::KeySep
+                | EventKind::KeyValSep
+                | EventKind::ValueSep => SemanticTokenType::Operator,
+                EventKind::SimpleKey if in_table_header => SemanticTokenType::Namespace,
+                EventKind::SimpleKey => SemanticTokenType::Variable,
+                EventKind::Scalar => {
+                    let scalar = toml_parser::Raw::new_unchecked(text, event.encoding(), span);
+
+                    match scalar.decode_scalar(&mut (), &mut ()) {
+                        ScalarKind::String => SemanticTokenType::String,
+                        ScalarKind::Boolean(_) => SemanticTokenType::BuiltinConstant,
+                        ScalarKind::DateTime => SemanticTokenType::Regexp,
+                        ScalarKind::Float | ScalarKind::Integer(_) => SemanticTokenType::Number,
+                    }
+                }
+                EventKind::Newline => {
+                    in_table_header = false;
+                    return;
+                }
+                _ => return,
+            };
+
+            let Ok(mut offset) = TextSize::try_from(span.start()) else {
+                return;
+            };
+
+            // Multiline TOML values are interrupted by Python comment prefixes in the source.
+            // Highlight each metadata line separately so those prefixes remain comments.
+            for line in text.split_inclusive('\n') {
+                let content = line.strip_suffix('\n').unwrap_or(line);
+                let token_range = tag
+                    .source_map()
+                    .map_range(TextRange::at(offset, content.text_len()));
+                offset += line.text_len();
+
+                if token_range.is_empty()
+                    || range.is_some_and(|requested| {
+                        token_range
+                            .intersect(requested)
+                            .is_none_or(TextRange::is_empty)
+                    })
+                {
+                    continue;
+                }
+
+                semantic_tokens.push(SemanticToken {
+                    range: token_range,
+                    token_type,
+                    modifiers: SemanticTokenModifier::empty(),
+                });
+            }
+        },
+        &mut (),
+    );
+
+    semantic_tokens
+}
+
 /// AST visitor that collects semantic tokens.
 #[expect(clippy::struct_excessive_bools)]
 struct SemanticTokenVisitor<'db> {
@@ -1301,6 +1406,209 @@ mod tests {
         assert_snapshot!(test.to_snapshot(&tokens), @r#""foo" @ 4..7: Function [definition]"#);
     }
 
+    #[test]
+    fn script_metadata_highlights_toml_in_source_order() {
+        let source = r#"before = 1
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["httpx", "attrs"]
+# [tool.uv]
+# enabled = true
+# disabled = false
+# retries = 2
+# "quoted-key" = { nested = "value" }
+# [[tool.packages]]
+# name = "example"
+# ///
+after = 2
+"#;
+        let test = SemanticTokenTest::new(source);
+        let tokens = test.highlight_file();
+        let source = ruff_db::source::source_text(&test.db, test.file);
+        let highlighted: Vec<_> = tokens
+            .iter()
+            .map(|token| (&source[token.range()], token.token_type))
+            .collect();
+
+        assert_eq!(
+            highlighted,
+            vec![
+                ("before", SemanticTokenType::Variable),
+                ("1", SemanticTokenType::Number),
+                ("requires-python", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("\">=3.12\"", SemanticTokenType::String),
+                ("dependencies", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("[", SemanticTokenType::Operator),
+                ("\"httpx\"", SemanticTokenType::String),
+                (",", SemanticTokenType::Operator),
+                ("\"attrs\"", SemanticTokenType::String),
+                ("]", SemanticTokenType::Operator),
+                ("[", SemanticTokenType::Operator),
+                ("tool", SemanticTokenType::Namespace),
+                (".", SemanticTokenType::Operator),
+                ("uv", SemanticTokenType::Namespace),
+                ("]", SemanticTokenType::Operator),
+                ("enabled", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("true", SemanticTokenType::BuiltinConstant),
+                ("disabled", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("false", SemanticTokenType::BuiltinConstant),
+                ("retries", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("2", SemanticTokenType::Number),
+                ("\"quoted-key\"", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("{", SemanticTokenType::Operator),
+                ("nested", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("\"value\"", SemanticTokenType::String),
+                ("}", SemanticTokenType::Operator),
+                ("[[", SemanticTokenType::Operator),
+                ("tool", SemanticTokenType::Namespace),
+                (".", SemanticTokenType::Operator),
+                ("packages", SemanticTokenType::Namespace),
+                ("]]", SemanticTokenType::Operator),
+                ("name", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("\"example\"", SemanticTokenType::String),
+                ("after", SemanticTokenType::Variable),
+                ("2", SemanticTokenType::Number),
+            ]
+        );
+    }
+
+    #[test]
+    fn script_metadata_distinguishes_numeric_and_datetime_values() {
+        let source = r#"# /// script
+# integer = 42
+# hexadecimal = 0xff
+# float = 1.5
+# infinity = +inf
+# not-a-number = nan
+# date = 2026-08-14
+# time = 08:15:30
+# datetime = 2026-08-14T08:15:30
+# offset-datetime = 2026-08-14T08:15:30Z
+# ///
+"#;
+        let test = SemanticTokenTest::new(source);
+        let tokens = test.highlight_file();
+        let source = ruff_db::source::source_text(&test.db, test.file);
+        let values: Vec<_> = tokens
+            .iter()
+            .filter(|token| {
+                matches!(
+                    token.token_type,
+                    SemanticTokenType::Number | SemanticTokenType::Regexp
+                )
+            })
+            .map(|token| (&source[token.range()], token.token_type))
+            .collect();
+
+        assert_eq!(
+            values,
+            vec![
+                ("42", SemanticTokenType::Number),
+                ("0xff", SemanticTokenType::Number),
+                ("1.5", SemanticTokenType::Number),
+                ("+inf", SemanticTokenType::Number),
+                ("nan", SemanticTokenType::Number),
+                ("2026-08-14", SemanticTokenType::Regexp),
+                ("08:15:30", SemanticTokenType::Regexp),
+                ("2026-08-14T08:15:30", SemanticTokenType::Regexp),
+                ("2026-08-14T08:15:30Z", SemanticTokenType::Regexp),
+            ]
+        );
+    }
+
+    #[test]
+    fn script_metadata_multiline_strings_exclude_comment_prefixes() {
+        let source = r#"# /// script
+# description = """
+# first
+#
+# last
+# """
+# ///
+"#;
+        let test = SemanticTokenTest::new(source);
+        let tokens = test.highlight_file();
+        let source = ruff_db::source::source_text(&test.db, test.file);
+        let highlighted: Vec<_> = tokens
+            .iter()
+            .map(|token| (&source[token.range()], token.token_type))
+            .collect();
+
+        assert_eq!(
+            highlighted,
+            vec![
+                ("description", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("\"\"\"", SemanticTokenType::String),
+                ("first", SemanticTokenType::String),
+                ("last", SemanticTokenType::String),
+                ("\"\"\"", SemanticTokenType::String),
+            ]
+        );
+    }
+
+    #[test]
+    fn script_metadata_inside_statement_remains_in_source_order() {
+        let source = r#"value = (
+    1
+# /// script
+# dependencies = ["httpx"]
+# ///
+    + 2
+)
+"#;
+        let test = SemanticTokenTest::new(source);
+        let tokens = test.highlight_file();
+        let source = ruff_db::source::source_text(&test.db, test.file);
+        let highlighted: Vec<_> = tokens
+            .iter()
+            .map(|token| (&source[token.range()], token.token_type))
+            .collect();
+
+        assert_eq!(
+            highlighted,
+            vec![
+                ("value", SemanticTokenType::Variable),
+                ("1", SemanticTokenType::Number),
+                ("dependencies", SemanticTokenType::Variable),
+                ("=", SemanticTokenType::Operator),
+                ("[", SemanticTokenType::Operator),
+                ("\"httpx\"", SemanticTokenType::String),
+                ("]", SemanticTokenType::Operator),
+                ("2", SemanticTokenType::Number),
+            ]
+        );
+    }
+
+    #[test]
+    fn script_metadata_respects_requested_token_range() {
+        let source = r#"# /// script
+# dependencies = ["httpx"]
+# ///
+value = 1
+"#;
+        let test = SemanticTokenTest::new(source);
+        let range = TextRange::at(
+            r"# /// script
+# dependencies = ["
+                .text_len(),
+            "\"httpx\"".text_len(),
+        );
+        let tokens = test.highlight_range(range);
+
+        assert_eq!(tokens.len(), 1);
+        assert_eq!(tokens[0].range(), range);
+        assert_eq!(tokens[0].token_type, SemanticTokenType::String);
+    }
+
     #[test]
     fn semantic_tokens_class() {
         let test = SemanticTokenTest::new("class MyClass: pass");

crates/ty_project/src/lib.rs

@@ -23,7 +23,7 @@ use ruff_db::parsed::parsed_module;
 use ruff_db::system::{SystemPath, SystemPathBuf, deduplicate_nested_paths};
 use rustc_hash::FxHashSet;
 use salsa::{Database, Durability, Setter};
-pub use script::{ScriptEnvironmentAvailability, ScriptEnvironments};
+pub use script::{ScriptEnvironmentAvailability, ScriptEnvironments, script_tag};
 use std::backtrace::BacktraceStatus;
 use std::collections::{BTreeSet, hash_set};
 use std::iter::FusedIterator;

crates/ty_project/src/script.rs

@@ -137,7 +137,7 @@ pub(crate) fn script(db: &dyn Db, file: File) -> Option<Script<'_>> {
 ///
 /// Most files have no script tag. Boxing keeps the cached result compact when it is `None`.
 #[salsa::tracked(returns(as_deref))]
-pub(crate) fn script_tag(db: &dyn SourceDb, file: File) -> Option<Box<ScriptTag>> {
+pub fn script_tag(db: &dyn SourceDb, file: File) -> Option<Box<ScriptTag>> {
     let path = file.path(db);
     if path.is_vendored_path() {
         return None;

crates/ty_server/tests/e2e/semantic_tokens.rs

@@ -3,6 +3,62 @@ use ruff_db::system::SystemPath;
 
 use crate::TestServerBuilder;
 
+#[test]
+fn script_metadata_is_highlighted_as_toml() -> Result<()> {
+    let workspace_root = SystemPath::new("src");
+    let script = SystemPath::new("src/script.py");
+    let source = r#"#!/usr/bin/env python3
+# /// script
+# dependencies = ["httpx"]
+# requires-python = ">=3.12"
+# ///
+value = 1
+"#;
+
+    let mut server = TestServerBuilder::new()?
+        .with_workspace(workspace_root, None)?
+        .with_file(script, source)?
+        .build()
+        .wait_until_workspaces_are_initialized();
+
+    server.open_text_document(script, source, 1);
+
+    let tokens = server
+        .semantic_tokens_full_request(&server.file_uri(script))
+        .ok_or_else(|| anyhow::anyhow!("expected semantic tokens for the script"))?;
+
+    let actual: Vec<_> = tokens
+        .data
+        .into_iter()
+        .map(|token| {
+            (
+                token.delta_line,
+                token.delta_start,
+                token.length,
+                token.token_type,
+            )
+        })
+        .collect();
+
+    assert_eq!(
+        actual,
+        vec![
+            (2, 2, 12, ty_ide::SemanticTokenType::Variable as u32),
+            (0, 13, 1, ty_ide::SemanticTokenType::Operator as u32),
+            (0, 2, 1, ty_ide::SemanticTokenType::Operator as u32),
+            (0, 1, 7, ty_ide::SemanticTokenType::String as u32),
+            (0, 7, 1, ty_ide::SemanticTokenType::Operator as u32),
+            (1, 2, 15, ty_ide::SemanticTokenType::Variable as u32),
+            (0, 16, 1, ty_ide::SemanticTokenType::Operator as u32),
+            (0, 2, 8, ty_ide::SemanticTokenType::String as u32),
+            (2, 0, 5, ty_ide::SemanticTokenType::Variable as u32),
+            (0, 8, 1, ty_ide::SemanticTokenType::Number as u32),
+        ]
+    );
+
+    Ok(())
+}
+
 #[test]
 fn multiline_token_client_not_supporting_multiline_tokens() -> Result<()> {
     let workspace_root = SystemPath::new("src");

crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization.snap

@@ -83,7 +83,9 @@ expression: initialization_result
           "number",
           "decorator",
           "builtinConstant",
-          "typeParameter"
+          "typeParameter",
+          "operator",
+          "regexp"
         ],
         "tokenModifiers": [
           "definition",

crates/ty_server/tests/e2e/snapshots/e2e__initialize__initialization_with_workspace.snap

@@ -83,7 +83,9 @@ expression: initialization_result
           "number",
           "decorator",
           "builtinConstant",
-          "typeParameter"
+          "typeParameter",
+          "operator",
+          "regexp"
         ],
         "tokenModifiers": [
           "definition",

crates/ty_wasm/src/lib.rs

@@ -1585,6 +1585,8 @@ pub enum SemanticTokenKind {
     Decorator,
     BuiltinConstant,
     TypeParameter,
+    Operator,
+    Regexp,
 }
 
 impl From<ty_ide::SemanticTokenType> for SemanticTokenKind {
@@ -1605,6 +1607,8 @@ impl From<ty_ide::SemanticTokenType> for SemanticTokenKind {
             ty_ide::SemanticTokenType::Decorator => Self::Decorator,
             ty_ide::SemanticTokenType::BuiltinConstant => Self::BuiltinConstant,
             ty_ide::SemanticTokenType::TypeParameter => Self::TypeParameter,
+            ty_ide::SemanticTokenType::Operator => Self::Operator,
+            ty_ide::SemanticTokenType::Regexp => Self::Regexp,
         }
     }
 }