Make stacker an opt-in feature (#29505)

This commit is contained in:
Micha Reiser authored and GitHub committed 2026-10-19 18:02:19 -07:00
1 parent 355cda387a
commit 3e3f22dcc4
11 files changed
+248 -213

No files matched your search

+3
View File
@@ -443,6 +443,9 @@ jobs:
run: uv run --locked --only-dev cargo insta test --all-features --unreferenced reject --test-runner nextest --disable-nextest-doctest
- name: "Run doctests"
run: cargo test --doc --all-features
# A workspace build enables stacker through the CLI crates' dependencies.
- name: "Check parser without stacker"
run: cargo check --locked -p ruff_python_parser --no-default-features --all-targets
# On a PR, the annotation mdtests step above can install a Python version
# that uv can reuse for py-fuzzer. When the annotation step is skipped (as
# on pushes to `main`), uv may select a newer Python for the step below, for
Generated
+2
View File
@@ -4665,6 +4665,7 @@ dependencies = [
"ruff_db",
"ruff_diagnostics",
"ruff_python_ast",
"ruff_python_parser",
"ruff_python_trivia",
"ruff_ranged_value",
"salsa",
@@ -5061,6 +5062,7 @@ dependencies = [
"ruff_diagnostics",
"ruff_notebook",
"ruff_python_formatter",
"ruff_python_parser",
"ruff_ranged_value",
"ruff_source_file",
"ruff_text_size",
+1 -1
View File
@@ -32,7 +32,7 @@ ruff_notebook = { workspace = true }
ruff_options_metadata = { workspace = true, features = ["serde"] }
ruff_python_ast = { workspace = true }
ruff_python_formatter = { workspace = true }
ruff_python_parser = { workspace = true }
ruff_python_parser = { workspace = true, features = ["stacker"] }
ruff_ranged_value = { workspace = true }
ruff_server = { workspace = true }
ruff_source_file = { workspace = true }
+2 -2
View File
@@ -20,7 +20,7 @@ doctest = false
ruff_db = { workspace = true, features = ["testing"] }
ruff_linter = { workspace = true, optional = true }
ruff_python_formatter = { workspace = true, optional = true }
ruff_python_parser = { workspace = true, optional = true }
ruff_python_parser = { workspace = true, features = ["stacker"] }
ruff_python_trivia = { workspace = true, optional = true }
ty_ide = { workspace = true, optional = true }
ty_module_resolver = { workspace = true, optional = true }
@@ -58,7 +58,6 @@ ruff_instrumented = [
"criterion",
"ruff_linter",
"ruff_python_formatter",
"ruff_python_parser",
"ruff_python_trivia",
"mimalloc",
"tikv-jemallocator",
@@ -133,6 +132,7 @@ ignored = [
"divan",
"ruff_linter",
"ruff_python_formatter",
# Also enables stack growth when only ty benchmarks are selected.
"ruff_python_parser",
"ruff_python_trivia",
"ty_ide",
+4 -1
View File
@@ -26,7 +26,7 @@ hashbrown = { workspace = true }
memchr = { workspace = true }
rustc-hash = { workspace = true }
static_assertions = { workspace = true }
stacker = { workspace = true }
stacker = { workspace = true, optional = true }
thin-vec = { workspace = true }
unicode-ident = { workspace = true }
unicode-normalization = { workspace = true }
@@ -48,5 +48,8 @@ walkdir = { workspace = true }
name = "fixtures"
harness = false
[features]
stacker = ["dep:stacker"]
[lints]
workspace = true
+52 -29
View File
@@ -57,12 +57,6 @@ impl NameInterner {
}
}
// Stack probes access thread-local state, so avoid them while recursive parser calls remain
// shallow. `STACK_RED_ZONE` must cover the stack used before the first deferred probe.
const STACK_RED_ZONE: usize = 100 * 1024;
const STACK_SIZE: usize = 1024 * 1024;
const MAX_UNCHECKED_RECURSION_DEPTH: usize = 20;
#[derive(Debug)]
pub(crate) struct Parser<'src> {
source: &'src str,
@@ -99,6 +93,7 @@ pub(crate) struct Parser<'src> {
start_offset: TextSize,
/// Number of active recursive statement, expression, and pattern parsing operations.
#[cfg(feature = "stacker")]
recursion_depth: usize,
/// Reusable, nesting-safe scratch storage for expression lists.
@@ -145,6 +140,7 @@ impl<'src> Parser<'src> {
recovery_context: RecoveryContext::empty(),
prev_token_end: TextSize::new(0),
start_offset,
#[cfg(feature = "stacker")]
recursion_depth: 0,
current_token_id: TokenId::default(),
expr_scratch: ScratchBuffer::with_capacity(16),
@@ -156,33 +152,13 @@ impl<'src> Parser<'src> {
}
}
/// Grows the stack for recursive parser calls only after shallow nesting is exceeded.
#[inline]
fn with_recursion<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
self.recursion_depth += 1;
let result = if self.recursion_depth > MAX_UNCHECKED_RECURSION_DEPTH {
self.grow_stack(f)
} else {
f(self)
};
self.recursion_depth -= 1;
result
}
#[cold]
fn grow_stack<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
stacker::maybe_grow(STACK_RED_ZONE, STACK_SIZE, || f(self))
}
/// Consumes the [`Parser`] and returns the parsed [`Parsed`].
pub(crate) fn parse(mut self) -> Parsed<Mod> {
let syntax = stacker::maybe_grow(STACK_RED_ZONE, STACK_SIZE, || match self.options.mode {
let syntax = self.maybe_grow_stack(|parser| match parser.options.mode {
Mode::Expression | Mode::ParenthesizedExpression => {
Mod::Expression(self.parse_single_expression())
Mod::Expression(parser.parse_single_expression())
}
Mode::Module | Mode::Ipython => Mod::Module(self.parse_module()),
Mode::Module | Mode::Ipython => Mod::Module(parser.parse_module()),
});
self.finish(syntax)
@@ -971,6 +947,53 @@ impl<'src> Parser<'src> {
}
}
#[cfg(feature = "stacker")]
impl Parser<'_> {
// Stack probes access thread-local state, so avoid them while recursive parser calls remain
// shallow. `STACK_RED_ZONE` must cover the stack used before the first deferred probe.
const STACK_RED_ZONE: usize = 100 * 1024;
const STACK_SIZE: usize = 1024 * 1024;
const MAX_UNCHECKED_RECURSION_DEPTH: usize = 20;
#[inline]
fn maybe_grow_stack<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
stacker::maybe_grow(Self::STACK_RED_ZONE, Self::STACK_SIZE, || f(self))
}
/// Grows the stack for recursive parser calls only after shallow nesting is exceeded.
#[inline]
fn with_recursion<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
self.recursion_depth += 1;
let result = if self.recursion_depth > Self::MAX_UNCHECKED_RECURSION_DEPTH {
self.grow_stack(f)
} else {
f(self)
};
self.recursion_depth -= 1;
result
}
#[cold]
fn grow_stack<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
self.maybe_grow_stack(f)
}
}
#[cfg(not(feature = "stacker"))]
impl Parser<'_> {
#[inline]
fn maybe_grow_stack<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
f(self)
}
#[inline]
fn with_recursion<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
f(self)
}
}
fn strip_underscores(text: &str) -> Cow<'_, str> {
if text.as_bytes().contains(&b'_') {
Cow::Owned(text.chars().filter(|&c| c != '_').collect())
+174 -178
View File
@@ -1,13 +1,8 @@
use std::assert_matches;
use ruff_python_ast::{Expr, InterpolatedStringElement, IpyEscapeKind, Number, Stmt};
use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::{Mode, ParseOptions, parse, parse_cells_unchecked, parse_expression, parse_module};
// Keep recursive ASTs shallow enough for Windows's 1 MiB test-thread stacks.
const RECURSIVE_AST_TEST_DEPTH: usize = 1_000;
#[test]
fn test_modes() {
let source = "a[0][1][2][3][4]";
@@ -344,185 +339,12 @@ fn test_tstring_fstring_middle_fuzzer() {
insta::assert_debug_snapshot!(error);
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_parens_grow_stack() {
let src = format!("{}1{}", "(".repeat(1_000), ")".repeat(1_000));
let parsed = stacker::grow(32 * 1024, || parse_module(&src));
assert!(parsed.is_ok());
}
#[test]
fn normal_python_unaffected() {
let src = format!("x = {}1{}", "(".repeat(50), ")".repeat(50));
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn deep_nesting_preserves_surrounding_statements() {
let src = format!(
"before = 1\n{}1{}\nafter = 2\n",
"(".repeat(1_000),
")".repeat(1_000),
);
let parsed = parse_module(&src).unwrap();
assert_matches!(parsed.suite().first(), Some(Stmt::Assign(_)));
assert_matches!(parsed.suite().last(), Some(Stmt::Assign(_)));
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_def_blocks_grow_stack() {
// Each nested function crosses the suite boundary where the parser rechecks the stack.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::new();
for i in 0..depth {
src.push_str(&"\t".repeat(i));
src.push_str("def f():\n");
}
src.push_str(&"\t".repeat(depth));
src.push_str("pass\n");
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_lists_grow_stack() {
let src = format!("{}1{}", "[".repeat(1_000), "]".repeat(1_000));
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_calls_grow_stack() {
let src = format!("x = {}1{}", "f(".repeat(1_000), ")".repeat(1_000));
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_subscripts_grow_stack() {
let src = format!("x = {}1{}", "a[".repeat(1_000), "]".repeat(1_000));
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_match_patterns_grow_stack() {
// Deeply parenthesised match patterns — exercises pattern-parsing
// instrumentation in addition to statement / expression paths.
let mut src = String::from("match x:\n case ");
for _ in 0..600 {
src.push('(');
}
src.push('y');
for _ in 0..600 {
src.push(')');
}
src.push_str(": pass\n");
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_invalid_mapping_pattern_keys_grow_stack() {
let depth = 512;
let src = format!(
"match value:\n case {}0{}:\n pass\n",
"{".repeat(depth),
": 0}".repeat(depth)
);
let parsed = crate::parse_unchecked(&src, ParseOptions::from(Mode::Module));
assert!(!parsed.errors().is_empty());
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn binary_paren_interplay_grows_stack() {
// `1+(1+(1+(1+...)))` — each level alternates a binary operator and a
// parenthesised sub-expression, exactly like the pattern described in
// the tracking issue.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::new();
for _ in 0..depth {
src.push_str("1+(");
}
src.push('1');
for _ in 0..depth {
src.push(')');
}
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn right_assoc_pow_chain_grows_stack() {
// `1**1**1**...**1` — `**` is right-associative, so the right operand
// is parsed by a recursive `parse_binary_expression_or_higher` call
// *without* any intervening parentheses or atom nesting. This exercises
// the binary-expression recursion path directly, unlike the
// `1+(1+(...))` interplay test which recurses through parenthesised
// atoms.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::with_capacity(depth * 3 + 1);
for _ in 0..depth {
src.push_str("1**");
}
src.push('1');
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn ternary_else_chain_grows_stack() {
// `1 if 1 else 1 if 1 else ...` — the `else` operand recurses at the
// conditional layer (`parse_if_expression` -> `orelse`), which is not
// covered by the binary-expression guard.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::with_capacity(depth * 12 + 1);
for _ in 0..depth {
src.push_str("1 if 1 else ");
}
src.push('1');
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_lambda_chain_grows_stack() {
// `lambda: lambda: lambda: ...` — the lambda body recurses at the
// conditional layer (`parse_lambda_expr` -> body), bypassing the
// binary-expression guard entirely.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::from("x = ");
for _ in 0..depth {
src.push_str("lambda: ");
}
src.push('1');
parse_module(&src).unwrap();
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn invalid_async_chain_grows_stack() {
let source = format!("{}x = 1\n", "async ".repeat(5_000));
let parsed = crate::parse_unchecked(&source, ParseOptions::from(Mode::Module));
assert!(!parsed.errors().is_empty());
}
#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
#[test]
fn nested_unary_chains_grow_stack() {
let depth = 300;
let source = format!("{}1\n", "-~+".repeat(depth));
parse_module(&source).unwrap();
let source = format!("{}True\n", "not ".repeat(depth));
parse_module(&source).unwrap();
}
/// The module range of a cell-wise parse matches a whole-source parse, so that offsets in the
/// notebook's trailing newline (for example, past the end of an empty last cell) are inside it.
#[test]
@@ -541,3 +363,177 @@ fn cells_module_range_spans_source() {
parse_module(source).unwrap().syntax().range()
);
}
#[cfg(all(
feature = "stacker",
any(target_os = "linux", target_os = "macos", target_os = "windows")
))]
mod stack_growth {
use std::assert_matches;
use ruff_python_ast::Stmt;
use crate::{Mode, ParseOptions, parse_module};
// Keep recursive ASTs shallow enough for Windows's 1 MiB test-thread stacks.
const RECURSIVE_AST_TEST_DEPTH: usize = 1_000;
#[test]
fn nested_parens_grow_stack() {
let src = format!("{}1{}", "(".repeat(1_000), ")".repeat(1_000));
let parsed = stacker::grow(32 * 1024, || parse_module(&src));
assert!(parsed.is_ok());
}
#[test]
fn deep_nesting_preserves_surrounding_statements() {
let src = format!(
"before = 1\n{}1{}\nafter = 2\n",
"(".repeat(1_000),
")".repeat(1_000),
);
let parsed = parse_module(&src).unwrap();
assert_matches!(parsed.suite().first(), Some(Stmt::Assign(_)));
assert_matches!(parsed.suite().last(), Some(Stmt::Assign(_)));
}
#[test]
fn nested_def_blocks_grow_stack() {
// Each nested function crosses the suite boundary where the parser rechecks the stack.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::new();
for i in 0..depth {
src.push_str(&"\t".repeat(i));
src.push_str("def f():\n");
}
src.push_str(&"\t".repeat(depth));
src.push_str("pass\n");
parse_module(&src).unwrap();
}
#[test]
fn nested_lists_grow_stack() {
let src = format!("{}1{}", "[".repeat(1_000), "]".repeat(1_000));
parse_module(&src).unwrap();
}
#[test]
fn nested_calls_grow_stack() {
let src = format!("x = {}1{}", "f(".repeat(1_000), ")".repeat(1_000));
parse_module(&src).unwrap();
}
#[test]
fn nested_subscripts_grow_stack() {
let src = format!("x = {}1{}", "a[".repeat(1_000), "]".repeat(1_000));
parse_module(&src).unwrap();
}
#[test]
fn nested_match_patterns_grow_stack() {
// Deeply parenthesised match patterns — exercises pattern-parsing
// instrumentation in addition to statement / expression paths.
let mut src = String::from("match x:\n case ");
for _ in 0..600 {
src.push('(');
}
src.push('y');
for _ in 0..600 {
src.push(')');
}
src.push_str(": pass\n");
parse_module(&src).unwrap();
}
#[test]
fn nested_invalid_mapping_pattern_keys_grow_stack() {
let depth = 512;
let src = format!(
"match value:\n case {}0{}:\n pass\n",
"{".repeat(depth),
": 0}".repeat(depth)
);
let parsed = crate::parse_unchecked(&src, ParseOptions::from(Mode::Module));
assert!(!parsed.errors().is_empty());
}
#[test]
fn binary_paren_interplay_grows_stack() {
// `1+(1+(1+(1+...)))` — each level alternates a binary operator and a
// parenthesised sub-expression, exactly like the pattern described in
// the tracking issue.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::new();
for _ in 0..depth {
src.push_str("1+(");
}
src.push('1');
for _ in 0..depth {
src.push(')');
}
parse_module(&src).unwrap();
}
#[test]
fn right_assoc_pow_chain_grows_stack() {
// `1**1**1**...**1` — `**` is right-associative, so the right operand
// is parsed by a recursive `parse_binary_expression_or_higher` call
// *without* any intervening parentheses or atom nesting. This exercises
// the binary-expression recursion path directly, unlike the
// `1+(1+(...))` interplay test which recurses through parenthesised
// atoms.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::with_capacity(depth * 3 + 1);
for _ in 0..depth {
src.push_str("1**");
}
src.push('1');
parse_module(&src).unwrap();
}
#[test]
fn ternary_else_chain_grows_stack() {
// `1 if 1 else 1 if 1 else ...` — the `else` operand recurses at the
// conditional layer (`parse_if_expression` -> `orelse`), which is not
// covered by the binary-expression guard.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::with_capacity(depth * 12 + 1);
for _ in 0..depth {
src.push_str("1 if 1 else ");
}
src.push('1');
parse_module(&src).unwrap();
}
#[test]
fn nested_lambda_chain_grows_stack() {
// `lambda: lambda: lambda: ...` — the lambda body recurses at the
// conditional layer (`parse_lambda_expr` -> body), bypassing the
// binary-expression guard entirely.
let depth = RECURSIVE_AST_TEST_DEPTH;
let mut src = String::from("x = ");
for _ in 0..depth {
src.push_str("lambda: ");
}
src.push('1');
parse_module(&src).unwrap();
}
#[test]
fn invalid_async_chain_grows_stack() {
let source = format!("{}x = 1\n", "async ".repeat(5_000));
let parsed = crate::parse_unchecked(&source, ParseOptions::from(Mode::Module));
assert!(!parsed.errors().is_empty());
}
#[test]
fn nested_unary_chains_grow_stack() {
let depth = 300;
let source = format!("{}1\n", "-~+".repeat(depth));
parse_module(&source).unwrap();
let source = format!("{}True\n", "not ".repeat(depth));
parse_module(&source).unwrap();
}
}
+1 -1
View File
@@ -23,7 +23,7 @@ ruff_python_ast = { workspace = true }
ruff_python_codegen = { workspace = true }
ruff_python_formatter = { workspace = true }
ruff_python_index = { workspace = true }
ruff_python_parser = { workspace = true }
ruff_python_parser = { workspace = true, features = ["stacker"] }
ruff_python_trivia = { workspace = true }
ruff_ranged_value = { workspace = true }
ruff_source_file = { workspace = true }
+5
View File
@@ -14,6 +14,10 @@ license.workspace = true
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[package.metadata.cargo-shear]
# Used only to enable parser stack growth.
ignored = ["ruff_python_parser"]
[lib]
doctest = false
@@ -22,6 +26,7 @@ ruff_command_line = { workspace = true }
ruff_db = { workspace = true, features = ["os", "cache", "junit"] }
ruff_diagnostics = { workspace = true }
ruff_python_ast = { workspace = true }
ruff_python_parser = { workspace = true, features = ["stacker"] }
ruff_ranged_value = { workspace = true }
ty_combine = { workspace = true }
ty_project = { workspace = true, features = ["zstd", "junit"] }
+3
View File
@@ -13,6 +13,8 @@ description = "WebAssembly bindings for ty"
[package.metadata.cargo-shear]
ignored = [
# Used only to enable parser stack growth.
"ruff_python_parser",
# Depended on only to enable `log` feature as of 2025-10-03.
"tracing",
]
@@ -35,6 +37,7 @@ ruff_db = { workspace = true, default-features = false, features = [] }
ruff_diagnostics = { workspace = true }
ruff_notebook = { workspace = true }
ruff_python_formatter = { workspace = true }
ruff_python_parser = { workspace = true, features = ["stacker"] }
ruff_ranged_value = { workspace = true }
ruff_source_file = { workspace = true }
ruff_text_size = { workspace = true }
+1 -1
View File
@@ -21,7 +21,7 @@ ruff_db = { path = "../crates/ruff_db" }
ruff_linter = { path = "../crates/ruff_linter" }
ruff_python_ast = { path = "../crates/ruff_python_ast" }
ruff_python_codegen = { path = "../crates/ruff_python_codegen" }
ruff_python_parser = { path = "../crates/ruff_python_parser" }
ruff_python_parser = { path = "../crates/ruff_python_parser", features = ["stacker"] }
ruff_python_formatter = { path = "../crates/ruff_python_formatter" }
ruff_text_size = { path = "../crates/ruff_text_size" }