PEP 750 tstring (#6744)

This commit is contained in:
Jeong, YunWon authored and GitHub committed 2026-01-17 18:41:40 +09:00
1 parent 02cc257b42
commit f4363a6b27
13 files changed
+1086 -76

No files matched your search

+2
View File
@@ -132,6 +132,8 @@ opmap = {
'BUILD_SET_FROM_TUPLES': 122,
'BUILD_TUPLE_FROM_ITER': 123,
'BUILD_TUPLE_FROM_TUPLES': 124,
'BUILD_TEMPLATE': 125,
'BUILD_INTERPOLATION': 126,
'CONTINUE': 128,
'JUMP_IF_FALSE_OR_POP': 129,
'JUMP_IF_TRUE_OR_POP': 130,
+114 -6
View File
@@ -24,13 +24,13 @@ use num_traits::{Num, ToPrimitive};
use ruff_python_ast::{
Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem,
ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprContext,
ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp,
FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedStringElement,
ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTString, ExprTuple,
ExprUnaryOp, FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedStringElement,
InterpolatedStringElements, Keyword, MatchCase, ModExpression, ModModule, Operator, Parameters,
Pattern, PatternMatchAs, PatternMatchClass, PatternMatchMapping, PatternMatchOr,
PatternMatchSequence, PatternMatchSingleton, PatternMatchStar, PatternMatchValue, Singleton,
Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple,
TypeParams, UnaryOp, WithItem,
Stmt, StmtExpr, TString, TypeParam, TypeParamParamSpec, TypeParamTypeVar,
TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem,
visitor::{Visitor, walk_expr},
};
use ruff_text_size::{Ranged, TextRange};
@@ -6282,8 +6282,8 @@ impl Compiler {
Expr::FString(fstring) => {
self.compile_expr_fstring(fstring)?;
}
Expr::TString(_) => {
return Err(self.error(CodegenErrorType::NotImplementedYet));
Expr::TString(tstring) => {
self.compile_expr_tstring(tstring)?;
}
Expr::StringLiteral(string) => {
let value = string.value.to_str();
@@ -7466,6 +7466,114 @@ impl Compiler {
Ok(())
}
fn compile_expr_tstring(&mut self, expr_tstring: &ExprTString) -> CompileResult<()> {
// TStringValue can contain multiple TString parts (implicit concatenation)
// Each TString part should be compiled and the results merged into a single Template
let tstring_value = &expr_tstring.value;
// Collect all strings and compile all interpolations
let mut all_strings: Vec<Wtf8Buf> = Vec::new();
let mut current_string = Wtf8Buf::new();
let mut interp_count: u32 = 0;
for tstring in tstring_value.iter() {
self.compile_tstring_into(
tstring,
&mut all_strings,
&mut current_string,
&mut interp_count,
)?;
}
// Add trailing string
all_strings.push(std::mem::take(&mut current_string));
// Now build the Template:
// Stack currently has all interpolations from compile_tstring_into calls
// 1. Build interpolations tuple from the interpolations on the stack
emit!(self, Instruction::BuildTuple { size: interp_count });
// 2. Load all string parts
let string_count: u32 = all_strings
.len()
.try_into()
.expect("t-string string count overflowed");
for s in &all_strings {
self.emit_load_const(ConstantData::Str { value: s.clone() });
}
// 3. Build strings tuple
emit!(self, Instruction::BuildTuple { size: string_count });
// 4. Swap so strings is below interpolations: [interps, strings] -> [strings, interps]
emit!(self, Instruction::Swap { index: 2 });
// 5. Build the Template
emit!(self, Instruction::BuildTemplate);
Ok(())
}
fn compile_tstring_into(
&mut self,
tstring: &TString,
strings: &mut Vec<Wtf8Buf>,
current_string: &mut Wtf8Buf,
interp_count: &mut u32,
) -> CompileResult<()> {
for element in &tstring.elements {
match element {
InterpolatedStringElement::Literal(lit) => {
// Accumulate literal parts into current_string
current_string.push_str(&lit.value);
}
InterpolatedStringElement::Interpolation(interp) => {
// Finish current string segment
strings.push(std::mem::take(current_string));
// Compile the interpolation value
self.compile_expression(&interp.expression)?;
// Load the expression source string
let expr_range = interp.expression.range();
let expr_source = self.source_file.slice(expr_range);
self.emit_load_const(ConstantData::Str {
value: expr_source.to_string().into(),
});
// Determine conversion code
let conversion: u32 = match interp.conversion {
ConversionFlag::None => 0,
ConversionFlag::Str => 1,
ConversionFlag::Repr => 2,
ConversionFlag::Ascii => 3,
};
// Handle format_spec
let has_format_spec = interp.format_spec.is_some();
if let Some(format_spec) = &interp.format_spec {
// Compile format_spec as a string using fstring element compilation
// Use default FStringFlags since format_spec syntax is independent of t-string flags
self.compile_fstring_elements(
FStringFlags::empty(),
&format_spec.elements,
)?;
}
// Emit BUILD_INTERPOLATION
// oparg encoding: (conversion << 2) | has_format_spec
let oparg = (conversion << 2) | (has_format_spec as u32);
emit!(self, Instruction::BuildInterpolation { oparg });
*interp_count += 1;
}
}
}
Ok(())
}
}
trait EmitArg<Arg: OpArgType> {
+13 -8
View File
@@ -1441,14 +1441,19 @@ impl SymbolTableBuilder {
}
}
Expr::TString(tstring) => {
return Err(SymbolTableError {
error: "not yet implemented".into(),
location: Some(
self.source_file
.to_source_code()
.source_location(tstring.range.start(), PositionEncoding::Utf8),
),
});
// Scan t-string interpolation expressions (similar to f-strings)
for expr in tstring
.value
.elements()
.filter_map(|x| x.as_interpolation())
{
self.scan_expression(&expr.expression, ExpressionContext::Load)?;
if let Some(format_spec) = &expr.format_spec {
for element in format_spec.elements.interpolations() {
self.scan_expression(&element.expression, ExpressionContext::Load)?
}
}
}
}
// Constants
Expr::StringLiteral(_)
@@ -264,6 +264,19 @@ pub enum Instruction {
BuildTupleFromTuples {
size: Arg<u32>,
} = 124,
/// Build a Template from strings tuple and interpolations tuple on stack.
/// Stack: [strings_tuple, interpolations_tuple] -> [template]
BuildTemplate = 125,
/// Build an Interpolation from value, expression string, and optional format_spec on stack.
///
/// oparg encoding: (conversion << 2) | has_format_spec
/// - has_format_spec (bit 0): if 1, format_spec is on stack
/// - conversion (bits 2+): 0=None, 1=Str, 2=Repr, 3=Ascii
///
/// Stack: [value, expression_str, format_spec?] -> [interpolation]
BuildInterpolation {
oparg: Arg<u32>,
} = 126,
Continue {
target: Arg<Label>,
} = 128,
@@ -425,7 +438,11 @@ impl TryFrom<u8> for Instruction {
u8::from(Self::BuildTupleFromTuples {
size: Arg::marker(),
}),
// 125, 126, 127 are unused
u8::from(Self::BuildTemplate),
u8::from(Self::BuildInterpolation {
oparg: Arg::marker(),
}),
// 127 is unused
u8::from(Self::Continue {
target: Arg::marker(),
}),
@@ -782,6 +799,14 @@ impl InstructionMetadata for Instruction {
Self::InstrumentedPopJumpIfNone => 0,
Self::InstrumentedPopJumpIfNotNone => 0,
Self::InstrumentedLine => 0,
// BuildTemplate: pops [strings_tuple, interpolations_tuple], pushes [template]
Self::BuildTemplate => -1,
// BuildInterpolation: pops [value, expr_str, format_spec?], pushes [interpolation]
// has_format_spec is bit 0 of oparg
Self::BuildInterpolation { oparg } => {
let has_format_spec = oparg.get(arg) & 1 != 0;
if has_format_spec { -2 } else { -1 }
}
}
}
@@ -976,6 +1001,8 @@ impl InstructionMetadata for Instruction {
Self::UnaryNot => w!(UNARY_NOT),
Self::YieldValue { arg } => w!(YIELD_VALUE, arg),
Self::GetYieldFromIter => w!(GET_YIELD_FROM_ITER),
Self::BuildTemplate => w!(BUILD_TEMPLATE),
Self::BuildInterpolation { oparg } => w!(BUILD_INTERPOLATION, oparg),
_ => w!(RUSTPYTHON_PLACEHOLDER),
}
}
+219
View File
@@ -0,0 +1,219 @@
use super::{PyStr, PyStrRef, PyTupleRef, PyType, tuple::IntoPyTuple};
use crate::{
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
class::PyClassImpl,
common::hash::PyHash,
convert::ToPyObject,
function::{OptionalArg, PyComparisonValue},
types::{Comparable, Constructor, Hashable, PyComparisonOp, Representable},
};
/// Interpolation object for t-strings (PEP 750).
///
/// Represents an interpolated expression within a template string.
#[pyclass(module = "string.templatelib", name = "Interpolation")]
#[derive(Debug, Clone)]
pub struct PyInterpolation {
pub value: PyObjectRef,
pub expression: PyStrRef,
pub conversion: PyObjectRef, // None or 's', 'r', 'a'
pub format_spec: PyStrRef,
}
impl PyPayload for PyInterpolation {
#[inline]
fn class(ctx: &Context) -> &'static Py<PyType> {
ctx.types.interpolation_type
}
}
impl PyInterpolation {
pub fn new(
value: PyObjectRef,
expression: PyStrRef,
conversion: PyObjectRef,
format_spec: PyStrRef,
vm: &VirtualMachine,
) -> PyResult<Self> {
// Validate conversion like _PyInterpolation_Build does
let is_valid = vm.is_none(&conversion)
|| conversion
.downcast_ref::<PyStr>()
.is_some_and(|s| matches!(s.as_str(), "s" | "r" | "a"));
if !is_valid {
return Err(vm.new_exception_msg(
vm.ctx.exceptions.system_error.to_owned(),
"Interpolation() argument 'conversion' must be one of 's', 'a' or 'r'".to_owned(),
));
}
Ok(Self {
value,
expression,
conversion,
format_spec,
})
}
}
impl Constructor for PyInterpolation {
type Args = InterpolationArgs;
fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> {
let conversion = match args.conversion {
OptionalArg::Present(c) => {
if vm.is_none(&c) {
vm.ctx.none()
} else {
let s = c.downcast::<PyStr>().map_err(|_| {
vm.new_type_error(
"Interpolation() argument 'conversion' must be str or None",
)
})?;
let s_str = s.as_str();
if s_str.len() != 1 || !matches!(s_str.chars().next(), Some('s' | 'r' | 'a')) {
return Err(vm.new_value_error(
"Interpolation() argument 'conversion' must be one of 's', 'a' or 'r'",
));
}
s.into()
}
}
OptionalArg::Missing => vm.ctx.none(),
};
let expression = args
.expression
.unwrap_or_else(|| vm.ctx.empty_str.to_owned());
let format_spec = args
.format_spec
.unwrap_or_else(|| vm.ctx.empty_str.to_owned());
Ok(PyInterpolation {
value: args.value,
expression,
conversion,
format_spec,
})
}
}
#[derive(FromArgs)]
pub struct InterpolationArgs {
#[pyarg(positional)]
value: PyObjectRef,
#[pyarg(any, optional)]
expression: OptionalArg<PyStrRef>,
#[pyarg(any, optional)]
conversion: OptionalArg<PyObjectRef>,
#[pyarg(any, optional)]
format_spec: OptionalArg<PyStrRef>,
}
#[pyclass(with(Constructor, Comparable, Hashable, Representable))]
impl PyInterpolation {
#[pyattr]
fn __match_args__(ctx: &Context) -> PyTupleRef {
ctx.new_tuple(vec![
ctx.intern_str("value").to_owned().into(),
ctx.intern_str("expression").to_owned().into(),
ctx.intern_str("conversion").to_owned().into(),
ctx.intern_str("format_spec").to_owned().into(),
])
}
#[pygetset]
fn value(&self) -> PyObjectRef {
self.value.clone()
}
#[pygetset]
fn expression(&self) -> PyStrRef {
self.expression.clone()
}
#[pygetset]
fn conversion(&self) -> PyObjectRef {
self.conversion.clone()
}
#[pygetset]
fn format_spec(&self) -> PyStrRef {
self.format_spec.clone()
}
#[pymethod]
fn __reduce__(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyTupleRef {
let cls = zelf.class().to_owned();
let args = (
zelf.value.clone(),
zelf.expression.clone(),
zelf.conversion.clone(),
zelf.format_spec.clone(),
);
(cls, args.to_pyobject(vm)).into_pytuple(vm)
}
}
impl Comparable for PyInterpolation {
fn cmp(
zelf: &Py<Self>,
other: &PyObject,
op: PyComparisonOp,
vm: &VirtualMachine,
) -> PyResult<PyComparisonValue> {
op.eq_only(|| {
let other = class_or_notimplemented!(Self, other);
let eq = vm.bool_eq(&zelf.value, &other.value)?
&& vm.bool_eq(zelf.expression.as_object(), other.expression.as_object())?
&& vm.bool_eq(&zelf.conversion, &other.conversion)?
&& vm.bool_eq(zelf.format_spec.as_object(), other.format_spec.as_object())?;
Ok(eq.into())
})
}
}
impl Hashable for PyInterpolation {
fn hash(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<PyHash> {
// Hash based on (value, expression, conversion, format_spec)
let value_hash = zelf.value.hash(vm)?;
let expr_hash = zelf.expression.as_object().hash(vm)?;
let conv_hash = zelf.conversion.hash(vm)?;
let spec_hash = zelf.format_spec.as_object().hash(vm)?;
// Combine hashes
Ok(value_hash
.wrapping_add(expr_hash.wrapping_mul(3))
.wrapping_add(conv_hash.wrapping_mul(5))
.wrapping_add(spec_hash.wrapping_mul(7)))
}
}
impl Representable for PyInterpolation {
#[inline]
fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {
let value_repr = zelf.value.repr(vm)?;
let expr_repr = zelf.expression.repr(vm)?;
let conv_str = if vm.is_none(&zelf.conversion) {
"None".to_owned()
} else {
zelf.conversion.repr(vm)?.as_str().to_owned()
};
let spec_repr = zelf.format_spec.repr(vm)?;
Ok(format!(
"Interpolation({}, {}, {}, {})",
value_repr.as_str(),
expr_repr.as_str(),
conv_str,
spec_repr.as_str()
))
}
}
pub fn init(context: &Context) {
PyInterpolation::extend_class(context, context.types.interpolation_type);
}
+4
View File
@@ -36,6 +36,8 @@ pub(crate) mod getset;
pub use getset::PyGetSet;
pub(crate) mod int;
pub use int::{PyInt, PyIntRef};
pub(crate) mod interpolation;
pub use interpolation::PyInterpolation;
pub(crate) mod iter;
pub use iter::*;
pub(crate) mod list;
@@ -76,6 +78,8 @@ pub(crate) mod slice;
pub use slice::{PyEllipsis, PySlice};
pub(crate) mod staticmethod;
pub use staticmethod::PyStaticMethod;
pub(crate) mod template;
pub use template::{PyTemplate, PyTemplateIter};
pub(crate) mod traceback;
pub use traceback::PyTraceback;
pub(crate) mod tuple;
+338
View File
@@ -0,0 +1,338 @@
use super::{PyStr, PyTupleRef, PyType};
use crate::{
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
atomic_func,
class::PyClassImpl,
function::{FuncArgs, PyComparisonValue},
protocol::{PyIterReturn, PySequenceMethods},
types::{
AsSequence, Comparable, Constructor, IterNext, Iterable, PyComparisonOp, Representable,
SelfIter,
},
};
use std::sync::LazyLock;
use super::interpolation::PyInterpolation;
/// Template object for t-strings (PEP 750).
///
/// Represents a template string with interpolated expressions.
#[pyclass(module = "string.templatelib", name = "Template")]
#[derive(Debug, Clone)]
pub struct PyTemplate {
pub strings: PyTupleRef,
pub interpolations: PyTupleRef,
}
impl PyPayload for PyTemplate {
#[inline]
fn class(ctx: &Context) -> &'static Py<PyType> {
ctx.types.template_type
}
}
impl PyTemplate {
pub fn new(strings: PyTupleRef, interpolations: PyTupleRef) -> Self {
Self {
strings,
interpolations,
}
}
}
impl Constructor for PyTemplate {
type Args = FuncArgs;
fn py_new(_cls: &Py<PyType>, args: Self::Args, vm: &VirtualMachine) -> PyResult<Self> {
if !args.kwargs.is_empty() {
return Err(vm.new_type_error("Template.__new__ only accepts *args arguments"));
}
let mut strings: Vec<PyObjectRef> = Vec::new();
let mut interpolations: Vec<PyObjectRef> = Vec::new();
let mut last_was_str = false;
for item in args.args.iter() {
if let Ok(s) = item.clone().downcast::<PyStr>() {
if last_was_str {
// Concatenate adjacent strings
if let Some(last) = strings.last_mut() {
let last_str = last.downcast_ref::<PyStr>().unwrap();
let concatenated = format!("{}{}", last_str.as_str(), s.as_str());
*last = vm.ctx.new_str(concatenated).into();
}
} else {
strings.push(s.into());
}
last_was_str = true;
} else if item.class().is(vm.ctx.types.interpolation_type) {
if !last_was_str {
// Add empty string before interpolation
strings.push(vm.ctx.empty_str.to_owned().into());
}
interpolations.push(item.clone());
last_was_str = false;
} else {
return Err(vm.new_type_error(format!(
"Template.__new__ *args need to be of type 'str' or 'Interpolation', got {}",
item.class().name()
)));
}
}
if !last_was_str {
// Add trailing empty string
strings.push(vm.ctx.empty_str.to_owned().into());
}
Ok(PyTemplate {
strings: vm.ctx.new_tuple(strings),
interpolations: vm.ctx.new_tuple(interpolations),
})
}
}
#[pyclass(with(Constructor, Comparable, Iterable, Representable, AsSequence))]
impl PyTemplate {
#[pygetset]
fn strings(&self) -> PyTupleRef {
self.strings.clone()
}
#[pygetset]
fn interpolations(&self) -> PyTupleRef {
self.interpolations.clone()
}
#[pygetset]
fn values(&self, vm: &VirtualMachine) -> PyTupleRef {
let values: Vec<PyObjectRef> = self
.interpolations
.iter()
.map(|interp| {
interp
.downcast_ref::<PyInterpolation>()
.map(|i| i.value.clone())
.unwrap_or_else(|| interp.clone())
})
.collect();
vm.ctx.new_tuple(values)
}
fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
let other = other.downcast_ref::<PyTemplate>().ok_or_else(|| {
vm.new_type_error(format!(
"can only concatenate Template (not '{}') to Template",
other.class().name()
))
})?;
// Concatenate the two templates
let mut new_strings: Vec<PyObjectRef> = Vec::new();
let mut new_interps: Vec<PyObjectRef> = Vec::new();
// Add all strings from self except the last one
let self_strings_len = self.strings.len();
for i in 0..self_strings_len.saturating_sub(1) {
new_strings.push(self.strings.get(i).unwrap().clone());
}
// Add all interpolations from self
for interp in self.interpolations.iter() {
new_interps.push(interp.clone());
}
// Concatenate last string of self with first string of other
let last_self = self
.strings
.get(self_strings_len.saturating_sub(1))
.and_then(|s| s.downcast_ref::<PyStr>().map(|s| s.as_str().to_owned()))
.unwrap_or_default();
let first_other = other
.strings
.first()
.and_then(|s| s.downcast_ref::<PyStr>().map(|s| s.as_str().to_owned()))
.unwrap_or_default();
let concatenated = format!("{}{}", last_self, first_other);
new_strings.push(vm.ctx.new_str(concatenated).into());
// Add remaining strings from other (skip first)
for i in 1..other.strings.len() {
new_strings.push(other.strings.get(i).unwrap().clone());
}
// Add all interpolations from other
for interp in other.interpolations.iter() {
new_interps.push(interp.clone());
}
let template = PyTemplate {
strings: vm.ctx.new_tuple(new_strings),
interpolations: vm.ctx.new_tuple(new_interps),
};
Ok(template.into_ref(&vm.ctx))
}
fn __add__(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
self.concat(&other, vm)
}
#[pymethod]
fn __reduce__(&self, vm: &VirtualMachine) -> PyResult<PyTupleRef> {
// Import string.templatelib._template_unpickle
// We need to import string first, then get templatelib from it,
// because import("string.templatelib", 0) with empty from_list returns the top-level module
let string_mod = vm.import("string.templatelib", 0)?;
let templatelib = string_mod.get_attr("templatelib", vm)?;
let unpickle_func = templatelib.get_attr("_template_unpickle", vm)?;
// Return (func, (strings, interpolations))
let args = vm.ctx.new_tuple(vec![
self.strings.clone().into(),
self.interpolations.clone().into(),
]);
Ok(vm.ctx.new_tuple(vec![unpickle_func, args.into()]))
}
}
impl AsSequence for PyTemplate {
fn as_sequence() -> &'static PySequenceMethods {
static AS_SEQUENCE: LazyLock<PySequenceMethods> = LazyLock::new(|| PySequenceMethods {
concat: atomic_func!(|seq, other, vm| {
let zelf = PyTemplate::sequence_downcast(seq);
zelf.concat(other, vm).map(|t| t.into())
}),
..PySequenceMethods::NOT_IMPLEMENTED
});
&AS_SEQUENCE
}
}
impl Comparable for PyTemplate {
fn cmp(
zelf: &Py<Self>,
other: &PyObject,
op: PyComparisonOp,
vm: &VirtualMachine,
) -> PyResult<PyComparisonValue> {
op.eq_only(|| {
let other = class_or_notimplemented!(Self, other);
let eq = vm.bool_eq(zelf.strings.as_object(), other.strings.as_object())?
&& vm.bool_eq(
zelf.interpolations.as_object(),
other.interpolations.as_object(),
)?;
Ok(eq.into())
})
}
}
impl Iterable for PyTemplate {
fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult {
Ok(PyTemplateIter::new(zelf).into_pyobject(vm))
}
}
impl Representable for PyTemplate {
#[inline]
fn repr_str(zelf: &Py<Self>, vm: &VirtualMachine) -> PyResult<String> {
let mut parts = Vec::new();
let strings_len = zelf.strings.len();
let interps_len = zelf.interpolations.len();
for i in 0..strings_len.max(interps_len * 2 + 1) {
if i % 2 == 0 {
// String position
let idx = i / 2;
if idx < strings_len {
let s = zelf.strings.get(idx).unwrap();
parts.push(s.repr(vm)?.as_str().to_owned());
}
} else {
// Interpolation position
let idx = i / 2;
if idx < interps_len {
let interp = zelf.interpolations.get(idx).unwrap();
parts.push(interp.repr(vm)?.as_str().to_owned());
}
}
}
Ok(format!("Template({})", parts.join(", ")))
}
}
/// Iterator for Template objects
#[pyclass(module = "string.templatelib", name = "TemplateIter")]
#[derive(Debug)]
pub struct PyTemplateIter {
template: PyRef<PyTemplate>,
index: std::sync::atomic::AtomicUsize,
from_strings: std::sync::atomic::AtomicBool,
}
impl PyPayload for PyTemplateIter {
#[inline]
fn class(ctx: &Context) -> &'static Py<PyType> {
ctx.types.template_iter_type
}
}
impl PyTemplateIter {
fn new(template: PyRef<PyTemplate>) -> Self {
Self {
template,
index: std::sync::atomic::AtomicUsize::new(0),
from_strings: std::sync::atomic::AtomicBool::new(true),
}
}
}
#[pyclass(with(IterNext, Iterable))]
impl PyTemplateIter {}
impl SelfIter for PyTemplateIter {}
impl IterNext for PyTemplateIter {
fn next(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyIterReturn> {
use std::sync::atomic::Ordering;
loop {
let from_strings = zelf.from_strings.load(Ordering::SeqCst);
let index = zelf.index.load(Ordering::SeqCst);
if from_strings {
if index < zelf.template.strings.len() {
let item = zelf.template.strings.get(index).unwrap();
zelf.from_strings.store(false, Ordering::SeqCst);
// Skip empty strings
if let Some(s) = item.downcast_ref::<PyStr>()
&& s.as_str().is_empty()
{
continue;
}
return Ok(PyIterReturn::Return(item.clone()));
} else {
return Ok(PyIterReturn::StopIteration(None));
}
} else if index < zelf.template.interpolations.len() {
let item = zelf.template.interpolations.get(index).unwrap();
zelf.index.fetch_add(1, Ordering::SeqCst);
zelf.from_strings.store(true, Ordering::SeqCst);
return Ok(PyIterReturn::Return(item.clone()));
} else {
return Ok(PyIterReturn::StopIteration(None));
}
}
}
}
pub fn init(context: &Context) {
PyTemplate::extend_class(context, context.types.template_type);
PyTemplateIter::extend_class(context, context.types.template_iter_type);
}
+52 -1
View File
@@ -2,7 +2,8 @@ use crate::{
AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine,
builtins::{
PyBaseException, PyBaseExceptionRef, PyCode, PyCoroutine, PyDict, PyDictRef, PyGenerator,
PyList, PySet, PySlice, PyStr, PyStrInterned, PyStrRef, PyTraceback, PyType,
PyInterpolation, PyList, PySet, PySlice, PyStr, PyStrInterned, PyStrRef, PyTemplate,
PyTraceback, PyType,
asyncgenerator::PyAsyncGenWrappedValue,
function::{PyCell, PyCellRef, PyFunction},
tuple::{PyTuple, PyTupleRef},
@@ -702,6 +703,56 @@ impl ExecutingFrame<'_> {
self.push_value(list_obj.into());
Ok(None)
}
Instruction::BuildTemplate => {
// Stack: [strings_tuple, interpolations_tuple] -> [template]
let interpolations = self.pop_value();
let strings = self.pop_value();
let strings = strings
.downcast::<PyTuple>()
.map_err(|_| vm.new_type_error("BUILD_TEMPLATE expected tuple for strings"))?;
let interpolations = interpolations.downcast::<PyTuple>().map_err(|_| {
vm.new_type_error("BUILD_TEMPLATE expected tuple for interpolations")
})?;
let template = PyTemplate::new(strings, interpolations);
self.push_value(template.into_pyobject(vm));
Ok(None)
}
Instruction::BuildInterpolation { oparg } => {
// oparg encoding: (conversion << 2) | has_format_spec
// Stack: [value, expression_str, (format_spec)?] -> [interpolation]
let oparg_val = oparg.get(arg);
let has_format_spec = (oparg_val & 1) != 0;
let conversion_code = oparg_val >> 2;
let format_spec = if has_format_spec {
self.pop_value().downcast::<PyStr>().map_err(|_| {
vm.new_type_error("BUILD_INTERPOLATION expected str for format_spec")
})?
} else {
vm.ctx.empty_str.to_owned()
};
let expression = self.pop_value().downcast::<PyStr>().map_err(|_| {
vm.new_type_error("BUILD_INTERPOLATION expected str for expression")
})?;
let value = self.pop_value();
// conversion: 0=None, 1=Str, 2=Repr, 3=Ascii
let conversion: PyObjectRef = match conversion_code {
0 => vm.ctx.none(),
1 => vm.ctx.new_str("s").into(),
2 => vm.ctx.new_str("r").into(),
3 => vm.ctx.new_str("a").into(),
_ => vm.ctx.none(), // should not happen
};
let interpolation =
PyInterpolation::new(value, expression, conversion, format_spec, vm)?;
self.push_value(interpolation.into_pyobject(vm));
Ok(None)
}
Instruction::Call { nargs } => {
// Stack: [callable, self_or_null, arg1, ..., argN]
let args = self.collect_positional_args(nargs.get(arg));
+1 -1
View File
@@ -36,7 +36,7 @@ impl Node for ruff::Expr {
Self::NumberLiteral(cons) => constant::number_literal_to_object(vm, source_file, cons),
Self::StringLiteral(cons) => constant::string_literal_to_object(vm, source_file, cons),
Self::FString(cons) => string::fstring_to_object(vm, source_file, cons),
Self::TString(_) => unimplemented!(),
Self::TString(cons) => string::tstring_to_object(vm, source_file, cons),
Self::BytesLiteral(cons) => constant::bytes_literal_to_object(vm, source_file, cons),
Self::BooleanLiteral(cons) => {
constant::boolean_literal_to_object(vm, source_file, cons)
+16
View File
@@ -448,6 +448,20 @@ impl_node!(
attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"],
);
impl_node!(
#[pyclass(module = "_ast", name = "TemplateStr", base = NodeExpr)]
pub(crate) struct NodeExprTemplateStr,
fields: ["values"],
attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"],
);
impl_node!(
#[pyclass(module = "_ast", name = "Interpolation", base = NodeExpr)]
pub(crate) struct NodeExprInterpolation,
fields: ["value", "str", "conversion", "format_spec"],
attributes: ["lineno", "col_offset", "end_lineno", "end_col_offset"],
);
impl_node!(
#[pyclass(module = "_ast", name = "Constant", base = NodeExpr)]
pub(crate) struct NodeExprConstant,
@@ -918,6 +932,8 @@ pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) {
"Call" => NodeExprCall::make_class(&vm.ctx),
"FormattedValue" => NodeExprFormattedValue::make_class(&vm.ctx),
"JoinedStr" => NodeExprJoinedStr::make_class(&vm.ctx),
"TemplateStr" => NodeExprTemplateStr::make_class(&vm.ctx),
"Interpolation" => NodeExprInterpolation::make_class(&vm.ctx),
"Constant" => NodeExprConstant::make_class(&vm.ctx),
"Attribute" => NodeExprAttribute::make_class(&vm.ctx),
"Subscript" => NodeExprSubscript::make_class(&vm.ctx),
+279 -52
View File
@@ -1,60 +1,20 @@
use super::constant::{Constant, ConstantLiteral};
use super::*;
fn ruff_fstring_value_into_iter(
mut fstring_value: ruff::FStringValue,
) -> impl Iterator<Item = ruff::FStringPart> + 'static {
let default = ruff::FStringPart::FString(ruff::FString {
node_index: Default::default(),
range: Default::default(),
elements: Default::default(),
flags: ruff::FStringFlags::empty(),
});
(0..fstring_value.as_slice().len()).map(move |i| {
let tmp = fstring_value.iter_mut().nth(i).unwrap();
core::mem::replace(tmp, default.clone())
})
}
fn ruff_fstring_element_into_iter(
mut fstring_element: ruff::InterpolatedStringElements,
) -> impl Iterator<Item = ruff::InterpolatedStringElement> + 'static {
) -> impl Iterator<Item = ruff::InterpolatedStringElement> {
let default =
ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement {
node_index: Default::default(),
range: Default::default(),
value: Default::default(),
});
(0..fstring_element.into_iter().len()).map(move |i| {
let fstring_element = &mut fstring_element;
let tmp = fstring_element.into_iter().nth(i).unwrap();
core::mem::replace(tmp, default.clone())
})
}
fn fstring_part_to_joined_str_part(fstring_part: ruff::FStringPart) -> Vec<JoinedStrPart> {
match fstring_part {
ruff::FStringPart::Literal(ruff::StringLiteral {
range,
value,
flags,
node_index: _,
}) => {
vec![JoinedStrPart::Constant(Constant::new_str(
value,
flags.prefix(),
range,
))]
}
ruff::FStringPart::FString(ruff::FString {
range: _,
elements,
flags: _, // TODO
node_index: _,
}) => ruff_fstring_element_into_iter(elements)
.map(ruff_fstring_element_to_joined_str_part)
.collect(),
}
fstring_element
.iter_mut()
.map(move |elem| core::mem::replace(elem, default.clone()))
.collect::<Vec<_>>()
.into_iter()
}
fn ruff_fstring_element_to_joined_str_part(
@@ -357,13 +317,280 @@ pub(super) fn fstring_to_object(
) -> PyObjectRef {
let ruff::ExprFString {
range,
value,
mut value,
node_index: _,
} = expression;
let values: Vec<_> = ruff_fstring_value_into_iter(value)
.flat_map(fstring_part_to_joined_str_part)
.collect();
let values = values.into_boxed_slice();
let c = JoinedStr { range, values };
let default_part = ruff::FStringPart::FString(ruff::FString {
node_index: Default::default(),
range: Default::default(),
elements: Default::default(),
flags: ruff::FStringFlags::empty(),
});
let mut values = Vec::new();
for i in 0..value.as_slice().len() {
let part = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_part.clone());
match part {
ruff::FStringPart::Literal(ruff::StringLiteral {
range,
value,
flags,
node_index: _,
}) => {
values.push(JoinedStrPart::Constant(Constant::new_str(
value,
flags.prefix(),
range,
)));
}
ruff::FStringPart::FString(ruff::FString {
range: _,
elements,
flags: _,
node_index: _,
}) => {
for element in ruff_fstring_element_into_iter(elements) {
values.push(ruff_fstring_element_to_joined_str_part(element));
}
}
}
}
let c = JoinedStr {
range,
values: values.into_boxed_slice(),
};
c.ast_to_object(vm, source_file)
}
// ===== TString (Template String) Support =====
fn ruff_tstring_element_to_template_str_part(
element: ruff::InterpolatedStringElement,
source_file: &SourceFile,
) -> TemplateStrPart {
match element {
ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement {
range,
value,
node_index: _,
}) => TemplateStrPart::Constant(Constant::new_str(
value,
ruff::str_prefix::StringLiteralPrefix::Empty,
range,
)),
ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement {
range,
expression,
debug_text,
conversion,
format_spec,
node_index: _,
}) => {
// Get the expression source text for the "str" field
let expr_str = debug_text
.map(|dt| dt.leading.to_string() + &dt.trailing)
.unwrap_or_else(|| source_file.slice(expression.range()).to_string());
TemplateStrPart::Interpolation(TStringInterpolation {
value: expression,
str: expr_str,
conversion,
format_spec: ruff_format_spec_to_template_str(format_spec, source_file),
range,
})
}
}
}
fn ruff_format_spec_to_template_str(
format_spec: Option<Box<ruff::InterpolatedStringFormatSpec>>,
source_file: &SourceFile,
) -> Option<Box<TemplateStr>> {
match format_spec {
None => None,
Some(format_spec) => {
let ruff::InterpolatedStringFormatSpec {
range,
elements,
node_index: _,
} = *format_spec;
let values: Vec<_> = ruff_fstring_element_into_iter(elements)
.map(|e| ruff_tstring_element_to_template_str_part(e, source_file))
.collect();
let values = values.into_boxed_slice();
Some(Box::new(TemplateStr { range, values }))
}
}
}
#[derive(Debug)]
pub(super) struct TemplateStr {
pub(super) range: TextRange,
pub(super) values: Box<[TemplateStrPart]>,
}
// constructor
impl Node for TemplateStr {
fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef {
let Self { values, range } = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeExprTemplateStr::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item(
"values",
BoxedSlice(values).ast_to_object(vm, source_file),
vm,
)
.unwrap();
node_add_location(&dict, range, vm, source_file);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_file: &SourceFile,
object: PyObjectRef,
) -> PyResult<Self> {
let values: BoxedSlice<_> = Node::ast_from_object(
vm,
source_file,
get_node_field(vm, &object, "values", "TemplateStr")?,
)?;
Ok(Self {
values: values.0,
range: range_from_object(vm, source_file, object, "TemplateStr")?,
})
}
}
#[derive(Debug)]
pub(super) enum TemplateStrPart {
Interpolation(TStringInterpolation),
Constant(Constant),
}
// constructor
impl Node for TemplateStrPart {
fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef {
match self {
Self::Interpolation(value) => value.ast_to_object(vm, source_file),
Self::Constant(value) => value.ast_to_object(vm, source_file),
}
}
fn ast_from_object(
vm: &VirtualMachine,
source_file: &SourceFile,
object: PyObjectRef,
) -> PyResult<Self> {
let cls = object.class();
if cls.is(pyast::NodeExprInterpolation::static_type()) {
Ok(Self::Interpolation(Node::ast_from_object(
vm,
source_file,
object,
)?))
} else {
Ok(Self::Constant(Node::ast_from_object(
vm,
source_file,
object,
)?))
}
}
}
#[derive(Debug)]
pub(super) struct TStringInterpolation {
value: Box<ruff::Expr>,
str: String,
conversion: ruff::ConversionFlag,
format_spec: Option<Box<TemplateStr>>,
range: TextRange,
}
// constructor
impl Node for TStringInterpolation {
fn ast_to_object(self, vm: &VirtualMachine, source_file: &SourceFile) -> PyObjectRef {
let Self {
value,
str,
conversion,
format_spec,
range,
} = self;
let node = NodeAst
.into_ref_with_type(vm, pyast::NodeExprInterpolation::static_type().to_owned())
.unwrap();
let dict = node.as_object().dict().unwrap();
dict.set_item("value", value.ast_to_object(vm, source_file), vm)
.unwrap();
dict.set_item("str", vm.ctx.new_str(str).into(), vm)
.unwrap();
dict.set_item("conversion", conversion.ast_to_object(vm, source_file), vm)
.unwrap();
dict.set_item(
"format_spec",
format_spec.ast_to_object(vm, source_file),
vm,
)
.unwrap();
node_add_location(&dict, range, vm, source_file);
node.into()
}
fn ast_from_object(
vm: &VirtualMachine,
source_file: &SourceFile,
object: PyObjectRef,
) -> PyResult<Self> {
let str_obj = get_node_field(vm, &object, "str", "Interpolation")?;
let str_val: String = str_obj.try_into_value(vm)?;
Ok(Self {
value: Node::ast_from_object(
vm,
source_file,
get_node_field(vm, &object, "value", "Interpolation")?,
)?,
str: str_val,
conversion: Node::ast_from_object(
vm,
source_file,
get_node_field(vm, &object, "conversion", "Interpolation")?,
)?,
format_spec: get_node_field_opt(vm, &object, "format_spec")?
.map(|obj| Node::ast_from_object(vm, source_file, obj))
.transpose()?,
range: range_from_object(vm, source_file, object, "Interpolation")?,
})
}
}
pub(super) fn tstring_to_object(
vm: &VirtualMachine,
source_file: &SourceFile,
expression: ruff::ExprTString,
) -> PyObjectRef {
let ruff::ExprTString {
range,
mut value,
node_index: _,
} = expression;
let default_tstring = ruff::TString {
node_index: Default::default(),
range: Default::default(),
elements: Default::default(),
flags: ruff::TStringFlags::empty(),
};
let mut values = Vec::new();
for i in 0..value.as_slice().len() {
let tstring = core::mem::replace(value.iter_mut().nth(i).unwrap(), default_tstring.clone());
for element in ruff_fstring_element_into_iter(tstring.elements) {
values.push(ruff_tstring_element_to_template_str_part(
element,
source_file,
));
}
}
let c = TemplateStr {
range,
values: values.into_boxed_slice(),
};
c.ast_to_object(vm, source_file)
}
+11 -3
View File
@@ -3,9 +3,9 @@ use crate::{
builtins::{
asyncgenerator, bool_, builtin_func, bytearray, bytes, classmethod, code, complex,
coroutine, descriptor, dict, enumerate, filter, float, frame, function, generator,
genericalias, getset, int, iter, list, map, mappingproxy, memory, module, namespace,
object, property, pystr, range, set, singletons, slice, staticmethod, super_, traceback,
tuple,
genericalias, getset, int, interpolation, iter, list, map, mappingproxy, memory, module,
namespace, object, property, pystr, range, set, singletons, slice, staticmethod, super_,
template, traceback, tuple,
type_::{self, PyType},
union_, weakproxy, weakref, zip,
},
@@ -93,6 +93,9 @@ pub struct TypeZoo {
pub not_implemented_type: &'static Py<PyType>,
pub generic_alias_type: &'static Py<PyType>,
pub union_type: &'static Py<PyType>,
pub interpolation_type: &'static Py<PyType>,
pub template_type: &'static Py<PyType>,
pub template_iter_type: &'static Py<PyType>,
pub member_descriptor_type: &'static Py<PyType>,
pub wrapper_descriptor_type: &'static Py<PyType>,
pub method_wrapper_type: &'static Py<PyType>,
@@ -188,6 +191,9 @@ impl TypeZoo {
not_implemented_type: singletons::PyNotImplemented::init_builtin_type(),
generic_alias_type: genericalias::PyGenericAlias::init_builtin_type(),
union_type: union_::PyUnion::init_builtin_type(),
interpolation_type: interpolation::PyInterpolation::init_builtin_type(),
template_type: template::PyTemplate::init_builtin_type(),
template_iter_type: template::PyTemplateIter::init_builtin_type(),
member_descriptor_type: descriptor::PyMemberDescriptor::init_builtin_type(),
wrapper_descriptor_type: descriptor::PyWrapper::init_builtin_type(),
method_wrapper_type: descriptor::PyMethodWrapper::init_builtin_type(),
@@ -243,6 +249,8 @@ impl TypeZoo {
traceback::init(context);
genericalias::init(context);
union_::init(context);
interpolation::init(context);
template::init(context);
descriptor::init(context);
crate::stdlib::typing::init(context);
}
+9 -4
View File
@@ -380,8 +380,10 @@ impl VirtualMachine {
if !result.is(&self.ctx.not_implemented) {
return Ok(result);
}
if let Ok(seq_a) = a.try_sequence(self) {
let result = seq_a.concat(b, self)?;
// Check if concat slot is available directly, matching PyNumber_Add behavior
let seq = a.sequence_unchecked();
if let Some(f) = seq.slots().concat.load() {
let result = f(seq, b, self)?;
if !result.is(&self.ctx.not_implemented) {
return Ok(result);
}
@@ -394,8 +396,11 @@ impl VirtualMachine {
if !result.is(&self.ctx.not_implemented) {
return Ok(result);
}
if let Ok(seq_a) = a.try_sequence(self) {
let result = seq_a.inplace_concat(b, self)?;
// Check inplace_concat or concat slot directly, matching PyNumber_InPlaceAdd behavior
let seq = a.sequence_unchecked();
let slots = seq.slots();
if let Some(f) = slots.inplace_concat.load().or_else(|| slots.concat.load()) {
let result = f(seq, b, self)?;
if !result.is(&self.ctx.not_implemented) {
return Ok(result);
}