From 20bce4875bec0afb6ec8278ef932e0b486aa61fd Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 27 Oct 2026 16:50:03 -0700 Subject: [PATCH] [ty] Reuse runtime operands in eager union annotations (#29770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checking an eager annotation such as `A | B | C` can involve looking up the same name twice. Ty needs both the type that the annotation denotes and the runtime value on which Python calls `|`. In longer annotations, it can also evaluate an inner union again while checking its parent. This change reuses the runtime values of bare names, `None`, and nested unions within the same eager annotation. A nested union computes its runtime value only when the enclosing `|` can use it under the same inference flags. Each expression still records its annotation type. For example, `bool | int` denotes `int` but evaluates to a union object, which matters for a subsequent `| None`. This addresses the eager-annotation case from #29312 on top of #29727. ## Test plan Measured CPU time for fresh, single-core `ty check` processes over 12 interleaved rounds. The comparison uses main at `18c5d27c41` and the eager operand changes from #29312 rebased onto the same commit. Times are medians in milliseconds for each whole input file. | Input | Main | Rebased #29312 | This change | | --- | ---: | ---: | ---: | | Three ordinary classes, 1,500 signatures | 58.5 | 59.3 | 56.6 | | Twelve ordinary classes, 450 signatures | 86.7 | 72.5 | 69.4 | | Twelve specialized generics, 350 signatures | 144.5 | 100.3 | 100.0 | | Direct `TypeForm` arguments | 113.9 | 98.3 | 91.7 | The source-only checks of NumPy’s `_typing` package and Pydantic showed no clear difference. Against rebased #29312, the stored 256-member `TypeForm` case used 1.0% more CPU time, and the `gradual_literal_union_equality` Criterion benchmark took 0.9% longer. Compared the inferred types of all 14 AST nodes in a pair of left- and right-nested annotations. Both alternatives and main yielded the same type at every node. --- .../resources/mdtest/annotations/union.md | 53 +++ .../resources/mdtest/type_form.md | 7 + .../types/infer/builder/binary_expressions.rs | 15 +- .../types/infer/builder/type_expression.rs | 384 +++++++++++------- 4 files changed, 307 insertions(+), 152 deletions(-) diff --git a/crates/ty_python_semantic/resources/mdtest/annotations/union.md b/crates/ty_python_semantic/resources/mdtest/annotations/union.md index 9e3e261386..4a232ceac8 100644 --- a/crates/ty_python_semantic/resources/mdtest/annotations/union.md +++ b/crates/ty_python_semantic/resources/mdtest/annotations/union.md @@ -138,6 +138,59 @@ def check(value: WithLiteral): isinstance(value, WithLiteral) # error: [invalid-argument-type] ``` +## Runtime operators in eager annotations + +```toml +[environment] +python-version = "3.12" +``` + +### Nested union objects + +`bool | int` denotes `int`, but evaluates to a union object that accepts `None` on either side. + +```py +def f(left: bool | int | None | bytes, right: bytes | (None | (bool | int))): + reveal_type(left) # revealed: int | None | bytes + reveal_type(right) # revealed: bytes | None | int +``` + +### Custom metaclass operators + +A metaclass can give `|` a different result. Here `C | "int"` evaluates to a string, so applying +`| None` to that result fails. + +```py +class Meta(type): + def __or__(self, other: str) -> str: + return other + +class C(metaclass=Meta): ... + +# error: 29 [unsupported-operator] "Operator `|` is unsupported between objects of type `str` and `None`" +def invalid(value: bytes | (C | "int" | None)): + reveal_type(value) # revealed: bytes | C | int | None +``` + +### Dictionary merges + +Even in an invalid annotation, merging a dictionary literal with a `TypedDict` instance produces +that `TypedDict`. The next `|` reports its runtime type. + +```py +from typing import TypedDict + +class Row(TypedDict): + key: int + +row: Row = {"key": 1} + +# error: [invalid-type-form] "Dict literals" +# error: [invalid-type-form] "Variable of type `Row`" +# error: [unsupported-operator] "Operator `|` is unsupported between objects of type `Row` and `None`" +def invalid(value: {} | row | None | bytes): ... +``` + ## Recursive assignments to invalid union operands An invalid operation still produces a diagnostic when its operand is inferred recursively: diff --git a/crates/ty_python_semantic/resources/mdtest/type_form.md b/crates/ty_python_semantic/resources/mdtest/type_form.md index 0d251b6e34..d1af276bf7 100644 --- a/crates/ty_python_semantic/resources/mdtest/type_form.md +++ b/crates/ty_python_semantic/resources/mdtest/type_form.md @@ -497,6 +497,13 @@ reveal_type(foo(str)) # revealed: str foo(float) # error: [no-matching-overload] ``` +An invalid union operator still produces a diagnostic when the argument matches an overload. + +```py +# error: [unsupported-operator] +reveal_type(foo("bool" | "int" | bool)) # revealed: int +``` + ## Narrowing to runtime classes A `TypeForm[T]` value may or may not be a runtime class object. An `isinstance(..., type)` check can diff --git a/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs b/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs index bd0ddb1dd9..322aafa814 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/binary_expressions.rs @@ -179,12 +179,23 @@ impl<'db> TypeInferenceBuilder<'db, '_> { BinaryExpressionOperandTypes::Inferred(left_ty, right_ty) => (left_ty, right_ty), }; + self.infer_binary_expression_from_types(binary, left_ty, right_ty) + } + + /// Evaluate a binary operator whose operands have already been inferred in their type contexts. + pub(super) fn infer_binary_expression_from_types( + &mut self, + binary: &ast::ExprBinOp, + left_ty: Type<'db>, + right_ty: Type<'db>, + ) -> Type<'db> { + let op = binary.op; let mut state = BinaryInferenceState::default(); let return_type = - self.infer_binary_expression_type(binary.into(), left_ty, right_ty, *op, &mut state); + self.infer_binary_expression_type(binary.into(), left_ty, right_ty, op, &mut state); self.report_deprecated_functions(binary, state.deprecated_functions); return_type.unwrap_or_else(|| { - report_unsupported_binary_operation(&self.context, binary, left_ty, right_ty, *op); + report_unsupported_binary_operation(&self.context, binary, left_ty, right_ty, op); Type::unknown() }) } diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index 1436f735ce..8e3b28fc4a 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -41,6 +41,13 @@ use crate::types::{ }; use crate::{FxOrderSet, SemanticModel, add_inferred_python_version_hint_to_diagnostic}; +/// The type an annotation denotes and, when valid in the enclosing `|`'s context, its runtime value. +/// For example, `bool | int` denotes `int` but evaluates to a union object. +struct InferredUnionOperand<'db> { + annotation: Type<'db>, + runtime_value: Option>, +} + enum LegacyTypeAliasReference<'db> { Alias(Type<'db>, Option>), /// The definition is invalid as a type alias, and its use has been rejected. @@ -193,6 +200,14 @@ impl<'db> TypeInferenceBuilder<'db, '_> { /// Infer the type of a type expression. pub(super) fn infer_type_expression(&mut self, expression: &ast::Expr) -> Type<'db> { + let ty = self.with_type_expression_context(|builder| { + builder.infer_type_expression_no_store(expression) + }); + self.store_expression_type(expression, ty); + ty + } + + fn with_type_expression_context(&mut self, infer: impl FnOnce(&mut Self) -> R) -> R { let previous_deferred_state = self.deferred_state; let was_in_type_expression = self .inference_flags() @@ -214,7 +229,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { self.replace_deferred_state(DeferredExpressionState::Deferred); } - let ty = self.infer_type_expression_no_store(expression); + let result = infer(self); self.deferred_state = previous_deferred_state; self.context.inference_flags.set( InferenceFlags::IN_NESTED_TYPE_EXPRESSION, @@ -224,8 +239,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { InferenceFlags::IN_TYPE_EXPRESSION, previously_in_type_expression, ); - self.store_expression_type(expression, ty); - ty + result } /// Similar to [`infer_type_expression`], but accepts a [`DeferredExpressionState`]. @@ -413,17 +427,18 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } } + fn should_check_runtime_errors(&self, expression: impl Ranged) -> bool { + !self.deferred_state.is_deferred() + && !self.in_stub() + && !self.is_in_type_checking_block(self.scope(), expression) + } + /// Infer the type of a type expression without storing the result. pub(super) fn infer_type_expression_no_store(&mut self, expression: &ast::Expr) -> Type<'db> { let db = self.db(); let env = self.program_environment(); - let ignore_runtime_errors = |builder: &Self| { - builder.deferred_state.is_deferred() - || builder.in_stub() - || builder.is_in_type_checking_block(builder.scope(), expression) - }; let ignore_experimental_runtime_errors = |builder: &Self| { - ignore_runtime_errors(builder) + !builder.should_check_runtime_errors(expression) || matches!(builder.scope.scope(db).kind(), ScopeKind::TypeAlias) }; @@ -529,148 +544,14 @@ impl<'db> TypeInferenceBuilder<'db, '_> { ast::Expr::BinOp(binary) => { match binary.op { // PEP-604 unions are okay, e.g., `int | str` + ast::Operator::BitOr if self.should_check_runtime_errors(binary) => { + self.infer_eager_union_type_expression(binary, false) + .annotation + } ast::Operator::BitOr => { - let left_ty = self.infer_type_expression(&binary.left); - let right_ty = self.infer_type_expression(&binary.right); - - // Detect runtime errors from e.g. `int | "bytes"` on Python <3.14 without `__future__` annotations. - if !ignore_runtime_errors(self) { - let mut speculative_builder = self.speculate_without_diagnostics(); - // If the left-hand side of the union is itself a PEP-604 union, - // we'll already have checked whether it can be used with `|` in a previous inference step - // and emitted a diagnostic if it was appropriate. We should skip inferring it here to - // avoid duplicate diagnostics; just assume that the l.h.s. is a `UnionType` instance - // in that case. - let left_type_value = speculative_builder - .infer_expression(&binary.left, TypeContext::default()); - let right_type_value = speculative_builder - .infer_expression(&binary.right, TypeContext::default()); - - let dunder_fails = Type::try_call_bin_op( - db, - env, - left_type_value, - ast::Operator::BitOr, - right_type_value, - ) - .is_err(); - - // As well as trying the normal dunder lookup, - // we also check for the case where one of the operands is a class-literal type - // or generic-alias type and the other is a string literal. The normal dunder lookup - // fails to catch this error, since typeshed annotates `type.__(r)or__` as accepting `Any`. - // ABCMeta and _ProtocolMeta inherit these operators unchanged. The typeshed - // protocol fallback does not establish a custom operator either. - let should_emit_error = if dunder_fails { - true - } else { - let literal = match (left_type_value, right_type_value) { - (Type::ClassLiteral(class), Type::LiteralValue(literal)) - | (Type::LiteralValue(literal), Type::ClassLiteral(class)) - if matches!( - class - .inferred_metaclass(db) - .for_inheritance(db, env) - .to_class_type(db) - .and_then(|metaclass| metaclass.known(db)), - Some( - KnownClass::Type - | KnownClass::ABCMeta - | KnownClass::ProtocolMeta - ) - ) => - { - Some(literal) - } - (Type::GenericAlias(_), Type::LiteralValue(literal)) - | (Type::LiteralValue(literal), Type::GenericAlias(_)) => { - Some(literal) - } - _ => None, - }; - literal.is_some_and(|literal| !literal.is_enum()) - }; - - if should_emit_error - && let Some(builder) = - self.context.report_lint(&UNSUPPORTED_OPERATOR, binary) - { - let mut diagnostic = - builder.into_diagnostic("Unsupported `|` operation"); - - if left_type_value.is_equivalent_to(db, env, right_type_value) { - diagnostic.set_primary_annotation_message(format_args!( - "Both operands have type `{}`", - left_type_value.display(db, env) - )); - diagnostic.set_concise_message(format_args!( - "Operator `|` is unsupported between \ - two objects of type `{}`", - left_type_value.display(db, env) - )); - } else { - for (operand, ty) in [ - (&*binary.left, left_type_value), - (&*binary.right, right_type_value), - ] { - diagnostic.annotate( - self.context.secondary(operand).message(format_args!( - "Has type `{}`", - ty.display(db, env) - )), - ); - } - diagnostic.set_concise_message(format_args!( - "Operator `|` is unsupported between \ - objects of type `{}` and `{}`", - left_type_value.display(db, env), - right_type_value.display(db, env) - )); - } - - match self.scope.scope(self.db()).kind() { - ScopeKind::TypeAlias => diagnostic.info( - "A type alias scope is lazy but will be \ - executed at runtime if the `__value__` property is \ - accessed", - ), - ScopeKind::TypeParams => diagnostic.info( - "Type parameter scopes are lazy but may be \ - executed at runtime if the `__bound__`, `__value__` - or `__constraints__` property of a type parameter is \ - accessed", - ), - _ => { - let python_version = - self.program_environment().python_version(db); - - if python_version < PythonVersion::PY314 { - diagnostic.info(format_args!( - "All {}s are evaluated at \ - runtime by default on Python <3.14", - self.type_expression_context() - )); - add_inferred_python_version_hint_to_diagnostic( - db, - self.file(), - &mut diagnostic, - "inferring types", - ); - if binary.left.is_string_literal_expr() - || binary.right.is_string_literal_expr() - { - diagnostic.help( - "Put quotes around the whole union \ - rather than just certain elements", - ); - } - } - } - } - } - } - - UnionType::from_elements_leave_aliases(db, env, [left_ty, right_ty]) + let left = self.infer_type_expression(&binary.left); + let right = self.infer_type_expression(&binary.right); + UnionType::from_elements_leave_aliases(db, env, [left, right]) } ast::Operator::BitAnd => { if let Some(builder) = @@ -1374,6 +1255,209 @@ impl<'db> TypeInferenceBuilder<'db, '_> { } } + fn infer_eager_union_operand(&mut self, expr: &ast::Expr) -> InferredUnionOperand<'db> { + // Overload resolution can share expression inference that also needs diagnostics. + if self.expression_cache.is_some() || self.cycle_recovery.is_some() { + return InferredUnionOperand { + annotation: self.infer_type_expression(expr), + runtime_value: None, + }; + } + + let parent_flags = self.inference_flags(); + let operand = self.with_type_expression_context(|builder| match expr { + ast::Expr::BinOp(binary) if binary.op == ast::Operator::BitOr => { + // Entering a nested type expression can change how type arguments are interpreted. + builder.infer_eager_union_type_expression( + binary, + builder.inference_flags() == parent_flags, + ) + } + ast::Expr::Name(name) if name.ctx.is_load() => { + // A bare name has the same runtime value under either set of flags. + let (value, definition) = builder.infer_type_expression_reference(expr); + InferredUnionOperand { + annotation: builder + .infer_name_or_attribute_type_expression(value, definition, expr), + runtime_value: Some(value), + } + } + _ => { + let annotation = builder.infer_type_expression_no_store(expr); + InferredUnionOperand { + annotation, + runtime_value: expr.is_none_literal_expr().then_some(annotation), + } + } + }); + self.store_expression_type(expr, operand.annotation); + operand + } + + fn infer_eager_union_type_expression( + &mut self, + binary: &ast::ExprBinOp, + need_runtime_value: bool, + ) -> InferredUnionOperand<'db> { + let db = self.db(); + let env = self.program_environment(); + let left = self.infer_eager_union_operand(&binary.left); + let right = self.infer_eager_union_operand(&binary.right); + let mut speculative = self.speculate_without_diagnostics(); + let mut operand_value = |expr: &ast::Expr, value: Option>| { + value + .filter(|_| speculative.cycle_recovery.is_none()) + .unwrap_or_else(|| speculative.infer_expression(expr, TypeContext::default())) + }; + let left_value = operand_value(&binary.left, left.runtime_value); + let right_value = operand_value(&binary.right, right.runtime_value); + self.check_union_operator(binary, left_value, right_value); + + // Ordinary value inference can use the TypedDict on one side of `|` to infer + // a dictionary literal on the other, even when the annotation is invalid. + let runtime_value = if need_runtime_value + && speculative.cycle_recovery.is_none() + && !binary.left.is_dict_expr() + && !binary.right.is_dict_expr() + { + let value = + speculative.infer_binary_expression_from_types(binary, left_value, right_value); + speculative.cycle_recovery.is_none().then_some(value) + } else { + None + }; + InferredUnionOperand { + annotation: UnionType::from_elements_leave_aliases( + db, + env, + [left.annotation, right.annotation], + ), + runtime_value, + } + } + + fn check_union_operator( + &self, + binary: &ast::ExprBinOp, + left_type_value: Type<'db>, + right_type_value: Type<'db>, + ) { + let db = self.db(); + let env = self.program_environment(); + let dunder_fails = Type::try_call_bin_op( + db, + env, + left_type_value, + ast::Operator::BitOr, + right_type_value, + ) + .is_err(); + + // As well as trying the normal dunder lookup, + // we also check for the case where one of the operands is a class-literal type + // or generic-alias type and the other is a string literal. The normal dunder lookup + // fails to catch this error, since typeshed annotates `type.__(r)or__` as accepting `Any`. + // ABCMeta and _ProtocolMeta inherit these operators unchanged. The typeshed + // protocol fallback does not establish a custom operator either. + let should_emit_error = if dunder_fails { + true + } else { + let literal = match (left_type_value, right_type_value) { + (Type::ClassLiteral(class), Type::LiteralValue(literal)) + | (Type::LiteralValue(literal), Type::ClassLiteral(class)) + if matches!( + class + .inferred_metaclass(db) + .for_inheritance(db, env) + .to_class_type(db) + .and_then(|metaclass| metaclass.known(db)), + Some(KnownClass::Type | KnownClass::ABCMeta | KnownClass::ProtocolMeta) + ) => + { + Some(literal) + } + (Type::GenericAlias(_), Type::LiteralValue(literal)) + | (Type::LiteralValue(literal), Type::GenericAlias(_)) => Some(literal), + _ => None, + }; + literal.is_some_and(|literal| !literal.is_enum()) + }; + + if should_emit_error + && let Some(builder) = self.context.report_lint(&UNSUPPORTED_OPERATOR, binary) + { + let mut diagnostic = builder.into_diagnostic("Unsupported `|` operation"); + + if left_type_value.is_equivalent_to(db, env, right_type_value) { + diagnostic.set_primary_annotation_message(format_args!( + "Both operands have type `{}`", + left_type_value.display(db, env) + )); + diagnostic.set_concise_message(format_args!( + "Operator `|` is unsupported between \ + two objects of type `{}`", + left_type_value.display(db, env) + )); + } else { + for (operand, ty) in [ + (&*binary.left, left_type_value), + (&*binary.right, right_type_value), + ] { + diagnostic.annotate( + self.context + .secondary(operand) + .message(format_args!("Has type `{}`", ty.display(db, env))), + ); + } + diagnostic.set_concise_message(format_args!( + "Operator `|` is unsupported between \ + objects of type `{}` and `{}`", + left_type_value.display(db, env), + right_type_value.display(db, env) + )); + } + + match self.scope.scope(self.db()).kind() { + ScopeKind::TypeAlias => diagnostic.info( + "A type alias scope is lazy but will be \ + executed at runtime if the `__value__` property is \ + accessed", + ), + ScopeKind::TypeParams => diagnostic.info( + "Type parameter scopes are lazy but may be \ + executed at runtime if the `__bound__`, `__value__` + or `__constraints__` property of a type parameter is \ + accessed", + ), + _ => { + let python_version = self.program_environment().python_version(db); + + if python_version < PythonVersion::PY314 { + diagnostic.info(format_args!( + "All {}s are evaluated at \ + runtime by default on Python <3.14", + self.type_expression_context() + )); + add_inferred_python_version_hint_to_diagnostic( + db, + self.file(), + &mut diagnostic, + "inferring types", + ); + if binary.left.is_string_literal_expr() + || binary.right.is_string_literal_expr() + { + diagnostic.help( + "Put quotes around the whole union \ + rather than just certain elements", + ); + } + } + } + } + } + } + fn infer_starred_type_expression(&mut self, starred: &ast::ExprStarred) -> Type<'db> { let db = self.db(); let ast::ExprStarred {