[ty] typing.Self is bound by the method, not the class (#19784)

This fixes our logic for binding a legacy typevar with its binding
context. (To recap, a legacy typevar starts out "unbound" when it is
first created, and each time it's used in a generic class or function,
we "bind" it with the corresponding `Definition`.)

We treat `typing.Self` the same as a legacy typevar, and so we apply
this binding logic to it too. Before, we were using the enclosing class
as its binding context. But that's not correct — it's the method where
`typing.Self` is used that binds the typevar. (Each invocation of the
method will find a new specialization of `Self` based on the specific
instance type containing the invoked method.)

This required plumbing through some additional state to the
`in_type_expression` method.

This also revealed that we weren't handling `Self`-typed instance
attributes correctly (but were coincidentally not getting the expected
false positive diagnostics).

Author: dcreager

crates/ty_ide/src/completion.rs

@@ -1247,7 +1247,7 @@ quux.<CURSOR>
         __init_subclass__ :: bound method Quux.__init_subclass__() -> None
         __module__ :: str
         __ne__ :: bound method Quux.__ne__(value: object, /) -> bool
-        __new__ :: bound method Quux.__new__() -> Self@object
+        __new__ :: bound method Quux.__new__() -> Self@__new__
         __reduce__ :: bound method Quux.__reduce__() -> str | tuple[Any, ...]
         __reduce_ex__ :: bound method Quux.__reduce_ex__(protocol: SupportsIndex, /) -> str | tuple[Any, ...]
         __repr__ :: bound method Quux.__repr__() -> str
@@ -1292,7 +1292,7 @@ quux.b<CURSOR>
         __init_subclass__ :: bound method Quux.__init_subclass__() -> None
         __module__ :: str
         __ne__ :: bound method Quux.__ne__(value: object, /) -> bool
-        __new__ :: bound method Quux.__new__() -> Self@object
+        __new__ :: bound method Quux.__new__() -> Self@__new__
         __reduce__ :: bound method Quux.__reduce__() -> str | tuple[Any, ...]
         __reduce_ex__ :: bound method Quux.__reduce_ex__(protocol: SupportsIndex, /) -> str | tuple[Any, ...]
         __repr__ :: bound method Quux.__repr__() -> str
@@ -1346,7 +1346,7 @@ C.<CURSOR>
         __mro__ :: tuple[<class 'C'>, <class 'object'>]
         __name__ :: str
         __ne__ :: def __ne__(self, value: object, /) -> bool
-        __new__ :: def __new__(cls) -> Self@object
+        __new__ :: def __new__(cls) -> Self@__new__
         __or__ :: bound method <class 'C'>.__or__(value: Any, /) -> UnionType
         __prepare__ :: bound method <class 'Meta'>.__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object]
         __qualname__ :: str
@@ -1522,7 +1522,7 @@ Quux.<CURSOR>
         __mro__ :: tuple[<class 'Quux'>, <class 'object'>]
         __name__ :: str
         __ne__ :: def __ne__(self, value: object, /) -> bool
-        __new__ :: def __new__(cls) -> Self@object
+        __new__ :: def __new__(cls) -> Self@__new__
         __or__ :: bound method <class 'Quux'>.__or__(value: Any, /) -> UnionType
         __prepare__ :: bound method <class 'type'>.__prepare__(name: str, bases: tuple[type, ...], /, **kwds: Any) -> MutableMapping[str, object]
         __qualname__ :: str
@@ -1574,8 +1574,8 @@ Answer.<CURSOR>
                 __bool__ :: bound method <class 'Answer'>.__bool__() -> Literal[True]
                 __class__ :: <class 'EnumMeta'>
                 __contains__ :: bound method <class 'Answer'>.__contains__(value: object) -> bool
-                __copy__ :: def __copy__(self) -> Self@Enum
-                __deepcopy__ :: def __deepcopy__(self, memo: Any) -> Self@Enum
+                __copy__ :: def __copy__(self) -> Self@__copy__
+                __deepcopy__ :: def __deepcopy__(self, memo: Any) -> Self@__deepcopy__
                 __delattr__ :: def __delattr__(self, name: str, /) -> None
                 __dict__ :: MappingProxyType[str, Any]
                 __dictoffset__ :: int
@@ -1599,7 +1599,7 @@ Answer.<CURSOR>
                 __mro__ :: tuple[<class 'Answer'>, <class 'Enum'>, <class 'object'>]
                 __name__ :: str
                 __ne__ :: def __ne__(self, value: object, /) -> bool
-                __new__ :: def __new__(cls, value: object) -> Self@Enum
+                __new__ :: def __new__(cls, value: object) -> Self@__new__
                 __or__ :: bound method <class 'Answer'>.__or__(value: Any, /) -> UnionType
                 __order__ :: str
                 __prepare__ :: bound method <class 'EnumMeta'>.__prepare__(cls: str, bases: tuple[type, ...], **kwds: Any) -> _EnumDict

crates/ty_python_semantic/resources/mdtest/annotations/self.md

@@ -16,18 +16,25 @@ from typing import Self
 
 class Shape:
     def set_scale(self: Self, scale: float) -> Self:
-        reveal_type(self)  # revealed: Self@Shape
+        reveal_type(self)  # revealed: Self@set_scale
         return self
 
     def nested_type(self: Self) -> list[Self]:
         return [self]
 
     def nested_func(self: Self) -> Self:
         def inner() -> Self:
-            reveal_type(self)  # revealed: Self@Shape
+            reveal_type(self)  # revealed: Self@nested_func
             return self
         return inner()
 
+    def nested_func_without_enclosing_binding(self):
+        def inner(x: Self):
+            # TODO: revealed: Self@nested_func_without_enclosing_binding
+            # (The outer method binds an implicit `Self`)
+            reveal_type(x)  # revealed: Self@inner
+        inner(self)
+
     def implicit_self(self) -> Self:
         # TODO: first argument in a method should be considered as "typing.Self"
         reveal_type(self)  # revealed: Unknown
@@ -38,13 +45,13 @@ reveal_type(Shape().nested_func())  # revealed: Shape
 
 class Circle(Shape):
     def set_scale(self: Self, scale: float) -> Self:
-        reveal_type(self)  # revealed: Self@Circle
+        reveal_type(self)  # revealed: Self@set_scale
         return self
 
 class Outer:
     class Inner:
         def foo(self: Self) -> Self:
-            reveal_type(self)  # revealed: Self@Inner
+            reveal_type(self)  # revealed: Self@foo
             return self
 ```
 
@@ -99,6 +106,9 @@ reveal_type(Shape.bar())  # revealed: Unknown
 python-version = "3.11"
 ```
 
+TODO: The use of `Self` to annotate the `next_node` attribute should be
+[modeled as a property][self attribute], using `Self` in its parameter and return type.
+
 ```py
 from typing import Self
 
@@ -108,6 +118,8 @@ class LinkedList:
 
     def next(self: Self) -> Self:
         reveal_type(self.value)  # revealed: int
+        # TODO: no error
+        # error: [invalid-return-type]
         return self.next_node
 
 reveal_type(LinkedList().next())  # revealed: LinkedList
@@ -151,7 +163,7 @@ from typing import Self
 
 class Shape:
     def union(self: Self, other: Self | None):
-        reveal_type(other)  # revealed: Self@Shape | None
+        reveal_type(other)  # revealed: Self@union | None
         return self
 ```
 
@@ -205,3 +217,5 @@ class MyMetaclass(type):
     def __new__(cls) -> Self:
         return super().__new__(cls)
 ```
+
+[self attribute]: https://typing.python.org/en/latest/spec/generics.html#use-in-attribute-annotations

crates/ty_python_semantic/resources/mdtest/annotations/unsupported_special_forms.md

@@ -27,7 +27,7 @@ def i(callback: Callable[Concatenate[int, P], R_co], *args: P.args, **kwargs: P.
 
 class Foo:
     def method(self, x: Self):
-        reveal_type(x)  # revealed: Self@Foo
+        reveal_type(x)  # revealed: Self@method
 ```
 
 ## Type expressions

crates/ty_python_semantic/resources/mdtest/generics/legacy/functions.md

@@ -365,3 +365,33 @@ def g(x: T) -> T | None:
 reveal_type(f(g("a")))  # revealed: tuple[Literal["a"] | None, int]
 reveal_type(g(f("a")))  # revealed: tuple[Literal["a"], int] | None
 ```
+
+## Opaque decorators don't affect typevar binding
+
+Inside the body of a generic function, we should be able to see that the typevars bound by that
+function are in fact bound by that function. This requires being able to see the enclosing
+function's _undecorated_ type and signature, especially in the case where a gradually typed
+decorator "hides" the function type from outside callers.
+
+```py
+from typing import cast, Any, Callable, TypeVar
+
+F = TypeVar("F", bound=Callable[..., Any])
+T = TypeVar("T")
+
+def opaque_decorator(f: Any) -> Any:
+    return f
+
+def transparent_decorator(f: F) -> F:
+    return f
+
+@opaque_decorator
+def decorated(t: T) -> None:
+    # error: [redundant-cast]
+    reveal_type(cast(T, t))  # revealed: T@decorated
+
+@transparent_decorator
+def decorated(t: T) -> None:
+    # error: [redundant-cast]
+    reveal_type(cast(T, t))  # revealed: T@decorated
+```

crates/ty_python_semantic/resources/mdtest/generics/pep695/functions.md

@@ -377,3 +377,30 @@ def f(
     # error: [invalid-argument-type] "does not satisfy upper bound"
     reveal_type(close_and_return(g))  # revealed: Unknown
 ```
+
+## Opaque decorators don't affect typevar binding
+
+Inside the body of a generic function, we should be able to see that the typevars bound by that
+function are in fact bound by that function. This requires being able to see the enclosing
+function's _undecorated_ type and signature, especially in the case where a gradually typed
+decorator "hides" the function type from outside callers.
+
+```py
+from typing import cast, Any, Callable
+
+def opaque_decorator(f: Any) -> Any:
+    return f
+
+def transparent_decorator[F: Callable[..., Any]](f: F) -> F:
+    return f
+
+@opaque_decorator
+def decorated[T](t: T) -> None:
+    # error: [redundant-cast]
+    reveal_type(cast(T, t))  # revealed: T@decorated
+
+@transparent_decorator
+def decorated[T](t: T) -> None:
+    # error: [redundant-cast]
+    reveal_type(cast(T, t))  # revealed: T@decorated
+```

crates/ty_python_semantic/resources/mdtest/named_tuple.md

@@ -150,9 +150,9 @@ class Person(NamedTuple):
 
 reveal_type(Person._field_defaults)  # revealed: dict[str, Any]
 reveal_type(Person._fields)  # revealed: tuple[str, ...]
-reveal_type(Person._make)  # revealed: bound method <class 'Person'>._make(iterable: Iterable[Any]) -> Self@NamedTupleFallback
+reveal_type(Person._make)  # revealed: bound method <class 'Person'>._make(iterable: Iterable[Any]) -> Self@_make
 reveal_type(Person._asdict)  # revealed: def _asdict(self) -> dict[str, Any]
-reveal_type(Person._replace)  # revealed: def _replace(self, **kwargs: Any) -> Self@NamedTupleFallback
+reveal_type(Person._replace)  # revealed: def _replace(self, **kwargs: Any) -> Self@_replace
 
 # TODO: should be `Person` once we support `Self`
 reveal_type(Person._make(("Alice", 42)))  # revealed: Unknown

crates/ty_python_semantic/src/types.rs

@@ -47,8 +47,8 @@ use crate::types::function::{
     DataclassTransformerParams, FunctionSpans, FunctionType, KnownFunction,
 };
 use crate::types::generics::{
-    GenericContext, PartialSpecialization, Specialization, walk_generic_context,
-    walk_partial_specialization, walk_specialization,
+    GenericContext, PartialSpecialization, Specialization, bind_legacy_typevar,
+    walk_generic_context, walk_partial_specialization, walk_specialization,
 };
 pub use crate::types::ide_support::{
     CallSignatureDetails, Member, all_members, call_signature_details, definition_kind_for_name,
@@ -5268,12 +5268,13 @@ impl<'db> Type<'db> {
     /// expression, it names the type `Type::NominalInstance(builtins.int)`, that is, all objects whose
     /// `__class__` is `int`.
     ///
-    /// The `scope_id` argument must always be a scope from the file we are currently inferring, so
+    /// The `scope_id` and `legacy_typevar_binding_context` arguments must always come from the file we are currently inferring, so
     /// as to avoid cross-module AST dependency.
     pub(crate) fn in_type_expression(
         &self,
         db: &'db dyn Db,
         scope_id: ScopeId<'db>,
+        legacy_typevar_binding_context: Option<Definition<'db>>,
     ) -> Result<Type<'db>, InvalidTypeExpressionError<'db>> {
         match self {
             // Special cases for `float` and `complex`
@@ -5402,16 +5403,26 @@ impl<'db> Type<'db> {
                         "nearest_enclosing_class must return type that can be instantiated",
                     );
                     let class_definition = class.definition(db);
-                    Ok(Type::TypeVar(TypeVarInstance::new(
+                    let typevar = TypeVarInstance::new(
                         db,
                         ast::name::Name::new_static("Self"),
                         Some(class_definition),
-                        Some(class_definition),
+                        None,
                         Some(TypeVarBoundOrConstraints::UpperBound(instance)),
                         TypeVarVariance::Invariant,
                         None,
                         TypeVarKind::Implicit,
-                    )))
+                    );
+                    let typevar = bind_legacy_typevar(
+                        db,
+                        &module,
+                        index,
+                        scope_id.file_scope_id(db),
+                        legacy_typevar_binding_context,
+                        typevar,
+                    )
+                    .unwrap_or(typevar);
+                    Ok(Type::TypeVar(typevar))
                 }
                 SpecialFormType::TypeAlias => Ok(Type::Dynamic(DynamicType::TodoTypeAlias)),
                 SpecialFormType::TypedDict => Err(InvalidTypeExpressionError {
@@ -5486,7 +5497,7 @@ impl<'db> Type<'db> {
                 let mut builder = UnionBuilder::new(db);
                 let mut invalid_expressions = smallvec::SmallVec::default();
                 for element in union.elements(db) {
-                    match element.in_type_expression(db, scope_id) {
+                    match element.in_type_expression(db, scope_id, legacy_typevar_binding_context) {
                         Ok(type_expr) => builder = builder.add(type_expr),
                         Err(InvalidTypeExpressionError {
                             fallback_type,
@@ -6660,7 +6671,7 @@ impl<'db> InvalidTypeExpression<'db> {
             return;
         };
         if module_member_with_same_name
-            .in_type_expression(db, scope)
+            .in_type_expression(db, scope, None)
             .is_err()
         {
             return;

crates/ty_python_semantic/src/types/generics.rs

@@ -9,6 +9,7 @@ use crate::semantic_index::scope::{FileScopeId, NodeWithScopeKind};
 use crate::semantic_index::{SemanticIndex, semantic_index};
 use crate::types::class::ClassType;
 use crate::types::class_base::ClassBase;
+use crate::types::infer::infer_definition_types;
 use crate::types::instance::{NominalInstanceType, Protocol, ProtocolInstanceType};
 use crate::types::signatures::{Parameter, Parameters, Signature};
 use crate::types::tuple::{TupleSpec, TupleType};
@@ -20,7 +21,7 @@ use crate::{Db, FxOrderSet};
 
 /// Returns an iterator of any generic context introduced by the given scope or any enclosing
 /// scope.
-pub(crate) fn enclosing_generic_contexts<'db>(
+fn enclosing_generic_contexts<'db>(
     db: &'db dyn Db,
     module: &ParsedModuleRef,
     index: &SemanticIndex<'db>,
@@ -35,7 +36,9 @@ pub(crate) fn enclosing_generic_contexts<'db>(
                     .generic_context(db)
             }
             NodeWithScopeKind::Function(function) => {
-                binding_type(db, index.expect_single_definition(function.node(module)))
+                infer_definition_types(db, index.expect_single_definition(function.node(module)))
+                    .undecorated_type()
+                    .expect("function should have undecorated type")
                     .into_function_literal()?
                     .signature(db)
                     .iter()
@@ -59,6 +62,37 @@ fn bound_legacy_typevars<'db>(
         .filter(|typevar| typevar.is_legacy(db))
 }
 
+/// Binds an unbound legacy typevar.
+///
+/// When a legacy typevar is first created, we will have a [`TypeVarInstance`] which does not have
+/// an associated binding context. When the typevar is used in a generic class or function, we
+/// "bind" it, adding the [`Definition`] of the generic class or function as its "binding context".
+///
+/// When an expression resolves to a legacy typevar, our inferred type will refer to the unbound
+/// [`TypeVarInstance`] from when the typevar was first created. This function walks the scopes
+/// that enclosing the expression, looking for the innermost binding context that binds the
+/// typevar.
+///
+/// If no enclosing scope has already bound the typevar, we might be in a syntactic position that
+/// is about to bind it (indicated by a non-`None` `legacy_typevar_binding_context`), in which case
+/// we bind the typevar with that new binding context.
+pub(crate) fn bind_legacy_typevar<'db>(
+    db: &'db dyn Db,
+    module: &ParsedModuleRef,
+    index: &SemanticIndex<'db>,
+    containing_scope: FileScopeId,
+    legacy_typevar_binding_context: Option<Definition<'db>>,
+    typevar: TypeVarInstance<'db>,
+) -> Option<TypeVarInstance<'db>> {
+    enclosing_generic_contexts(db, module, index, containing_scope)
+        .find_map(|enclosing_context| enclosing_context.binds_legacy_typevar(db, typevar))
+        .or_else(|| {
+            legacy_typevar_binding_context.map(|legacy_typevar_binding_context| {
+                typevar.with_binding_context(db, legacy_typevar_binding_context)
+            })
+        })
+}
+
 /// A list of formal type variables for a generic function, class, or type alias.
 ///
 /// TODO: Handle nested generic contexts better, with actual parent links to the lexically
@@ -259,12 +293,13 @@ impl<'db> GenericContext<'db> {
         db: &'db dyn Db,
         typevar: TypeVarInstance<'db>,
     ) -> Option<TypeVarInstance<'db>> {
-        assert!(typevar.is_legacy(db));
+        assert!(typevar.is_legacy(db) || typevar.is_implicit(db));
         let typevar_def = typevar.definition(db);
         self.variables(db)
             .iter()
             .find(|self_typevar| {
-                self_typevar.is_legacy(db) && self_typevar.definition(db) == typevar_def
+                (self_typevar.is_legacy(db) || self_typevar.is_implicit(db))
+                    && self_typevar.definition(db) == typevar_def
             })
             .copied()
     }