pythongh-132261: Store annotations at hidden internal keys in the class dict (python#132345)

JelleZijlstra · web-flow · commit 07b8d3117fdb · 2025-04-10T21:13:26.000-07:00
diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst
@@ -303,12 +303,12 @@ Functions
 .. function:: get_annotate_function(obj)
 
    Retrieve the :term:`annotate function` for *obj*. Return :const:`!None`
-   if *obj* does not have an annotate function.
+   if *obj* does not have an annotate function. *obj* may be a class, function,
+   module, or a namespace dictionary for a class. The last case is useful during
+   class creation, e.g. in the ``__new__`` method of a metaclass.
 
    This is usually equivalent to accessing the :attr:`~object.__annotate__`
-   attribute of *obj*, but direct access to the attribute may return the wrong
-   object in certain situations involving metaclasses. This function should be
-   used instead of accessing the attribute directly.
+   attribute of *obj*, but access through this public function is preferred.
 
    .. versionadded:: 3.14
 
diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h
diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h
@@ -78,7 +78,9 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(__and__)
         STRUCT_FOR_ID(__anext__)
         STRUCT_FOR_ID(__annotate__)
+        STRUCT_FOR_ID(__annotate_func__)
         STRUCT_FOR_ID(__annotations__)
+        STRUCT_FOR_ID(__annotations_cache__)
         STRUCT_FOR_ID(__args__)
         STRUCT_FOR_ID(__await__)
         STRUCT_FOR_ID(__bases__)
diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h
@@ -274,6 +274,7 @@ Known values:
     Python 3.14a6 3619 (Renumber RESUME opcode from 149 to 128)
     Python 3.14a6 3620 (Optimize bytecode for all/any/tuple called on a genexp)
     Python 3.14a7 3621 (Optimize LOAD_FAST opcodes into LOAD_FAST_BORROW)
+    Python 3.14a7 3622 (Store annotations in different class dict keys)
 
     Python 3.15 will start with 3650
 
@@ -286,7 +287,7 @@ PC/launcher.c must also be updated.
 
 */
 
-#define PYC_MAGIC_NUMBER 3621
+#define PYC_MAGIC_NUMBER 3622
 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes
    (little-endian) and then appending b'\r\n'. */
 #define PYC_MAGIC_NUMBER_TOKEN \
diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h
diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h
diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py
@@ -619,14 +619,6 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False):
         raise ValueError(f"Invalid format: {format!r}")
 
 
-# We use the descriptors from builtins.type instead of accessing
-# .__annotations__ and .__annotate__ directly on class objects, because
-# otherwise we could get wrong results in some cases involving metaclasses.
-# See PEP 749.
-_BASE_GET_ANNOTATE = type.__dict__["__annotate__"].__get__
-_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__
-
-
 def get_annotate_function(obj):
     """Get the __annotate__ function for an object.
 
@@ -635,12 +627,11 @@ def get_annotate_function(obj):
 
     Returns the __annotate__ function or None.
     """
-    if isinstance(obj, type):
+    if isinstance(obj, dict):
         try:
-            return _BASE_GET_ANNOTATE(obj)
-        except AttributeError:
-            # AttributeError is raised for static types.
-            return None
+            return obj["__annotate__"]
+        except KeyError:
+            return obj.get("__annotate_func__", None)
     return getattr(obj, "__annotate__", None)
 
 
@@ -833,7 +824,7 @@ def _get_and_call_annotate(obj, format):
 def _get_dunder_annotations(obj):
     if isinstance(obj, type):
         try:
-            ann = _BASE_GET_ANNOTATIONS(obj)
+            ann = obj.__annotations__
         except AttributeError:
             # For static types, the descriptor raises AttributeError.
             return {}
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
@@ -330,7 +330,8 @@ def visiblename(name, all=None, obj=None):
                 '__date__', '__doc__', '__file__', '__spec__',
                 '__loader__', '__module__', '__name__', '__package__',
                 '__path__', '__qualname__', '__slots__', '__version__',
-                '__static_attributes__', '__firstlineno__'}:
+                '__static_attributes__', '__firstlineno__',
+                '__annotate_func__', '__annotations_cache__'}:
         return 0
     # Private names are hidden, but special names are displayed.
     if name.startswith('__') and name.endswith('__'): return 1
diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py
@@ -298,7 +298,7 @@ def test_arguments(self):
         x = ast.arguments()
         self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs',
                                      'kw_defaults', 'kwarg', 'defaults'))
-        self.assertEqual(x.__annotations__, {
+        self.assertEqual(ast.arguments.__annotations__, {
             'posonlyargs': list[ast.arg],
             'args': list[ast.arg],
             'vararg': ast.arg | None,
diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py
@@ -78,11 +78,6 @@ class A(builtins.object)
      |  __weakref__%s
 
     class B(builtins.object)
-     |  Methods defined here:
-     |
-     |  __annotate__(format, /)
-     |
-     |  ----------------------------------------------------------------------
      |  Data descriptors defined here:
      |
      |  __dict__%s
@@ -180,9 +175,6 @@ class A(builtins.object)
             list of weak references to the object
 
 class B(builtins.object)
-    Methods defined here:
-        __annotate__(format, /)
-    ----------------------------------------------------------------------
     Data descriptors defined here:
         __dict__
             dictionary for instance variables
diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py
@@ -4,32 +4,33 @@
 import types
 import unittest
 from test.support import run_code, check_syntax_error, cpython_only
+from test.test_inspect import inspect_stringized_annotations
 
 
 class TypeAnnotationTests(unittest.TestCase):
 
     def test_lazy_create_annotations(self):
         # type objects lazy create their __annotations__ dict on demand.
-        # the annotations dict is stored in type.__dict__.
+        # the annotations dict is stored in type.__dict__ (as __annotations_cache__).
         # a freshly created type shouldn't have an annotations dict yet.
         foo = type("Foo", (), {})
         for i in range(3):
-            self.assertFalse("__annotations__" in foo.__dict__)
+            self.assertFalse("__annotations_cache__" in foo.__dict__)
             d = foo.__annotations__
-            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertTrue("__annotations_cache__" in foo.__dict__)
             self.assertEqual(foo.__annotations__, d)
-            self.assertEqual(foo.__dict__['__annotations__'], d)
+            self.assertEqual(foo.__dict__['__annotations_cache__'], d)
             del foo.__annotations__
 
     def test_setting_annotations(self):
         foo = type("Foo", (), {})
         for i in range(3):
-            self.assertFalse("__annotations__" in foo.__dict__)
+            self.assertFalse("__annotations_cache__" in foo.__dict__)
             d = {'a': int}
             foo.__annotations__ = d
-            self.assertTrue("__annotations__" in foo.__dict__)
+            self.assertTrue("__annotations_cache__" in foo.__dict__)
             self.assertEqual(foo.__annotations__, d)
-            self.assertEqual(foo.__dict__['__annotations__'], d)
+            self.assertEqual(foo.__dict__['__annotations_cache__'], d)
             del foo.__annotations__
 
     def test_annotations_getset_raises(self):
@@ -53,9 +54,30 @@ class C:
             a:int=3
             b:str=4
         self.assertEqual(C.__annotations__, {"a": int, "b": str})
-        self.assertTrue("__annotations__" in C.__dict__)
+        self.assertTrue("__annotations_cache__" in C.__dict__)
         del C.__annotations__
-        self.assertFalse("__annotations__" in C.__dict__)
+        self.assertFalse("__annotations_cache__" in C.__dict__)
+
+    def test_pep563_annotations(self):
+        isa = inspect_stringized_annotations
+        self.assertEqual(
+            isa.__annotations__, {"a": "int", "b": "str"},
+        )
+        self.assertEqual(
+            isa.MyClass.__annotations__, {"a": "int", "b": "str"},
+        )
+
+    def test_explicitly_set_annotations(self):
+        class C:
+            __annotations__ = {"what": int}
+        self.assertEqual(C.__annotations__, {"what": int})
+
+    def test_explicitly_set_annotate(self):
+        class C:
+            __annotate__ = lambda format: {"what": int}
+        self.assertEqual(C.__annotations__, {"what": int})
+        self.assertIsInstance(C.__annotate__, types.FunctionType)
+        self.assertEqual(C.__annotate__(annotationlib.Format.VALUE), {"what": int})
 
     def test_del_annotations_and_annotate(self):
         # gh-132285
diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py
@@ -3825,6 +3825,7 @@ def meth(self): pass
         acceptable_extra_attrs = {
             '_is_protocol', '_is_runtime_protocol', '__parameters__',
             '__init__', '__annotations__', '__subclasshook__', '__annotate__',
+            '__annotations_cache__', '__annotate_func__',
         }
         self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs)
         self.assertLessEqual(
diff --git a/Lib/typing.py b/Lib/typing.py
@@ -1784,7 +1784,7 @@ class _TypingEllipsis:
     '__init__', '__module__', '__new__', '__slots__',
     '__subclasshook__', '__weakref__', '__class_getitem__',
     '__match_args__', '__static_attributes__', '__firstlineno__',
-    '__annotate__',
+    '__annotate__', '__annotate_func__', '__annotations_cache__',
 })
 
 # These special attributes will be not collected as protocol members.
@@ -2875,7 +2875,8 @@ def annotate(format):
                          '_fields', '_field_defaults',
                          '_make', '_replace', '_asdict', '_source'})
 
-_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__'})
+_special = frozenset({'__module__', '__name__', '__annotations__', '__annotate__',
+                      '__annotate_func__', '__annotations_cache__'})
 
 
 class NamedTupleMeta(type):
@@ -2893,8 +2894,7 @@ def __new__(cls, typename, bases, ns):
             types = ns["__annotations__"]
             field_names = list(types)
             annotate = _make_eager_annotate(types)
-        elif "__annotate__" in ns:
-            original_annotate = ns["__annotate__"]
+        elif (original_annotate := _lazy_annotationlib.get_annotate_function(ns)) is not None:
             types = _lazy_annotationlib.call_annotate_function(
                 original_annotate, _lazy_annotationlib.Format.FORWARDREF)
             field_names = list(types)
@@ -3080,8 +3080,7 @@ def __new__(cls, name, bases, ns, total=True):
         if "__annotations__" in ns:
             own_annotate = None
             own_annotations = ns["__annotations__"]
-        elif "__annotate__" in ns:
-            own_annotate = ns["__annotate__"]
+        elif (own_annotate := _lazy_annotationlib.get_annotate_function(ns)) is not None:
             own_annotations = _lazy_annotationlib.call_annotate_function(
                 own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict
             )
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-09-21-51-37.gh-issue-132261.gL8thm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-09-21-51-37.gh-issue-132261.gL8thm.rst
@@ -0,0 +1,4 @@
+The internal storage for annotations and annotate functions on classes now
+uses different keys in the class dictionary. This eliminates various edge
+cases where access to the ``__annotate__`` and ``__annotations__``
+attributes would behave unpredictably.
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
@@ -1915,10 +1915,17 @@ type_get_annotate(PyObject *tp, void *Py_UNUSED(closure))
 
     PyObject *annotate;
     PyObject *dict = PyType_GetDict(type);
+    // First try __annotate__, in case that's been set explicitly
     if (PyDict_GetItemRef(dict, &_Py_ID(__annotate__), &annotate) < 0) {
         Py_DECREF(dict);
         return NULL;
     }
+    if (!annotate) {
+        if (PyDict_GetItemRef(dict, &_Py_ID(__annotate_func__), &annotate) < 0) {
+            Py_DECREF(dict);
+            return NULL;
+        }
+    }
     if (annotate) {
         descrgetfunc get = Py_TYPE(annotate)->tp_descr_get;
         if (get) {
@@ -1927,7 +1934,7 @@ type_get_annotate(PyObject *tp, void *Py_UNUSED(closure))
     }
     else {
         annotate = Py_None;
-        int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), annotate);
+        int result = PyDict_SetItem(dict, &_Py_ID(__annotate_func__), annotate);
         if (result < 0) {
             Py_DECREF(dict);
             return NULL;
@@ -1959,13 +1966,13 @@ type_set_annotate(PyObject *tp, PyObject *value, void *Py_UNUSED(closure))
 
     PyObject *dict = PyType_GetDict(type);
     assert(PyDict_Check(dict));
-    int result = PyDict_SetItem(dict, &_Py_ID(__annotate__), value);
+    int result = PyDict_SetItem(dict, &_Py_ID(__annotate_func__), value);
     if (result < 0) {
         Py_DECREF(dict);
         return -1;
     }
     if (!Py_IsNone(value)) {
-        if (PyDict_Pop(dict, &_Py_ID(__annotations__), NULL) == -1) {
+        if (PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL) == -1) {
             Py_DECREF(dict);
             PyType_Modified(type);
             return -1;
@@ -1987,18 +1994,26 @@ type_get_annotations(PyObject *tp, void *Py_UNUSED(closure))
 
     PyObject *annotations;
     PyObject *dict = PyType_GetDict(type);
+    // First try __annotations__ (e.g. for "from __future__ import annotations")
     if (PyDict_GetItemRef(dict, &_Py_ID(__annotations__), &annotations) < 0) {
         Py_DECREF(dict);
         return NULL;
     }
+    if (!annotations) {
+        if (PyDict_GetItemRef(dict, &_Py_ID(__annotations_cache__), &annotations) < 0) {
+            Py_DECREF(dict);
+            return NULL;
+        }
+    }
+
     if (annotations) {
         descrgetfunc get = Py_TYPE(annotations)->tp_descr_get;
         if (get) {
             Py_SETREF(annotations, get(annotations, NULL, tp));
         }
     }
     else {
-        PyObject *annotate = type_get_annotate(tp, NULL);
+        PyObject *annotate = PyObject_GetAttrString((PyObject *)type, "__annotate__");
         if (annotate == NULL) {
             Py_DECREF(dict);
             return NULL;
@@ -2026,7 +2041,7 @@ type_get_annotations(PyObject *tp, void *Py_UNUSED(closure))
         Py_DECREF(annotate);
         if (annotations) {
             int result = PyDict_SetItem(
-                    dict, &_Py_ID(__annotations__), annotations);
+                    dict, &_Py_ID(__annotations_cache__), annotations);
             if (result) {
                 Py_CLEAR(annotations);
             } else {
@@ -2053,10 +2068,10 @@ type_set_annotations(PyObject *tp, PyObject *value, void *Py_UNUSED(closure))
     PyObject *dict = PyType_GetDict(type);
     if (value != NULL) {
         /* set */
-        result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value);
+        result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value);
     } else {
         /* delete */
-        result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL);
+        result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL);
         if (result == 0) {
             PyErr_SetString(PyExc_AttributeError, "__annotations__");
             Py_DECREF(dict);
@@ -2067,6 +2082,11 @@ type_set_annotations(PyObject *tp, PyObject *value, void *Py_UNUSED(closure))
         Py_DECREF(dict);
         return -1;
     } else {  // result can be 0 or 1
+        if (PyDict_Pop(dict, &_Py_ID(__annotate_func__), NULL) < 0) {
+            PyType_Modified(type);
+            Py_DECREF(dict);
+            return -1;
+        }
         if (PyDict_Pop(dict, &_Py_ID(__annotate__), NULL) < 0) {
             PyType_Modified(type);
             Py_DECREF(dict);
diff --git a/Python/codegen.c b/Python/codegen.c
