mirrors/cpython
0
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2024-11-23 01:45:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

GH-115776: Embed the values array into the object, for "normal" Python objects. (GH-116115)

Browse Source
This commit is contained in:
Mark Shannon 2024-04-02 11:59:21 +01:00 committed by GitHub
parent c97d3af239
commit c32dc47aca
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
35 changed files with 787 additions and 537 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Include/cpython/pystats.h
View File

@ -77,12 +77,11 @@ typedef struct _object_stats {
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
uint64_t new_values;
uint64_t inline_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
uint64_t dict_dematerialized;
uint64_t type_cache_hits;
uint64_t type_cache_misses;
uint64_t type_cache_dunder_hits;

5
Include/internal/pycore_code.h
View File

@ -79,7 +79,10 @@ typedef struct {
typedef struct {
uint16_t counter;
uint16_t type_version[2];
uint16_t keys_version[2];
union {
uint16_t keys_version[2];
uint16_t dict_offset;
};
uint16_t descr[4];
} _PyLoadMethodCache;

64
Include/internal/pycore_dict.h
View File

@ -11,7 +11,7 @@ extern "C" {
#include "pycore_freelist.h" // _PyFreeListState
#include "pycore_identifier.h" // _Py_Identifier
#include "pycore_object.h" // PyDictOrValues
#include "pycore_object.h" // PyManagedDictPointer
// Unsafe flavor of PyDict_GetItemWithError(): no error checking
extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
@ -181,6 +181,10 @@ struct _dictkeysobject {
* [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
*/
struct _dictvalues {
uint8_t capacity;
uint8_t size;
uint8_t embedded;
uint8_t valid;
PyObject *values[1];
};
@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
size_t index = (size_t)1 << dk->dk_log2_index_bytes;
return (&indices[index]);
}
static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
assert(dk->dk_kind == DICT_KEYS_GENERAL);
return (PyDictKeyEntry*)_DK_ENTRIES(dk);
@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
#ifdef Py_GIL_DISABLED
#define DICT_NEXT_VERSION(INTERP) \
(_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
}
extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
PyAPI_FUNC(PyObject *)_PyDict_FromItems(
PyObject *const *keys, Py_ssize_t keys_offset,
PyObject *const *values, Py_ssize_t values_offset,
Py_ssize_t length);
static inline uint8_t *
get_insertion_order_array(PyDictValues *values)
{
return (uint8_t *)&values->values[values->capacity];
}
static inline void
_PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
{
assert(ix < SHARED_KEYS_MAX_SIZE);
uint8_t *size_ptr = ((uint8_t *)values)-2;
int size = *size_ptr;
assert(size+2 < DICT_VALUES_SIZE(values));
size++;
size_ptr[-size] = (uint8_t)ix;
*size_ptr = size;
int size = values->size;
uint8_t *array = get_insertion_order_array(values);
assert(size < values->capacity);
assert(((uint8_t)ix) == ix);
array[size] = (uint8_t)ix;
values->size = size+1;
}
static inline size_t
shared_keys_usable_size(PyDictKeysObject *keys)
{
#ifdef Py_GIL_DISABLED
// dk_usable will decrease for each instance that is created and each
// value that is added. dk_nentries will increase for each value that
// is added. We want to always return the right value or larger.
// We therefore increase dk_nentries first and we decrease dk_usable
// second, and conversely here we read dk_usable first and dk_entries
// second (to avoid the case where we read entries before the increment
// and read usable after the decrement)
return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
_Py_atomic_load_ssize_acquire(&keys->dk_nentries));
#else
return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
#endif
}
static inline size_t
_PyInlineValuesSize(PyTypeObject *tp)
{
PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
assert(keys != NULL);
size_t size = shared_keys_usable_size(keys);
size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
assert(prefix_size < 256);
return prefix_size + (size + 1) * sizeof(PyObject *);
}
int
_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
#ifdef __cplusplus
}
#endif

48
Include/internal/pycore_object.h
View File

@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
{
assert(op != NULL);
Py_SET_TYPE(op, typeobj);
if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) {
Py_INCREF(typeobj);
}
assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
Py_INCREF(typeobj);
_Py_NewReference(op);
}
@ -611,8 +610,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
extern int _PyObject_InitializeDict(PyObject *obj);
int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
PyObject *name, PyObject *value);
PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
#endif
typedef union {
PyObject *dict;
/* Use a char* to generate a warning if directly assigning a PyDictValues */
char *values;
} PyDictOrValues;
PyDictObject *dict;
} PyManagedDictPointer;
static inline PyDictOrValues *
_PyObject_DictOrValuesPointer(PyObject *obj)
static inline PyManagedDictPointer *
_PyObject_ManagedDictPointer(PyObject *obj)
{
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET);
}
static inline int
_PyDictOrValues_IsValues(PyDictOrValues dorv)
{
return ((uintptr_t)dorv.values) & 1;
return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
}
static inline PyDictValues *
_PyDictOrValues_GetValues(PyDictOrValues dorv)
_PyObject_InlineValues(PyObject *obj)
{
assert(_PyDictOrValues_IsValues(dorv));
return (PyDictValues *)(dorv.values + 1);
}
static inline PyObject *
_PyDictOrValues_GetDict(PyDictOrValues dorv)
{
assert(!_PyDictOrValues_IsValues(dorv));
return dorv.dict;
}
static inline void
_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values)
{
ptr->values = ((char *)values) - 1;
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject));
return (PyDictValues *)((char *)obj + sizeof(PyObject));
}
extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
extern int _PyObject_IsInstanceDictEmpty(PyObject *);
// Export for 'math' shared extension

6
Include/internal/pycore_opcode_metadata.h generated
View File

@ -1085,7 +1085,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = {
[LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG },
[LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG },
[LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
[LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG | HAS_PURE_FLAG },
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG },
@ -1269,7 +1269,7 @@ _PyOpcode_macro_expansion[256] = {
[LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, 0, 0 } } },
[LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } },
[LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } },
[LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 0, 0 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
[LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } },
[LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } },
[LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, 0, 0 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } },
[LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } },
@ -1316,7 +1316,7 @@ _PyOpcode_macro_expansion[256] = {
[SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, 0, 0 } } },
[SET_UPDATE] = { .nuops = 1, .uops = { { _SET_UPDATE, 0, 0 } } },
[STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, 0, 0 } } },
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
[STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_NO_DICT, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } },
[STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } },
[STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, 0, 0 } } },
[STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, 0, 0 } } },

2
Include/internal/pycore_uop_ids.h generated
View File

@ -109,7 +109,7 @@ extern "C" {
#define _GUARD_BOTH_INT 347
#define _GUARD_BOTH_UNICODE 348
#define _GUARD_BUILTINS_VERSION 349
#define _GUARD_DORV_VALUES 350
#define _GUARD_DORV_NO_DICT 350
#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 351
#define _GUARD_GLOBALS_VERSION 352
#define _GUARD_IS_FALSE_POP 353

10
Include/internal/pycore_uop_metadata.h generated
View File

@ -136,8 +136,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_ATTR_INSTANCE_VALUE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG,
[_CHECK_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG,
[_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG | HAS_PASSTHROUGH_FLAG,
[_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG,
[_CHECK_ATTR_WITH_HINT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG,
[_LOAD_ATTR_SLOT_0] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_SLOT_1] = HAS_DEOPT_FLAG,
[_LOAD_ATTR_SLOT] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_OPARG_AND_1_FLAG,
@ -145,7 +145,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
[_LOAD_ATTR_CLASS_0] = 0,
[_LOAD_ATTR_CLASS_1] = 0,
[_LOAD_ATTR_CLASS] = HAS_ARG_FLAG | HAS_OPARG_AND_1_FLAG,
[_GUARD_DORV_VALUES] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_GUARD_DORV_NO_DICT] = HAS_DEOPT_FLAG | HAS_PASSTHROUGH_FLAG,
[_STORE_ATTR_INSTANCE_VALUE] = 0,
[_STORE_ATTR_SLOT] = HAS_ESCAPES_FLAG,
[_COMPARE_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG,
@ -342,7 +342,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
[_GUARD_BOTH_INT] = "_GUARD_BOTH_INT",
[_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE",
[_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION",
[_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES",
[_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT",
[_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT",
[_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION",
[_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP",
@ -736,7 +736,7 @@ int _PyUop_num_popped(int opcode, int oparg)
return 1;
case _LOAD_ATTR_CLASS:
return 1;
case _GUARD_DORV_VALUES:
case _GUARD_DORV_NO_DICT:
return 1;
case _STORE_ATTR_INSTANCE_VALUE:
return 2;

7
Include/object.h
View File

@ -629,13 +629,18 @@ given type object has a specified feature.
/* Track types initialized using _PyStaticType_InitBuiltin(). */
#define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1)
/* The values array is placed inline directly after the rest of
* the object. Implies Py_TPFLAGS_HAVE_GC.
*/
#define Py_TPFLAGS_INLINE_VALUES (1 << 2)
/* Placement of weakref pointers are managed by the VM, not by the type.
* The VM will automatically set tp_weaklistoffset.
*/
#define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3)
/* Placement of dict (and values) pointers are managed by the VM, not by the type.
* The VM will automatically set tp_dictoffset.
* The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC.
*/
#define Py_TPFLAGS_MANAGED_DICT (1 << 4)

4
Lib/test/test_capi/test_mem.py
View File

@ -148,8 +148,8 @@ class PyMemDebugTests(unittest.TestCase):
self.assertIn(b'MemoryError', out)
*_, count = line.split(b' ')
count = int(count)
self.assertLessEqual(count, i*5)
self.assertGreaterEqual(count, i*5-2)
self.assertLessEqual(count, i*10)
self.assertGreaterEqual(count, i*10-4)
# Py_GIL_DISABLED requires mimalloc (not malloc)

77
Lib/test/test_class.py
View File

@ -2,7 +2,6 @@
import unittest
testmeths = [
# Binary operations
@ -789,5 +788,81 @@ class ClassTests(unittest.TestCase):
self.assertEqual(calls, 100)
from _testinternalcapi import has_inline_values
Py_TPFLAGS_MANAGED_DICT = (1 << 2)
class Plain:
pass
class WithAttrs:
def __init__(self):
self.a = 1
self.b = 2
self.c = 3
self.d = 4
class TestInlineValues(unittest.TestCase):
def test_flags(self):
self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT)
self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT)
def test_has_inline_values(self):
c = Plain()
self.assertTrue(has_inline_values(c))
del c.__dict__
self.assertFalse(has_inline_values(c))
def test_instances(self):
self.assertTrue(has_inline_values(Plain()))
self.assertTrue(has_inline_values(WithAttrs()))
def test_inspect_dict(self):
for cls in (Plain, WithAttrs):
c = cls()
c.__dict__
self.assertTrue(has_inline_values(c))
def test_update_dict(self):
d = { "e": 5, "f": 6 }
for cls in (Plain, WithAttrs):
c = cls()
c.__dict__.update(d)
self.assertTrue(has_inline_values(c))
@staticmethod
def set_100(obj):
for i in range(100):
setattr(obj, f"a{i}", i)
def check_100(self, obj):
for i in range(100):
self.assertEqual(getattr(obj, f"a{i}"), i)
def test_many_attributes(self):
class C: pass
c = C()
self.assertTrue(has_inline_values(c))
self.set_100(c)
self.assertFalse(has_inline_values(c))
self.check_100(c)
c = C()
self.assertTrue(has_inline_values(c))
def test_many_attributes_with_dict(self):
class C: pass
c = C()
d = c.__dict__
self.assertTrue(has_inline_values(c))
self.set_100(c)
self.assertFalse(has_inline_values(c))
self.check_100(c)
if __name__ == '__main__':
unittest.main()

13
Lib/test/test_opcache.py
View File

@ -1044,20 +1044,13 @@ class TestInstanceDict(unittest.TestCase):
c.a = 1
c.b = 2
c.__dict__
self.assertIs(
_testinternalcapi.get_object_dict_values(c),
None
)
self.assertEqual(c.__dict__, {"a":1, "b": 2})
def test_dict_dematerialization(self):
c = C()
c.a = 1
c.b = 2
c.__dict__
self.assertIs(
_testinternalcapi.get_object_dict_values(c),
None
)
for _ in range(100):
c.a
self.assertEqual(
@ -1072,10 +1065,6 @@ class TestInstanceDict(unittest.TestCase):
d = c.__dict__
for _ in range(100):
c.a
self.assertIs(
_testinternalcapi.get_object_dict_values(c),
None
)
self.assertIs(c.__dict__, d)
def test_dict_dematerialization_copy(self):

4
Misc/NEWS.d/next/Core and Builtins/2024-02-24-03-39-09.gh-issue-115776.THJXqg.rst Normal file
View File

@ -0,0 +1,4 @@
The array of values, the ``PyDictValues`` struct is now embedded in the
object during allocation. This provides better performance in the common
case, and does not degrade as much when the object's ``__dict__`` is
materialized.

3
Modules/_testbuffer.c
View File

@ -2820,6 +2820,9 @@ static int
_testbuffer_exec(PyObject *mod)
{
Py_SET_TYPE(&NDArray_Type, &PyType_Type);
if (PyType_Ready(&NDArray_Type)) {
return -1;
}
if (PyModule_AddType(mod, &NDArray_Type) < 0) {
return -1;
}

4
Modules/_testcapimodule.c
View File

@ -3869,7 +3869,9 @@ PyInit__testcapi(void)
return NULL;
Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type);
if (PyType_Ready(&_HashInheritanceTester_Type) < 0) {
return NULL;
}
if (PyType_Ready(&matmulType) < 0)
return NULL;
Py_INCREF(&matmulType);

20
Modules/_testinternalcapi.c
View File

@ -15,7 +15,7 @@
#include "pycore_ceval.h" // _PyEval_AddPendingCall()
#include "pycore_compile.h" // _PyCompile_CodeGen()
#include "pycore_context.h" // _PyContext_NewHamtForTests()
#include "pycore_dict.h" // _PyDictOrValues_GetValues()
#include "pycore_dict.h" // _PyManagedDictPointer_GetValues()
#include "pycore_fileutils.h" // _Py_normpath()
#include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_gc.h" // PyGC_Head
@ -1297,14 +1297,13 @@ static PyObject *
get_object_dict_values(PyObject *self, PyObject *obj)
{
PyTypeObject *type = Py_TYPE(obj);
if (!_PyType_HasFeature(type, Py_TPFLAGS_MANAGED_DICT)) {
if (!_PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) {
Py_RETURN_NONE;
}
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
if (!_PyDictOrValues_IsValues(dorv)) {
PyDictValues *values = _PyObject_InlineValues(obj);
if (!values->valid) {
Py_RETURN_NONE;
}
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(keys != NULL);
int size = (int)keys->dk_nentries;
@ -1784,6 +1783,16 @@ get_py_thread_id(PyObject *self, PyObject *Py_UNUSED(ignored))
}
#endif
static PyObject *
has_inline_values(PyObject *self, PyObject *obj)
{
if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
_PyObject_InlineValues(obj)->valid) {
Py_RETURN_TRUE;
}
Py_RETURN_FALSE;
}
static PyMethodDef module_functions[] = {
{"get_configs", get_configs, METH_NOARGS},
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@ -1857,6 +1866,7 @@ static PyMethodDef module_functions[] = {
_TESTINTERNALCAPI_TEST_LONG_NUMBITS_METHODDEF
{"get_rare_event_counters", get_rare_event_counters, METH_NOARGS},
{"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS},
{"has_inline_values", has_inline_values, METH_O},
#ifdef Py_GIL_DISABLED
{"py_thread_id", get_py_thread_id, METH_NOARGS},
#endif

464
Objects/dictobject.c
View File

@ -361,6 +361,10 @@ static int
dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_value,
PyObject **result, int incref_result);
#ifndef NDEBUG
static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj);
#endif
#include "clinic/dictobject.c.h"
@ -624,8 +628,9 @@ static inline int
get_index_from_order(PyDictObject *mp, Py_ssize_t i)
{
assert(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
assert(i < (((char *)mp->ma_values)[-2]));
return ((char *)mp->ma_values)[-3-i];
assert(i < mp->ma_values->size);
uint8_t *array = get_insertion_order_array(mp->ma_values);
return array[i];
}
#ifdef DEBUG_PYDICT
@ -672,6 +677,10 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
else {
CHECK(keys->dk_kind == DICT_KEYS_SPLIT);
CHECK(mp->ma_used <= SHARED_KEYS_MAX_SIZE);
if (mp->ma_values->embedded) {
CHECK(mp->ma_values->embedded == 1);
CHECK(mp->ma_values->valid == 1);
}
}
if (check_content) {
@ -821,33 +830,44 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr)
PyMem_Free(keys);
}
static size_t
values_size_from_count(size_t count)
{
assert(count >= 1);
size_t suffix_size = _Py_SIZE_ROUND_UP(count, sizeof(PyObject *));
assert(suffix_size < 128);
assert(suffix_size % sizeof(PyObject *) == 0);
return (count + 1) * sizeof(PyObject *) + suffix_size;
}
#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
static inline PyDictValues*
new_values(size_t size)
{
assert(size >= 1);
size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *));
assert(prefix_size < 256);
size_t n = prefix_size + size * sizeof(PyObject *);
uint8_t *mem = PyMem_Malloc(n);
if (mem == NULL) {
size_t n = values_size_from_count(size);
PyDictValues *res = (PyDictValues *)PyMem_Malloc(n);
if (res == NULL) {
return NULL;
}
assert(prefix_size % sizeof(PyObject *) == 0);
mem[prefix_size-1] = (uint8_t)prefix_size;
return (PyDictValues*)(mem + prefix_size);
res->embedded = 0;
res->size = 0;
assert(size < 256);
res->capacity = (uint8_t)size;
return res;
}
static inline void
free_values(PyDictValues *values, bool use_qsbr)
{
int prefix_size = DICT_VALUES_SIZE(values);
assert(values->embedded == 0);
#ifdef Py_GIL_DISABLED
if (use_qsbr) {
_PyMem_FreeDelayed(((char *)values)-prefix_size);
_PyMem_FreeDelayed(values);
return;
}
#endif
PyMem_Free(((char *)values)-prefix_size);
PyMem_Free(values);
}
/* Consumes a reference to the keys object */
@ -887,24 +907,6 @@ new_dict(PyInterpreterState *interp,
return (PyObject *)mp;
}
static inline size_t
shared_keys_usable_size(PyDictKeysObject *keys)
{
#ifdef Py_GIL_DISABLED
// dk_usable will decrease for each instance that is created and each
// value that is added. dk_nentries will increase for each value that
// is added. We want to always return the right value or larger.
// We therefore increase dk_nentries first and we decrease dk_usable
// second, and conversely here we read dk_usable first and dk_entries
// second (to avoid the case where we read entries before the increment
// and read usable after the decrement)
return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
_Py_atomic_load_ssize_acquire(&keys->dk_nentries));
#else
return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
#endif
}
/* Consumes a reference to the keys object */
static PyObject *
new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys)
@ -915,7 +917,6 @@ new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys)
dictkeys_decref(interp, keys, false);
return PyErr_NoMemory();
}
((char *)values)[-2] = 0;
for (size_t i = 0; i < size; i++) {
values->values[i] = NULL;
}
@ -1419,7 +1420,7 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb
if (values == NULL)
goto read_failed;
uint8_t capacity = _Py_atomic_load_uint8_relaxed(&DICT_VALUES_SIZE(values));
uint8_t capacity = _Py_atomic_load_uint8_relaxed(&values->capacity);
if (ix >= (Py_ssize_t)capacity)
goto read_failed;
@ -1525,6 +1526,7 @@ _PyDict_MaybeUntrack(PyObject *op)
return;
mp = (PyDictObject *) op;
ASSERT_CONSISTENT(mp);
numentries = mp->ma_keys->dk_nentries;
if (_PyDict_HasSplitTable(mp)) {
for (i = 0; i < numentries; i++) {
@ -1945,7 +1947,14 @@ dictresize(PyInterpreterState *interp, PyDictObject *mp,
set_keys(mp, newkeys);
dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp));
set_values(mp, NULL);
free_values(oldvalues, IS_DICT_SHARED(mp));
if (oldvalues->embedded) {
assert(oldvalues->embedded == 1);
assert(oldvalues->valid == 1);
oldvalues->valid = 0;
}
else {
free_values(oldvalues, IS_DICT_SHARED(mp));
}
}
else { // oldkeys is combined.
if (oldkeys->dk_kind == DICT_KEYS_GENERAL) {
@ -2464,17 +2473,19 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value,
static void
delete_index_from_values(PyDictValues *values, Py_ssize_t ix)
{
uint8_t *size_ptr = ((uint8_t *)values)-2;
int size = *size_ptr;
uint8_t *array = get_insertion_order_array(values);
int size = values->size;
assert(size <= values->capacity);
int i;
for (i = 1; size_ptr[-i] != ix; i++) {
assert(i <= size);
for (i = 0; array[i] != ix; i++) {
assert(i < size);
}
assert(i <= size);
assert(i < size);
size--;
for (; i < size; i++) {
size_ptr[-i] = size_ptr[-i-1];
array[i] = array[i+1];
}
*size_ptr = size -1;
values->size = size;
}
static int
@ -2669,10 +2680,12 @@ clear_lock_held(PyObject *op)
mp->ma_version_tag = new_version;
/* ...then clear the keys and values */
if (oldvalues != NULL) {
n = oldkeys->dk_nentries;
for (i = 0; i < n; i++)
Py_CLEAR(oldvalues->values[i]);
free_values(oldvalues, IS_DICT_SHARED(mp));
if (!oldvalues->embedded) {
n = oldkeys->dk_nentries;
for (i = 0; i < n; i++)
Py_CLEAR(oldvalues->values[i]);
free_values(oldvalues, IS_DICT_SHARED(mp));
}
dictkeys_decref(interp, oldkeys, false);
}
else {
@ -3059,10 +3072,12 @@ dict_dealloc(PyObject *self)
PyObject_GC_UnTrack(mp);
Py_TRASHCAN_BEGIN(mp, dict_dealloc)
if (values != NULL) {
for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) {
Py_XDECREF(values->values[i]);
if (values->embedded == 0) {
for (i = 0, n = mp->ma_keys->dk_nentries; i < n; i++) {
Py_XDECREF(values->values[i]);
}
free_values(values, false);
}
free_values(values, false);
dictkeys_decref(interp, keys, false);
}
else if (keys != NULL) {
@ -3595,10 +3610,12 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
PyDictKeysObject *okeys = other->ma_keys;
// If other is clean, combined, and just allocated, just clone it.
if (other->ma_values == NULL &&
other->ma_used == okeys->dk_nentries &&
(DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) {
if (mp->ma_values == NULL &&
other->ma_values == NULL &&
other->ma_used == okeys->dk_nentries &&
(DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE ||
USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)
) {
uint64_t new_version = _PyDict_NotifyEvent(
interp, PyDict_EVENT_CLONED, mp, (PyObject *)other, NULL);
PyDictKeysObject *keys = clone_combined_dict_keys(other);
@ -3608,11 +3625,6 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe
ensure_shared_on_resize(mp);
dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp));
mp->ma_keys = keys;
if (_PyDict_HasSplitTable(mp)) {
free_values(mp->ma_values, IS_DICT_SHARED(mp));
mp->ma_values = NULL;
}
mp->ma_used = other->ma_used;
mp->ma_version_tag = new_version;
ASSERT_CONSISTENT(mp);
@ -3816,6 +3828,27 @@ dict_copy_impl(PyDictObject *self)
return PyDict_Copy((PyObject *)self);
}
/* Copies the values, but does not change the reference
* counts of the objects in the array. */
static PyDictValues *
copy_values(PyDictValues *values)
{
PyDictValues *newvalues = new_values(values->capacity);
if (newvalues == NULL) {
PyErr_NoMemory();
return NULL;
}
newvalues->size = values->size;
uint8_t *values_order = get_insertion_order_array(values);
uint8_t *new_values_order = get_insertion_order_array(newvalues);
memcpy(new_values_order, values_order, values->capacity);
for (int i = 0; i < values->capacity; i++) {
newvalues->values[i] = values->values[i];
}
assert(newvalues->embedded == 0);
return newvalues;
}
static PyObject *
copy_lock_held(PyObject *o)
{
@ -3833,26 +3866,23 @@ copy_lock_held(PyObject *o)
if (_PyDict_HasSplitTable(mp)) {
PyDictObject *split_copy;
size_t size = shared_keys_usable_size(mp->ma_keys);
PyDictValues *newvalues = new_values(size);
if (newvalues == NULL)
PyDictValues *newvalues = copy_values(mp->ma_values);
if (newvalues == NULL) {
return PyErr_NoMemory();
}
split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type);
if (split_copy == NULL) {
free_values(newvalues, false);
return NULL;
}
size_t prefix_size = ((uint8_t *)newvalues)[-1];
memcpy(((char *)newvalues)-prefix_size, ((char *)mp->ma_values)-prefix_size, prefix_size-1);
for (size_t i = 0; i < newvalues->capacity; i++) {
Py_XINCREF(newvalues->values[i]);
}
split_copy->ma_values = newvalues;
split_copy->ma_keys = mp->ma_keys;
split_copy->ma_used = mp->ma_used;
split_copy->ma_version_tag = DICT_NEXT_VERSION(interp);
dictkeys_incref(mp->ma_keys);
for (size_t i = 0; i < size; i++) {
PyObject *value = mp->ma_values->values[i];
split_copy->ma_values->values[i] = Py_XNewRef(value);
}
if (_PyObject_GC_IS_TRACKED(mp))
_PyObject_GC_TRACK(split_copy);
return (PyObject *)split_copy;
@ -4406,8 +4436,10 @@ dict_traverse(PyObject *op, visitproc visit, void *arg)
if (DK_IS_UNICODE(keys)) {
if (_PyDict_HasSplitTable(mp)) {
for (i = 0; i < n; i++) {
Py_VISIT(mp->ma_values->values[i]);
if (!mp->ma_values->embedded) {
for (i = 0; i < n; i++) {
Py_VISIT(mp->ma_values->values[i]);
}
}
}
else {
@ -5296,12 +5328,6 @@ acquire_key_value(PyObject **key_loc, PyObject *value, PyObject **value_loc,
return 0;
}
static Py_ssize_t
load_values_used_size(PyDictValues *values)
{
return (Py_ssize_t)_Py_atomic_load_uint8(&DICT_VALUES_USED_SIZE(values));
}
static int
dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self,
PyObject **out_key, PyObject **out_value)
@ -5330,7 +5356,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self,
goto concurrent_modification;
}
Py_ssize_t used = load_values_used_size(values);
Py_ssize_t used = (Py_ssize_t)_Py_atomic_load_uint8(&values->size);
if (i >= used) {
goto fail;
}
@ -6539,15 +6565,15 @@ _PyDict_NewKeysForClass(void)
return keys;
}
#define CACHED_KEYS(tp) (((PyHeapTypeObject*)tp)->ht_cached_keys)
int
void
_PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
{
assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictKeysObject *keys = CACHED_KEYS(tp);
assert(keys != NULL);
OBJECT_STAT_INC(inline_values);
#ifdef Py_GIL_DISABLED
Py_ssize_t usable = _Py_atomic_load_ssize_relaxed(&keys->dk_usable);
if (usable > 1) {
@ -6563,49 +6589,19 @@ _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp)
}
#endif
size_t size = shared_keys_usable_size(keys);
PyDictValues *values = new_values(size);
if (values == NULL) {
PyErr_NoMemory();
return -1;
}
assert(((uint8_t *)values)[-1] >= (size + 2));
((uint8_t *)values)[-2] = 0;
PyDictValues *values = _PyObject_InlineValues(obj);
assert(size < 256);
values->capacity = (uint8_t)size;
values->size = 0;
values->embedded = 1;
values->valid = 1;
for (size_t i = 0; i < size; i++) {
values->values[i] = NULL;
}
_PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values);
return 0;
_PyObject_ManagedDictPointer(obj)->dict = NULL;
}
int
_PyObject_InitializeDict(PyObject *obj)
{
PyInterpreterState *interp = _PyInterpreterState_GET();
PyTypeObject *tp = Py_TYPE(obj);
if (tp->tp_dictoffset == 0) {
return 0;
}
if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
OBJECT_STAT_INC(new_values);
return _PyObject_InitInlineValues(obj, tp);
}
PyObject *dict;
if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
dictkeys_incref(CACHED_KEYS(tp));
dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
}
else {
dict = PyDict_New();
}
if (dict == NULL) {
return -1;
}
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
*dictptr = dict;
return 0;
}
static PyObject *
static PyDictObject *
make_dict_from_instance_attributes(PyInterpreterState *interp,
PyDictKeysObject *keys, PyDictValues *values)
{
@ -6620,56 +6616,24 @@ make_dict_from_instance_attributes(PyInterpreterState *interp,
track += _PyObject_GC_MAY_BE_TRACKED(val);
}
}
PyObject *res = new_dict(interp, keys, values, used, 0);
PyDictObject *res = (PyDictObject *)new_dict(interp, keys, values, used, 0);
if (track && res) {
_PyObject_GC_TRACK(res);
}
return res;
}
PyObject *
_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values)
PyDictObject *
_PyObject_MakeDictFromInstanceAttributes(PyObject *obj)
{
PyDictValues *values = _PyObject_InlineValues(obj);
PyInterpreterState *interp = _PyInterpreterState_GET();
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
OBJECT_STAT_INC(dict_materialized_on_request);
return make_dict_from_instance_attributes(interp, keys, values);
}
// Return true if the dict was dematerialized, false otherwise.
bool
_PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv)
{
assert(_PyObject_DictOrValuesPointer(obj) == dorv);
assert(!_PyDictOrValues_IsValues(*dorv));
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
if (dict == NULL) {
return false;
}
// It's likely that this dict still shares its keys (if it was materialized
// on request and not heavily modified):
if (!PyDict_CheckExact(dict)) {
return false;
}
assert(_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_HEAPTYPE));
if (dict->ma_keys != CACHED_KEYS(Py_TYPE(obj)) ||
!has_unique_reference((PyObject *)dict))
{
return false;
}
ensure_shared_on_resize(dict);
assert(dict->ma_values);
// We have an opportunity to do something *really* cool: dematerialize it!
_PyDictKeys_DecRef(dict->ma_keys);
_PyDictOrValues_SetValues(dorv, dict->ma_values);
OBJECT_STAT_INC(dict_dematerialized);
// Don't try this at home, kids:
dict->ma_keys = NULL;
dict->ma_values = NULL;
Py_DECREF(dict);
return true;
}
int
_PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
@ -6679,7 +6643,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj));
assert(keys != NULL);
assert(values != NULL);
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
Py_ssize_t ix = DKIX_EMPTY;
if (PyUnicode_CheckExact(name)) {
Py_hash_t hash = unicode_get_hash(name);
@ -6717,18 +6681,21 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
}
#endif
}
PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
if (ix == DKIX_EMPTY) {
PyObject *dict = make_dict_from_instance_attributes(
interp, keys, values);
if (dict == NULL) {
return -1;
dict = make_dict_from_instance_attributes(
interp, keys, values);
if (dict == NULL) {
return -1;
}
_PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
}
_PyObject_DictOrValuesPointer(obj)->dict = dict;
if (value == NULL) {
return PyDict_DelItem(dict, name);
return PyDict_DelItem((PyObject *)dict, name);
}
else {
return PyDict_SetItem(dict, name, value);
return PyDict_SetItem((PyObject *)dict, name, value);
}
}
PyObject *old_value = values->values[ix];
@ -6741,10 +6708,18 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
return -1;
}
_PyDictValues_AddToInsertionOrder(values, ix);
if (dict) {
assert(dict->ma_values == values);
dict->ma_used++;
}
}
else {
if (value == NULL) {
delete_index_from_values(values, ix);
if (dict) {
assert(dict->ma_values == values);
dict->ma_used--;
}
}
Py_DECREF(old_value);
}
@ -6760,9 +6735,9 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
{
PyTypeObject *tp = Py_TYPE(obj);
CHECK(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
if (_PyManagedDictPointer_IsValues(*managed_dict)) {
PyDictValues *values = _PyManagedDictPointer_GetValues(*managed_dict);
int size = ((uint8_t *)values)[-2];
int count = 0;
PyDictKeysObject *keys = CACHED_KEYS(tp);
@ -6774,8 +6749,8 @@ _PyObject_ManagedDictValidityCheck(PyObject *obj)
CHECK(size == count);
}
else {
if (dorv_ptr->dict != NULL) {
CHECK(PyDict_Check(dorv_ptr->dict));
if (managed_dict->dict != NULL) {
CHECK(PyDict_Check(managed_dict->dict));
}
}
return 1;
@ -6804,23 +6779,27 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
if (tp->tp_dictoffset == 0) {
return 1;
}
PyObject *dict;
if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(dorv)) {
PyDictObject *dict;
if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictValues *values = _PyObject_InlineValues(obj);
if (values->valid) {
PyDictKeysObject *keys = CACHED_KEYS(tp);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
if (_PyDictOrValues_GetValues(dorv)->values[i] != NULL) {
if (values->values[i] != NULL) {
return 0;
}
}
return 1;
}
dict = _PyDictOrValues_GetDict(dorv);
dict = _PyObject_ManagedDictPointer(obj)->dict;
}
else if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
dict = managed_dict->dict;
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
dict = *dictptr;
dict = (PyDictObject *)*dictptr;
}
if (dict == NULL) {
return 1;
@ -6828,23 +6807,6 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj)
return ((PyDictObject *)dict)->ma_used == 0;
}
void
_PyObject_FreeInstanceAttributes(PyObject *self)
{
PyTypeObject *tp = Py_TYPE(self);
assert(Py_TYPE(self)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(self);
if (!_PyDictOrValues_IsValues(dorv)) {
return;
}
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
PyDictKeysObject *keys = CACHED_KEYS(tp);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
Py_XDECREF(values->values[i]);
}
free_values(values, IS_DICT_SHARED((PyDictObject*)self));
}
int
PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
{
@ -6852,74 +6814,101 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return 0;
}
assert(tp->tp_dictoffset);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(dorv)) {
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
PyDictKeysObject *keys = CACHED_KEYS(tp);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
Py_VISIT(values->values[i]);
if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictValues *values = _PyObject_InlineValues(obj);
if (values->valid) {
for (Py_ssize_t i = 0; i < values->capacity; i++) {
Py_VISIT(values->values[i]);
}
return 0;
}
}
else {
PyObject *dict = _PyDictOrValues_GetDict(dorv);
Py_VISIT(dict);
}
Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict);
return 0;
}
void
PyObject_ClearManagedDict(PyObject *obj)
{
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
assert(_PyObject_InlineValuesConsistencyCheck(obj));
PyTypeObject *tp = Py_TYPE(obj);
if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return;
}
PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
PyDictKeysObject *keys = CACHED_KEYS(tp);
for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) {
Py_CLEAR(values->values[i]);
}
dorv_ptr->dict = NULL;
free_values(values, IS_DICT_SHARED((PyDictObject*)obj));
}
else {
PyObject *dict = dorv_ptr->dict;
if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict;
if (dict) {
dorv_ptr->dict = NULL;
_PyDict_DetachFromObject(dict, obj);
_PyObject_ManagedDictPointer(obj)->dict = NULL;
Py_DECREF(dict);
}
else {
PyDictValues *values = _PyObject_InlineValues(obj);
if (values->valid) {
for (Py_ssize_t i = 0; i < values->capacity; i++) {
Py_CLEAR(values->values[i]);
}
values->valid = 0;
}
}
}
else {
Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict);
}
assert(_PyObject_InlineValuesConsistencyCheck(obj));
}
int
_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
{
assert(_PyObject_ManagedDictPointer(obj)->dict == mp);
assert(_PyObject_InlineValuesConsistencyCheck(obj));
if (mp->ma_values == NULL || mp->ma_values != _PyObject_InlineValues(obj)) {
return 0;
}
assert(mp->ma_values->embedded == 1);
assert(mp->ma_values->valid == 1);
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
Py_BEGIN_CRITICAL_SECTION(mp);
mp->ma_values = copy_values(mp->ma_values);
_PyObject_InlineValues(obj)->valid = 0;
Py_END_CRITICAL_SECTION();
if (mp->ma_values == NULL) {
return -1;
}
assert(_PyObject_InlineValuesConsistencyCheck(obj));
ASSERT_CONSISTENT(mp);
return 0;
}
PyObject *
PyObject_GenericGetDict(PyObject *obj, void *context)
{
PyObject *dict;
PyInterpreterState *interp = _PyInterpreterState_GET();
PyTypeObject *tp = Py_TYPE(obj);
if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) {
PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
PyDictObject *dict = managed_dict->dict;
if (dict == NULL &&
(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) &&
_PyObject_InlineValues(obj)->valid
) {
PyDictValues *values = _PyObject_InlineValues(obj);
OBJECT_STAT_INC(dict_materialized_on_request);
dict = make_dict_from_instance_attributes(
interp, CACHED_KEYS(tp), values);
if (dict != NULL) {
dorv_ptr->dict = dict;
managed_dict->dict = (PyDictObject *)dict;
}
}
else {
dict = _PyDictOrValues_GetDict(*dorv_ptr);
dict = managed_dict->dict;
if (dict == NULL) {
dictkeys_incref(CACHED_KEYS(tp));
OBJECT_STAT_INC(dict_materialized_on_request);
dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
dorv_ptr->dict = dict;
dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp));
managed_dict->dict = (PyDictObject *)dict;
}
}
return Py_XNewRef((PyObject *)dict);
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@ -6928,7 +6917,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
"This object has no __dict__");
return NULL;
}
dict = *dictptr;
PyObject *dict = *dictptr;
if (dict == NULL) {
PyTypeObject *tp = Py_TYPE(obj);
if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) {
@ -6940,8 +6929,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context)
*dictptr = dict = PyDict_New();
}
}
return Py_XNewRef(dict);
}
return Py_XNewRef(dict);
}
int
@ -6958,7 +6947,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr,
assert(dictptr != NULL);
dict = *dictptr;
if (dict == NULL) {
assert(!_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT));
assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES));
dictkeys_incref(cached);
dict = new_dict_with_shared_keys(interp, cached);
if (dict == NULL)
@ -7118,3 +7107,24 @@ _PyDict_SendEvent(int watcher_bits,
watcher_bits >>= 1;
}
}
#ifndef NDEBUG
static int
_PyObject_InlineValuesConsistencyCheck(PyObject *obj)
{
if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
return 1;
}
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictObject *dict = (PyDictObject *)_PyObject_ManagedDictPointer(obj)->dict;
if (dict == NULL) {
return 1;
}
if (dict->ma_values == _PyObject_InlineValues(obj) ||
_PyObject_InlineValues(obj)->valid == 0) {
return 1;
}
assert(0);
return 0;
}
#endif

100
Objects/object.c
View File

@ -1396,16 +1396,16 @@ _PyObject_GetDictPtr(PyObject *obj)
if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
return _PyObject_ComputedDictPointer(obj);
}
PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
PyObject *dict = _PyObject_MakeDictFromInstanceAttributes(obj, _PyDictOrValues_GetValues(*dorv_ptr));
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
if (managed_dict->dict == NULL && Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictObject *dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
if (dict == NULL) {
PyErr_Clear();
return NULL;
}
dorv_ptr->dict = dict;
managed_dict->dict = dict;
}
return &dorv_ptr->dict;
return (PyObject **)&managed_dict->dict;
}
PyObject *
@ -1474,21 +1474,19 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method)
}
}
PyObject *dict;
if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
if (attr != NULL) {
*method = attr;
Py_XDECREF(descr);
return 0;
}
dict = NULL;
}
else {
dict = dorv_ptr->dict;
if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
PyDictValues *values = _PyObject_InlineValues(obj);
PyObject *attr = _PyObject_GetInstanceAttribute(obj, values, name);
if (attr != NULL) {
*method = attr;
Py_XDECREF(descr);
return 0;
}
dict = NULL;
}
else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
dict = (PyObject *)managed_dict->dict;
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
@ -1581,29 +1579,27 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name,
}
}
if (dict == NULL) {
if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
PyDictOrValues* dorv_ptr = _PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr);
if (PyUnicode_CheckExact(name)) {
res = _PyObject_GetInstanceAttribute(obj, values, name);
if (res != NULL) {
goto done;
}
}
else {
dict = _PyObject_MakeDictFromInstanceAttributes(obj, values);
if (dict == NULL) {
res = NULL;
goto done;
}
dorv_ptr->dict = dict;
if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
PyDictValues *values = _PyObject_InlineValues(obj);
if (PyUnicode_CheckExact(name)) {
res = _PyObject_GetInstanceAttribute(obj, values, name);
if (res != NULL) {
goto done;
}
}
else {
dict = _PyDictOrValues_GetDict(*dorv_ptr);
dict = (PyObject *)_PyObject_MakeDictFromInstanceAttributes(obj);
if (dict == NULL) {
res = NULL;
goto done;
}
_PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)dict;
}
}
else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
PyManagedDictPointer* managed_dict = _PyObject_ManagedDictPointer(obj);
dict = (PyObject *)managed_dict->dict;
}
else {
PyObject **dictptr = _PyObject_ComputedDictPointer(obj);
if (dictptr) {
@ -1697,22 +1693,14 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name,
if (dict == NULL) {
PyObject **dictptr;
if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
res = _PyObject_StoreInstanceAttribute(
obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
goto error_check;
}
dictptr = &dorv_ptr->dict;
if (*dictptr == NULL) {
if (_PyObject_InitInlineValues(obj, tp) < 0) {
goto done;
}
res = _PyObject_StoreInstanceAttribute(
obj, _PyDictOrValues_GetValues(*dorv_ptr), name, value);
goto error_check;
}
if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && _PyObject_InlineValues(obj)->valid) {
res = _PyObject_StoreInstanceAttribute(
obj, _PyObject_InlineValues(obj), name, value);
goto error_check;
}
else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) {
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(obj);
dictptr = (PyObject **)&managed_dict->dict;
}
else {
dictptr = _PyObject_ComputedDictPointer(obj);
@ -1783,9 +1771,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context)
{
PyObject **dictptr = _PyObject_GetDictPtr(obj);
if (dictptr == NULL) {
if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_MANAGED_DICT) &&
_PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(obj)))
{
if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) &&
_PyObject_ManagedDictPointer(obj)->dict == NULL
) {
/* Was unable to convert to dict */
PyErr_NoMemory();
}

93
Objects/object_layout.md
View File

@ -16,25 +16,23 @@ Since the introduction of the cycle GC, there has also been a pre-header.
Before 3.11, this pre-header was two words in size.
It should be considered opaque to all code except the cycle GC.
## 3.11 pre-header
### 3.13
In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``.
The reason for moving the ``__dict__`` to the pre-header is that it allows
faster access, as it is at a fixed offset, and it also allows object's
dictionaries to be lazily created when the ``__dict__`` attribute is
specifically asked for.
In 3.13, the values array is embedded into the object, so there is no
need for a values pointer (it is just a fixed offset into the object).
So the pre-header is these two fields:
In the 3.11 the non-GC part of the pre-header consists of two pointers:
* weakreflist
* dict_pointer
* dict
* values
If the object has no physical dictionary, then the ``dict_pointer``
is set to `NULL`.
The values pointer refers to the ``PyDictValues`` array which holds the
values of the objects's attributes.
Should the dictionary be needed, then ``values`` is set to ``NULL``
and the ``dict`` field points to the dictionary.
## 3.12 pre-header
<details>
<summary> 3.12 </summary>
### 3.12
In 3.12, the pointer to the list of weak references is added to the
pre-header. In order to make space for it, the ``dict`` and ``values``
@ -51,9 +49,38 @@ has its low bit set to zero, and points to the dictionary.
The untagged form is chosen for the dictionary pointer, rather than
the values pointer, to enable the (legacy) C-API function
`_PyObject_GetDictPtr(PyObject *obj)` to work.
</details>
<details>
<summary> 3.11 </summary>
## Layout of a "normal" Python object in 3.12:
### 3.11
In 3.11 the pre-header was extended to include pointers to the VM managed ``__dict__``.
The reason for moving the ``__dict__`` to the pre-header is that it allows
faster access, as it is at a fixed offset, and it also allows object's
dictionaries to be lazily created when the ``__dict__`` attribute is
specifically asked for.
In the 3.11 the non-GC part of the pre-header consists of two pointers:
* dict
* values
The values pointer refers to the ``PyDictValues`` array which holds the
values of the objects's attributes.
Should the dictionary be needed, then ``values`` is set to ``NULL``
and the ``dict`` field points to the dictionary.
</details>
## Layout of a "normal" Python object
A "normal" Python object is one that doesn't inherit from a builtin
class, doesn't have slots.
### 3.13
In 3.13 the values are embedded into the object, as follows:
* weakreflist
* dict_or_values
@ -61,9 +88,39 @@ the values pointer, to enable the (legacy) C-API function
* GC 2
* ob_refcnt
* ob_type
* Inlined values:
* Flags
* values 0
* values 1
* ...
* Insertion order bytes
For a "normal" Python object, one that doesn't inherit from a builtin
class or have slots, the header and pre-header form the entire object.
This has all the advantages of the layout used in 3.12, plus:
* Access to values is even faster as there is one less load
* Fast access is mostly maintained when the `__dict__` is materialized
![Layout of "normal" object in 3.13](./object_layout_313.png)
For objects with opaque parts defined by a C extension,
the layout is much the same as for 3.12
![Layout of "full" object in 3.13](./object_layout_full_313.png)
<details>
<summary> 3.12 </summary>
### 3.12:
In 3.12, the header and pre-header form the entire object for "normal"
Python objects:
* weakreflist
* dict_or_values
* GC 1
* GC 2
* ob_refcnt
* ob_type
![Layout of "normal" object in 3.12](./object_layout_312.png)
@ -79,4 +136,6 @@ The full layout object, with an opaque part defined by a C extension,
and `__slots__` looks like this:
![Layout of "full" object in 3.12](./object_layout_full_312.png)
</details>

1
Objects/object_layout_312.gv
View File

@ -20,6 +20,7 @@ digraph ideal {
shape = none
label = <<table border="0" cellspacing="0">
<tr><td><b>values</b></td></tr>
<tr><td border="1">Insertion order</td></tr>
<tr><td port="0" border="1">values[0]</td></tr>
<tr><td border="1">values[1]</td></tr>
<tr><td border="1">...</td></tr>

BIN
Objects/object_layout_312.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 30 KiB

After

Width:  |  Height:  |  Size: 32 KiB

45
Objects/object_layout_313.gv Normal file
View File

@ -0,0 +1,45 @@
digraph ideal {
rankdir = "LR"
object [
shape = none
label = <<table border="0" cellspacing="0">
<tr><td><b>object</b></td></tr>
<tr><td port="w" border="1">weakrefs</td></tr>
<tr><td port="dv" border="1">dict pointer</td></tr>
<tr><td border="1" >GC info 0</td></tr>
<tr><td border="1" >GC info 1</td></tr>
<tr><td port="r" border="1" >refcount</td></tr>
<tr><td port="h" border="1" >__class__</td></tr>
<tr><td border="1">values flags</td></tr>
<tr><td port="0" border="1">values[0]</td></tr>
<tr><td border="1">values[1]</td></tr>
<tr><td border="1">...</td></tr>
<tr><td border="1">Insertion order</td></tr>
</table>>
]
class [
shape = none
label = <<table border="0" cellspacing="0">
<tr><td><b>class</b></td></tr>
<tr><td port="head" bgcolor="lightgreen" border="1">...</td></tr>
<tr><td border="1" bgcolor="lightgreen">dict_offset</td></tr>
<tr><td border="1" bgcolor="lightgreen">...</td></tr>
<tr><td port="k" border="1" bgcolor="lightgreen">cached_keys</td></tr>
</table>>
]
keys [label = "dictionary keys"; fillcolor="lightgreen"; style="filled"]
NULL [ label = " NULL"; shape="plain"]
object:w -> NULL
object:h -> class:head
object:dv -> NULL
class:k -> keys
oop [ label = "pointer"; shape="plain"]
oop -> object:r
}

BIN
Objects/object_layout_313.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 34 KiB

25
Objects/object_layout_full_313.gv Normal file
View File

@ -0,0 +1,25 @@
digraph ideal {
rankdir = "LR"
object [
shape = none
label = <<table border="0" cellspacing="0">
<tr><td><b>object</b></td></tr>
<tr><td port="w" border="1">weakrefs</td></tr>
<tr><td port="dv" border="1">dict pointer</td></tr>
<tr><td border="1" >GC info 0</td></tr>
<tr><td border="1" >GC info 1</td></tr>
<tr><td port="r" border="1" >refcount</td></tr>
<tr><td port="h" border="1" >__class__</td></tr>
<tr><td border="1">opaque (extension) data </td></tr>
<tr><td border="1">...</td></tr>
<tr><td border="1">__slot__ 0</td></tr>
<tr><td border="1">...</td></tr>
</table>>
]
oop [ label = "pointer"; shape="plain"]
oop -> object:r
}

BIN
Objects/object_layout_full_313.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 17 KiB

84
Objects/typeobject.c
View File

@ -1861,7 +1861,7 @@ type_call(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *
_PyType_NewManagedObject(PyTypeObject *type)
{
assert(type->tp_flags & Py_TPFLAGS_MANAGED_DICT);
assert(type->tp_flags & Py_TPFLAGS_INLINE_VALUES);
assert(_PyType_IS_GC(type));
assert(type->tp_new == PyBaseObject_Type.tp_new);
assert(type->tp_alloc == PyType_GenericAlloc);
@ -1870,11 +1870,6 @@ _PyType_NewManagedObject(PyTypeObject *type)
if (obj == NULL) {
return PyErr_NoMemory();
}
_PyObject_DictOrValuesPointer(obj)->dict = NULL;
if (_PyObject_InitInlineValues(obj, type)) {
Py_DECREF(obj);
return NULL;
}
return obj;
}
@ -1888,9 +1883,13 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
* flag to indicate when that is safe) it does not seem worth the memory
* savings. An example type that doesn't need the +1 is a subclass of
* tuple. See GH-100659 and GH-81381. */
const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
size_t size = _PyObject_VAR_SIZE(type, nitems+1);
const size_t presize = _PyType_PreHeaderSize(type);
if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
assert(type->tp_itemsize == 0);
size += _PyInlineValuesSize(type);
}
char *alloc = _PyObject_MallocWithType(type, size + presize);
if (alloc == NULL) {
return PyErr_NoMemory();
@ -1911,6 +1910,9 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
else {
_PyObject_InitVar((PyVarObject *)obj, type, nitems);
}
if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
_PyObject_InitInlineValues(obj, type);
}
return obj;
}
@ -2060,6 +2062,10 @@ subtype_clear(PyObject *self)
if ((base->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
PyObject_ClearManagedDict(self);
}
else {
assert((base->tp_flags & Py_TPFLAGS_INLINE_VALUES) ==
(type->tp_flags & Py_TPFLAGS_INLINE_VALUES));
}
}
else if (type->tp_dictoffset != base->tp_dictoffset) {
PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@ -2210,14 +2216,7 @@ subtype_dealloc(PyObject *self)
/* If we added a dict, DECREF it, or free inline values. */
if (type->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(self);
if (_PyDictOrValues_IsValues(*dorv_ptr)) {
_PyObject_FreeInstanceAttributes(self);
}
else {
Py_XDECREF(_PyDictOrValues_GetDict(*dorv_ptr));
}
dorv_ptr->values = NULL;
PyObject_ClearManagedDict(self);
}
else if (type->tp_dictoffset && !base->tp_dictoffset) {
PyObject **dictptr = _PyObject_ComputedDictPointer(self);
@ -3161,19 +3160,26 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context)
return func(descr, obj, value);
}
/* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */
dictptr = _PyObject_GetDictPtr(obj);
if (dictptr == NULL) {
PyErr_SetString(PyExc_AttributeError,
"This object has no __dict__");
return -1;
}
if (value != NULL && !PyDict_Check(value)) {
PyErr_Format(PyExc_TypeError,
"__dict__ must be set to a dictionary, "
"not a '%.200s'", Py_TYPE(value)->tp_name);
return -1;
}
Py_XSETREF(*dictptr, Py_XNewRef(value));
if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
PyObject_ClearManagedDict(obj);
_PyObject_ManagedDictPointer(obj)->dict = (PyDictObject *)Py_XNewRef(value);
}
else {
dictptr = _PyObject_ComputedDictPointer(obj);
if (dictptr == NULL) {
PyErr_SetString(PyExc_AttributeError,
"This object has no __dict__");
return -1;
}
Py_CLEAR(*dictptr);
*dictptr = Py_XNewRef(value);
}
return 0;
}
@ -5849,10 +5855,6 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
if (obj == NULL) {
return NULL;
}
if (_PyObject_InitializeDict(obj)) {
Py_DECREF(obj);
return NULL;
}
return obj;
}
@ -6036,6 +6038,11 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char*
!same_slots_added(newbase, oldbase))) {
goto differs;
}
if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) !=
((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES)))
{
goto differs;
}
/* The above does not check for the preheader */
if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) ==
((newto->tp_flags & Py_TPFLAGS_PREHEADER)))
@ -6137,14 +6144,18 @@ object_set_class(PyObject *self, PyObject *value, void *closure)
if (compatible_for_assignment(oldto, newto, "__class__")) {
/* Changing the class will change the implicit dict keys,
* so we must materialize the dictionary first. */
assert((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == (newto->tp_flags & Py_TPFLAGS_PREHEADER));
_PyObject_GetDictPtr(self);
if (oldto->tp_flags & Py_TPFLAGS_MANAGED_DICT &&
_PyDictOrValues_IsValues(*_PyObject_DictOrValuesPointer(self)))
{
/* Was unable to convert to dict */
PyErr_NoMemory();
return -1;
if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictObject *dict = _PyObject_ManagedDictPointer(self)->dict;
if (dict == NULL) {
dict = (PyDictObject *)_PyObject_MakeDictFromInstanceAttributes(self);
if (dict == NULL) {
return -1;
}
_PyObject_ManagedDictPointer(self)->dict = dict;
}
if (_PyDict_DetachFromObject(dict, self)) {
return -1;
}
}
if (newto->tp_flags & Py_TPFLAGS_HEAPTYPE) {
Py_INCREF(newto);
@ -7774,6 +7785,9 @@ type_ready_managed_dict(PyTypeObject *type)
return -1;
}
}
if (type->tp_itemsize == 0 && type->tp_basicsize == sizeof(PyObject)) {
type->tp_flags |= Py_TPFLAGS_INLINE_VALUES;
}
return 0;
}
@ -7901,6 +7915,8 @@ PyType_Ready(PyTypeObject *type)
/* Historically, all static types were immutable. See bpo-43908 */
if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE;
/* Static types must be immortal */
_Py_SetImmortalUntracked((PyObject *)type);
}
int res;

52
Python/bytecodes.c
View File

@ -1897,14 +1897,12 @@ dummy_func(
op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
}
split op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) {
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
attr = _PyObject_InlineValues(owner)->values[index];
DEOPT_IF(attr == NULL);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@ -1947,16 +1945,15 @@ dummy_func(
op(_CHECK_ATTR_WITH_HINT, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(_PyDictOrValues_IsValues(dorv));
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
}
op(_LOAD_ATTR_WITH_HINT, (hint/1, owner -- attr, null if (oparg & 1))) {
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
@ -2070,16 +2067,17 @@ dummy_func(
DISPATCH_INLINED(new_frame);
}
op(_GUARD_DORV_VALUES, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv));
op(_GUARD_DORV_NO_DICT, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict);
DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0);
}
op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) {
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
STAT_INC(STORE_ATTR, hit);
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
@ -2094,7 +2092,7 @@ dummy_func(
macro(STORE_ATTR_INSTANCE_VALUE) =
unused/1 +
_GUARD_TYPE_VERSION +
_GUARD_DORV_VALUES +
_GUARD_DORV_NO_DICT +
_STORE_ATTR_INSTANCE_VALUE;
inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) {
@ -2102,9 +2100,8 @@ dummy_func(
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(_PyDictOrValues_IsValues(dorv));
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);
@ -2898,9 +2895,8 @@ dummy_func(
}
op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) {
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv));
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid);
}
op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) {
@ -2972,10 +2968,9 @@ dummy_func(
unused/2 +
_LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
op(_CHECK_ATTR_METHOD_LAZY_DICT, (owner -- owner)) {
Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
assert(dictoffset > 0);
PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
op(_CHECK_ATTR_METHOD_LAZY_DICT, (dictoffset/1, owner -- owner)) {
char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL);
}
@ -2993,7 +2988,7 @@ dummy_func(
unused/1 +
_GUARD_TYPE_VERSION +
_CHECK_ATTR_METHOD_LAZY_DICT +
unused/2 +
unused/1 +
_LOAD_ATTR_METHOD_LAZY_DICT;
inst(INSTRUMENTED_CALL, (unused/3 -- )) {
@ -3294,6 +3289,7 @@ dummy_func(
DEOPT_IF(!PyType_Check(callable));
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version));
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;

44
Python/executor_cases.c.h generated
View File

@ -1753,9 +1753,8 @@
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET();
break;
}
@ -1766,8 +1765,7 @@
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
attr = _PyObject_InlineValues(owner)->values[index];
if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@ -1784,8 +1782,7 @@
(void)null;
owner = stack_pointer[-1];
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
attr = _PyObject_InlineValues(owner)->values[index];
if (attr == NULL) JUMP_TO_JUMP_TARGET();
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@ -1837,9 +1834,8 @@
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
if (_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
if (dict == NULL) JUMP_TO_JUMP_TARGET();
assert(PyDict_CheckExact((PyObject *)dict));
break;
@ -1852,8 +1848,8 @@
oparg = CURRENT_OPARG();
owner = stack_pointer[-1];
uint16_t hint = (uint16_t)CURRENT_OPERAND();
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
if (hint >= (size_t)dict->ma_keys->dk_nentries) JUMP_TO_JUMP_TARGET();
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
@ -1967,12 +1963,13 @@
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */
case _GUARD_DORV_VALUES: {
case _GUARD_DORV_NO_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
if (!_PyDictOrValues_IsValues(dorv)) JUMP_TO_JUMP_TARGET();
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (_PyObject_ManagedDictPointer(owner)->dict) JUMP_TO_JUMP_TARGET();
if (_PyObject_InlineValues(owner)->valid == 0) JUMP_TO_JUMP_TARGET();
break;
}
@ -1982,9 +1979,9 @@
owner = stack_pointer[-1];
value = stack_pointer[-2];
uint16_t index = (uint16_t)CURRENT_OPERAND();
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
STAT_INC(STORE_ATTR, hit);
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
@ -2568,9 +2565,8 @@
case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
if (!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)) JUMP_TO_JUMP_TARGET();
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
if (!_PyObject_InlineValues(owner)->valid) JUMP_TO_JUMP_TARGET();
break;
}
@ -2658,9 +2654,9 @@
case _CHECK_ATTR_METHOD_LAZY_DICT: {
PyObject *owner;
owner = stack_pointer[-1];
Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
assert(dictoffset > 0);
PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
uint16_t dictoffset = (uint16_t)CURRENT_OPERAND();
char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
if (dict != NULL) JUMP_TO_JUMP_TARGET();
break;

7
Python/gc.c
View File

@ -2031,11 +2031,16 @@ gc_alloc(PyTypeObject *tp, size_t basicsize, size_t presize)
return op;
}
PyObject *
_PyObject_GC_New(PyTypeObject *tp)
{
size_t presize = _PyType_PreHeaderSize(tp);
PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize);
size_t size = _PyObject_SIZE(tp);
if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) {
size += _PyInlineValuesSize(tp);
}
PyObject *op = gc_alloc(tp, size, presize);
if (op == NULL) {
return NULL;
}

6
Python/gc_free_threading.c
View File

@ -1639,7 +1639,11 @@ PyObject *
_PyObject_GC_New(PyTypeObject *tp)
{
size_t presize = _PyType_PreHeaderSize(tp);
PyObject *op = gc_alloc(tp, _PyObject_SIZE(tp), presize);
size_t size = _PyObject_SIZE(tp);
if (_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)) {
size += _PyInlineValuesSize(tp);
}
PyObject *op = gc_alloc(tp, size, presize);
if (op == NULL) {
return NULL;
}

54
Python/generated_cases.c.h generated
View File

@ -870,6 +870,7 @@
DEOPT_IF(!PyType_Check(callable), CALL);
PyTypeObject *tp = (PyTypeObject *)callable;
DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL);
assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES);
PyHeapTypeObject *cls = (PyHeapTypeObject *)callable;
PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init;
PyCodeObject *code = (PyCodeObject *)init->func_code;
@ -3680,15 +3681,13 @@
// _CHECK_MANAGED_OBJECT_HAS_VALUES
{
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _LOAD_ATTR_INSTANCE_VALUE
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
attr = _PyDictOrValues_GetValues(dorv)->values[index];
attr = _PyObject_InlineValues(owner)->values[index];
DEOPT_IF(attr == NULL, LOAD_ATTR);
STAT_INC(LOAD_ATTR, hit);
Py_INCREF(attr);
@ -3721,13 +3720,13 @@
}
// _CHECK_ATTR_METHOD_LAZY_DICT
{
Py_ssize_t dictoffset = Py_TYPE(owner)->tp_dictoffset;
assert(dictoffset > 0);
PyObject *dict = *(PyObject **)((char *)owner + dictoffset);
uint16_t dictoffset = read_u16(&this_instr[4].cache);
char *ptr = ((char *)owner) + MANAGED_DICT_OFFSET + dictoffset;
PyObject *dict = *(PyObject **)ptr;
/* This object has a __dict__, just not yet created */
DEOPT_IF(dict != NULL, LOAD_ATTR);
}
/* Skip 2 cache entries */
/* Skip 1 cache entry */
// _LOAD_ATTR_METHOD_LAZY_DICT
{
PyObject *descr = read_obj(&this_instr[6].cache);
@ -3798,9 +3797,8 @@
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
@ -3914,9 +3912,8 @@
}
// _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(!_PyObject_InlineValues(owner)->valid, LOAD_ATTR);
}
// _GUARD_KEYS_VERSION
{
@ -4026,17 +4023,16 @@
// _CHECK_ATTR_WITH_HINT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL, LOAD_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
}
// _LOAD_ATTR_WITH_HINT
{
uint16_t hint = read_u16(&this_instr[4].cache);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR);
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1);
if (DK_IS_UNICODE(dict->ma_keys)) {
@ -5315,19 +5311,20 @@
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
}
// _GUARD_DORV_VALUES
// _GUARD_DORV_NO_DICT
{
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR);
assert(Py_TYPE(owner)->tp_dictoffset < 0);
assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
DEOPT_IF(_PyObject_ManagedDictPointer(owner)->dict, STORE_ATTR);
DEOPT_IF(_PyObject_InlineValues(owner)->valid == 0, STORE_ATTR);
}
// _STORE_ATTR_INSTANCE_VALUE
value = stack_pointer[-2];
{
uint16_t index = read_u16(&this_instr[4].cache);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
STAT_INC(STORE_ATTR, hit);
PyDictValues *values = _PyDictOrValues_GetValues(dorv);
assert(_PyObject_ManagedDictPointer(owner)->dict == NULL);
PyDictValues *values = _PyObject_InlineValues(owner);
PyObject *old_value = values->values[index];
values->values[index] = value;
if (old_value == NULL) {
@ -5389,9 +5386,8 @@
assert(type_version != 0);
DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR);
assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT);
PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner);
DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR);
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
DEOPT_IF(dict == NULL, STORE_ATTR);
assert(PyDict_CheckExact((PyObject *)dict));
PyObject *name = GETITEM(FRAME_CO_NAMES, oparg);

2
Python/optimizer_cases.c.h generated
View File

@ -1104,7 +1104,7 @@
/* _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN is not a viable micro-op for tier 2 */
case _GUARD_DORV_VALUES: {
case _GUARD_DORV_NO_DICT: {
break;
}

51
Python/specialize.c
View File

@ -188,7 +188,7 @@ print_object_stats(FILE *out, ObjectStats *stats)
fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k);
fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values);
fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs);
fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);
fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
@ -197,7 +197,6 @@ print_object_stats(FILE *out, ObjectStats *stats)
fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);
fprintf(out, "Object dematerialize dict: %" PRIu64 "\n", stats->dict_dematerialized);
fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits);
fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses);
fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions);
@ -479,12 +478,11 @@ _PyCode_Quicken(PyCodeObject *code)
#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 18
#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 19
#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 20
#define SPEC_FAIL_ATTR_SHADOWED 21
#define SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD 22
#define SPEC_FAIL_ATTR_CLASS_METHOD_OBJ 23
#define SPEC_FAIL_ATTR_OBJECT_SLOT 24
#define SPEC_FAIL_ATTR_HAS_MANAGED_DICT 25
#define SPEC_FAIL_ATTR_INSTANCE_ATTRIBUTE 26
#define SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE 27
#define SPEC_FAIL_ATTR_PROPERTY_NOT_PY_FUNCTION 28
@ -558,6 +556,7 @@ _PyCode_Quicken(PyCodeObject *code)
#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29
#define SPEC_FAIL_CALL_INIT_NOT_SIMPLE 30
#define SPEC_FAIL_CALL_METACLASS 31
#define SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES 32
/* COMPARE_OP */
#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
@ -829,11 +828,7 @@ specialize_dict_access(
return 0;
}
_PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
if (_PyDictOrValues_IsValues(*dorv) ||
_PyObject_MakeInstanceAttributesFromDict(owner, dorv))
{
// Virtual dictionary
if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) {
PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
assert(PyUnicode_CheckExact(name));
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
@ -850,7 +845,8 @@ specialize_dict_access(
instr->op.code = values_op;
}
else {
PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(*dorv);
PyManagedDictPointer *managed_dict = _PyObject_ManagedDictPointer(owner);
PyDictObject *dict = managed_dict->dict;
if (dict == NULL || !PyDict_CheckExact(dict)) {
SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
return 0;
@ -1258,15 +1254,8 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
assert(descr != NULL);
assert((is_method && kind == METHOD) || (!is_method && kind == NON_DESCRIPTOR));
if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner);
if (owner_cls->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
if (!_PyDictOrValues_IsValues(*dorv) &&
!_PyObject_MakeInstanceAttributesFromDict(owner, dorv))
{
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT);
return 0;
}
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
if (index != DKIX_EMPTY) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED);
@ -1282,10 +1271,16 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
instr->op.code = is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES;
}
else {
Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
if (dictoffset < 0 || dictoffset > INT16_MAX) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
return 0;
Py_ssize_t dictoffset;
if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
dictoffset = MANAGED_DICT_OFFSET;
}
else {
dictoffset = owner_cls->tp_dictoffset;
if (dictoffset < 0 || dictoffset > INT16_MAX + MANAGED_DICT_OFFSET) {
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
return 0;
}
}
if (dictoffset == 0) {
instr->op.code = is_method ? LOAD_ATTR_METHOD_NO_DICT : LOAD_ATTR_NONDESCRIPTOR_NO_DICT;
@ -1296,8 +1291,12 @@ PyObject *descr, DescriptorClassification kind, bool is_method)
SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
return 0;
}
assert(owner_cls->tp_dictoffset > 0);
assert(owner_cls->tp_dictoffset <= INT16_MAX);
/* Cache entries must be unsigned values, so we offset the
* dictoffset by MANAGED_DICT_OFFSET.
* We do the reverese offset in LOAD_ATTR_METHOD_LAZY_DICT */
dictoffset -= MANAGED_DICT_OFFSET;
assert(((uint16_t)dictoffset) == dictoffset);
cache->dict_offset = (uint16_t)dictoffset;
instr->op.code = LOAD_ATTR_METHOD_LAZY_DICT;
}
else {
@ -1729,8 +1728,8 @@ get_init_for_simple_managed_python_class(PyTypeObject *tp)
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN);
return NULL;
}
if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_NO_DICT);
if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) {
SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES);
return NULL;
}
if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) {

7
Tools/cases_generator/analyzer.py
View File

@ -359,11 +359,10 @@ def has_error_without_pop(op: parser.InstDef) -> bool:
NON_ESCAPING_FUNCTIONS = (
"Py_INCREF",
"_PyDictOrValues_IsValues",
"_PyObject_DictOrValuesPointer",
"_PyDictOrValues_GetValues",
"_PyManagedDictPointer_IsValues",
"_PyObject_ManagedDictPointer",
"_PyObject_InlineValues",
"_PyDictValues_AddToInsertionOrder",
"_PyObject_MakeInstanceAttributesFromDict",
"Py_DECREF",
"_Py_DECREF_SPECIALIZED",
"DECREF_INPUTS_AND_REUSE_FLOAT",

14
Tools/gdb/libpython.py
View File

@ -66,10 +66,12 @@ def _type_unsigned_short_ptr():
def _type_unsigned_int_ptr():
return gdb.lookup_type('unsigned int').pointer()
def _sizeof_void_p():
return gdb.lookup_type('void').pointer().sizeof
def _sizeof_pyobject():
return gdb.lookup_type('PyObject').sizeof
def _managed_dict_offset():
# See pycore_object.h
pyobj = gdb.lookup_type("PyObject")
@ -79,6 +81,7 @@ def _managed_dict_offset():
return -3 * _sizeof_void_p()
Py_TPFLAGS_INLINE_VALUES = (1 << 2)
Py_TPFLAGS_MANAGED_DICT = (1 << 4)
Py_TPFLAGS_HEAPTYPE = (1 << 9)
Py_TPFLAGS_LONG_SUBCLASS = (1 << 24)
@ -493,11 +496,12 @@ class HeapTypeObjectPtr(PyObjectPtr):
has_values = int_from_int(typeobj.field('tp_flags')) & Py_TPFLAGS_MANAGED_DICT
if not has_values:
return None
ptr = self._gdbval.cast(_type_char_ptr()) + _managed_dict_offset()
char_ptr = ptr.cast(_type_char_ptr().pointer()).dereference()
if (int(char_ptr) & 1) == 0:
obj_ptr = self._gdbval.cast(_type_char_ptr())
dict_ptr_ptr = obj_ptr + _managed_dict_offset()
dict_ptr = dict_ptr_ptr.cast(_type_char_ptr().pointer()).dereference()
if int(dict_ptr):
return None
char_ptr += 1
char_ptr = obj_ptr + _sizeof_pyobject()
values_ptr = char_ptr.cast(gdb.lookup_type("PyDictValues").pointer())
values = values_ptr['values']
return PyKeysValuesPair(self.get_cached_keys(), values)

5
Tools/scripts/summarize_stats.py
View File

@ -394,7 +394,7 @@ class Stats:
return result
def get_object_stats(self) -> dict[str, tuple[int, int]]:
total_materializations = self._data.get("Object new values", 0)
total_materializations = self._data.get("Object inline values", 0)
total_allocations = self._data.get("Object allocations", 0) + self._data.get(
"Object allocations from freelist", 0
)
@ -1094,8 +1094,7 @@ def object_stats_section() -> Section:
Below, "allocations" means "allocations that are not from a freelist".
Total allocations = "Allocations from freelist" + "Allocations".
"New values" is the number of values arrays created for objects with
managed dicts.
"Inline values" is the number of values arrays inlined into objects.
The cache hit/miss numbers are for the MRO cache, split into dunder and
other names.