Skip to content

Commit c32dc47

Browse files
markshannonmarkshannon
authored
GH-115776: Embed the values array into the object, for "normal" Python objects. (GH-116115)
1 parent c97d3af commit c32dc47

35 files changed

+786
-536
lines changed

‎Include/cpython/pystats.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,11 @@ typedef struct _object_stats {
7777
uint64_t frees;
7878
uint64_t to_freelist;
7979
uint64_t from_freelist;
80-
uint64_t new_values;
80+
uint64_t inline_values;
8181
uint64_t dict_materialized_on_request;
8282
uint64_t dict_materialized_new_key;
8383
uint64_t dict_materialized_too_big;
8484
uint64_t dict_materialized_str_subclass;
85-
uint64_t dict_dematerialized;
8685
uint64_t type_cache_hits;
8786
uint64_t type_cache_misses;
8887
uint64_t type_cache_dunder_hits;

‎Include/internal/pycore_code.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,10 @@ typedef struct {
7979
typedef struct {
8080
uint16_t counter;
8181
uint16_t type_version[2];
82-
uint16_t keys_version[2];
82+
union {
83+
uint16_t keys_version[2];
84+
uint16_t dict_offset;
85+
};
8386
uint16_t descr[4];
8487
} _PyLoadMethodCache;
8588

‎Include/internal/pycore_dict.h

Lines changed: 52 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ extern "C" {
1111

1212
#include "pycore_freelist.h" // _PyFreeListState
1313
#include "pycore_identifier.h" // _Py_Identifier
14-
#include "pycore_object.h" // PyDictOrValues
14+
#include "pycore_object.h" // PyManagedDictPointer
1515

1616
// Unsafe flavor of PyDict_GetItemWithError(): no error checking
1717
extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
@@ -181,6 +181,10 @@ struct _dictkeysobject {
181181
* [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
182182
*/
183183
struct _dictvalues {
184+
uint8_t capacity;
185+
uint8_t size;
186+
uint8_t embedded;
187+
uint8_t valid;
184188
PyObject *values[1];
185189
};
186190

@@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
196200
size_t index = (size_t)1 << dk->dk_log2_index_bytes;
197201
return (&indices[index]);
198202
}
203+
199204
static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
200205
assert(dk->dk_kind == DICT_KEYS_GENERAL);
201206
return (PyDictKeyEntry*)_DK_ENTRIES(dk);
@@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
211216
#define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
212217
#define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
213218

214-
#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
215-
#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
216-
217219
#ifdef Py_GIL_DISABLED
218220
#define DICT_NEXT_VERSION(INTERP) \
219221
(_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
@@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
246248
return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
247249
}
248250

249-
extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
250-
PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
251+
extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
252+
251253
PyAPI_FUNC(PyObject *)_PyDict_FromItems(
252254
PyObject *const *keys, Py_ssize_t keys_offset,
253255
PyObject *const *values, Py_ssize_t values_offset,
254256
Py_ssize_t length);
255257

258+
static inline uint8_t *
259+
get_insertion_order_array(PyDictValues *values)
260+
{
261+
return (uint8_t *)&values->values[values->capacity];
262+
}
263+
256264
static inline void
257265
_PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
258266
{
259267
assert(ix < SHARED_KEYS_MAX_SIZE);
260-
uint8_t *size_ptr = ((uint8_t *)values)-2;
261-
int size = *size_ptr;
262-
assert(size+2 < DICT_VALUES_SIZE(values));
263-
size++;
264-
size_ptr[-size] = (uint8_t)ix;
265-
*size_ptr = size;
268+
int size = values->size;
269+
uint8_t *array = get_insertion_order_array(values);
270+
assert(size < values->capacity);
271+
assert(((uint8_t)ix) == ix);
272+
array[size] = (uint8_t)ix;
273+
values->size = size+1;
274+
}
275+
276+
static inline size_t
277+
shared_keys_usable_size(PyDictKeysObject *keys)
278+
{
279+
#ifdef Py_GIL_DISABLED
280+
// dk_usable will decrease for each instance that is created and each
281+
// value that is added. dk_nentries will increase for each value that
282+
// is added. We want to always return the right value or larger.
283+
// We therefore increase dk_nentries first and we decrease dk_usable
284+
// second, and conversely here we read dk_usable first and dk_entries
285+
// second (to avoid the case where we read entries before the increment
286+
// and read usable after the decrement)
287+
return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
288+
_Py_atomic_load_ssize_acquire(&keys->dk_nentries));
289+
#else
290+
return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
291+
#endif
266292
}
267293

294+
static inline size_t
295+
_PyInlineValuesSize(PyTypeObject *tp)
296+
{
297+
PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
298+
assert(keys != NULL);
299+
size_t size = shared_keys_usable_size(keys);
300+
size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
301+
assert(prefix_size < 256);
302+
return prefix_size + (size + 1) * sizeof(PyObject *);
303+
}
304+
305+
int
306+
_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
307+
268308
#ifdef __cplusplus
269309
}
270310
#endif

‎Include/internal/pycore_object.h

Lines changed: 13 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -265,9 +265,8 @@ _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
265265
{
266266
assert(op != NULL);
267267
Py_SET_TYPE(op, typeobj);
268-
if (_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE)) {
269-
Py_INCREF(typeobj);
270-
}
268+
assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
269+
Py_INCREF(typeobj);
271270
_Py_NewReference(op);
272271
}
273272

@@ -611,8 +610,7 @@ extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
611610
extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
612611
extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
613612

614-
extern int _PyObject_InitializeDict(PyObject *obj);
615-
int _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
613+
void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
616614
extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values,
617615
PyObject *name, PyObject *value);
618616
PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
@@ -627,46 +625,26 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
627625
#endif
628626

629627
typedef union {
630-
PyObject *dict;
631-
/* Use a char* to generate a warning if directly assigning a PyDictValues */
632-
char *values;
633-
} PyDictOrValues;
628+
PyDictObject *dict;
629+
} PyManagedDictPointer;
634630

635-
static inline PyDictOrValues *
636-
_PyObject_DictOrValuesPointer(PyObject *obj)
631+
static inline PyManagedDictPointer *
632+
_PyObject_ManagedDictPointer(PyObject *obj)
637633
{
638634
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
639-
return (PyDictOrValues *)((char *)obj + MANAGED_DICT_OFFSET);
640-
}
641-
642-
static inline int
643-
_PyDictOrValues_IsValues(PyDictOrValues dorv)
644-
{
645-
return ((uintptr_t)dorv.values) & 1;
635+
return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
646636
}
647637

648638
static inline PyDictValues *
649-
_PyDictOrValues_GetValues(PyDictOrValues dorv)
650-
{
651-
assert(_PyDictOrValues_IsValues(dorv));
652-
return (PyDictValues *)(dorv.values + 1);
653-
}
654-
655-
static inline PyObject *
656-
_PyDictOrValues_GetDict(PyDictOrValues dorv)
639+
_PyObject_InlineValues(PyObject *obj)
657640
{
658-
assert(!_PyDictOrValues_IsValues(dorv));
659-
return dorv.dict;
660-
}
661-
662-
static inline void
663-
_PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values)
664-
{
665-
ptr->values = ((char *)values) - 1;
641+
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
642+
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
643+
assert(Py_TYPE(obj)->tp_basicsize == sizeof(PyObject));
644+
return (PyDictValues *)((char *)obj + sizeof(PyObject));
666645
}
667646

668647
extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
669-
extern void _PyObject_FreeInstanceAttributes(PyObject *obj);
670648
extern int _PyObject_IsInstanceDictEmpty(PyObject *);
671649

672650
// Export for 'math' shared extension

‎Include/internal/pycore_opcode_metadata.h

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_uop_ids.h

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/internal/pycore_uop_metadata.h

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Include/object.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,13 +629,18 @@ given type object has a specified feature.
629629
/* Track types initialized using _PyStaticType_InitBuiltin(). */
630630
#define _Py_TPFLAGS_STATIC_BUILTIN (1 << 1)
631631

632+
/* The values array is placed inline directly after the rest of
633+
* the object. Implies Py_TPFLAGS_HAVE_GC.
634+
*/
635+
#define Py_TPFLAGS_INLINE_VALUES (1 << 2)
636+
632637
/* Placement of weakref pointers are managed by the VM, not by the type.
633638
* The VM will automatically set tp_weaklistoffset.
634639
*/
635640
#define Py_TPFLAGS_MANAGED_WEAKREF (1 << 3)
636641

637642
/* Placement of dict (and values) pointers are managed by the VM, not by the type.
638-
* The VM will automatically set tp_dictoffset.
643+
* The VM will automatically set tp_dictoffset. Implies Py_TPFLAGS_HAVE_GC.
639644
*/
640645
#define Py_TPFLAGS_MANAGED_DICT (1 << 4)
641646

‎Lib/test/test_capi/test_mem.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,8 +148,8 @@ class C(): pass
148148
self.assertIn(b'MemoryError', out)
149149
*_, count = line.split(b' ')
150150
count = int(count)
151-
self.assertLessEqual(count, i*5)
152-
self.assertGreaterEqual(count, i*5-2)
151+
self.assertLessEqual(count, i*10)
152+
self.assertGreaterEqual(count, i*10-4)
153153

154154

155155
# Py_GIL_DISABLED requires mimalloc (not malloc)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy