Where Online Learning is simpler!
The C and C++ Include Header Files
cat -n /usr/include/python3.14/internal/pycore_object.h
1 #ifndef Py_INTERNAL_OBJECT_H 2 #define Py_INTERNAL_OBJECT_H 3 #ifdef __cplusplus 4 extern "C" { 5 #endif 6 7 #ifndef Py_BUILD_CORE 8 # error "this header requires Py_BUILD_CORE define" 9 #endif 10 11 #include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall() 12 #include "pycore_gc.h" // _PyObject_GC_TRACK() 13 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE() 14 #include "pycore_pystate.h" // _PyInterpreterState_GET() 15 #include "pycore_runtime.h" // _PyRuntime 16 #include "pycore_typeobject.h" // _PyStaticType_GetState() 17 #include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow() 18 19 #include <stdbool.h> // bool 20 21 22 // This value is added to `ob_ref_shared` for objects that use deferred 23 // reference counting so that they are not immediately deallocated when the 24 // non-deferred reference count drops to zero. 25 // 26 // The value is half the maximum shared refcount because the low two bits of 27 // `ob_ref_shared` are used for flags. 28 #define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8) 29 30 /* For backwards compatibility -- Do not use this */ 31 #define _Py_IsImmortalLoose(op) _Py_IsImmortal 32 33 34 /* Check if an object is consistent. For example, ensure that the reference 35 counter is greater than or equal to 1, and ensure that ob_type is not NULL. 36 37 Call _PyObject_AssertFailed() if the object is inconsistent. 38 39 If check_content is zero, only check header fields: reduce the overhead. 40 41 The function always return 1. The return value is just here to be able to 42 write: 43 44 assert(_PyObject_CheckConsistency(obj, 1)); */ 45 extern int _PyObject_CheckConsistency(PyObject *op, int check_content); 46 47 extern void _PyDebugAllocatorStats(FILE *out, const char *block_name, 48 int num_blocks, size_t sizeof_block); 49 50 extern void _PyObject_DebugTypeStats(FILE *out); 51 52 #ifdef Py_TRACE_REFS 53 // Forget a reference registered by _Py_NewReference(). Function called by 54 // _Py_Dealloc(). 55 // 56 // On a free list, the function can be used before modifying an object to 57 // remove the object from traced objects. Then _Py_NewReference() or 58 // _Py_NewReferenceNoTotal() should be called again on the object to trace 59 // it again. 60 extern void _Py_ForgetReference(PyObject *); 61 #endif 62 63 // Export for shared _testinternalcapi extension 64 PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); 65 66 /* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid 67 designated initializer conflicts in C++20. If we use the definition in 68 object.h, we will be mixing designated and non-designated initializers in 69 pycore objects which is forbiddent in C++20. However, if we then use 70 designated initializers in object.h then Extensions without designated break. 71 Furthermore, we can't use designated initializers in Extensions since these 72 are not supported pre-C++20. Thus, keeping an internal copy here is the most 73 backwards compatible solution */ 74 #if defined(Py_GIL_DISABLED) 75 #define _PyObject_HEAD_INIT(type) \ 76 { \ 77 .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ 78 .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ 79 .ob_gc_bits = _PyGC_BITS_DEFERRED, \ 80 .ob_type = (type) \ 81 } 82 #else 83 #if SIZEOF_VOID_P > 4 84 #define _PyObject_HEAD_INIT(type) \ 85 { \ 86 .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ 87 .ob_flags = _Py_STATIC_FLAG_BITS, \ 88 .ob_type = (type) \ 89 } 90 #else 91 #define _PyObject_HEAD_INIT(type) \ 92 { \ 93 .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \ 94 .ob_type = (type) \ 95 } 96 #endif 97 #endif 98 #define _PyVarObject_HEAD_INIT(type, size) \ 99 { \ 100 .ob_base = _PyObject_HEAD_INIT(type), \ 101 .ob_size = size \ 102 } 103 104 PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( 105 const char *func, 106 const char *message); 107 108 #define _Py_FatalRefcountError(message) \ 109 _Py_FatalRefcountErrorFunc(__func__, (message)) 110 111 #define _PyReftracerTrack(obj, operation) \ 112 do { \ 113 struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \ 114 if (tracer->tracer_func != NULL) { \ 115 void *data = tracer->tracer_data; \ 116 tracer->tracer_func((obj), (operation), data); \ 117 } \ 118 } while(0) 119 120 #ifdef Py_REF_DEBUG 121 /* The symbol is only exposed in the API for the sake of extensions 122 built against the pre-3.12 stable ABI. */ 123 PyAPI_DATA(Py_ssize_t) _Py_RefTotal; 124 125 extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t); 126 extern PyAPI_FUNC(void) _Py_IncRefTotal(PyThreadState *); 127 extern PyAPI_FUNC(void) _Py_DecRefTotal(PyThreadState *); 128 129 # define _Py_DEC_REFTOTAL(interp) \ 130 interp->object_state.reftotal-- 131 #endif 132 133 // Increment reference count by n 134 static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) 135 { 136 if (_Py_IsImmortal(op)) { 137 _Py_INCREF_IMMORTAL_STAT_INC(); 138 return; 139 } 140 #ifndef Py_GIL_DISABLED 141 Py_ssize_t refcnt = _Py_REFCNT(op); 142 Py_ssize_t new_refcnt = refcnt + n; 143 if (new_refcnt >= (Py_ssize_t)_Py_IMMORTAL_MINIMUM_REFCNT) { 144 new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; 145 } 146 # if SIZEOF_VOID_P > 4 147 op->ob_refcnt = (PY_UINT32_T)new_refcnt; 148 # else 149 op->ob_refcnt = new_refcnt; 150 # endif 151 # ifdef Py_REF_DEBUG 152 _Py_AddRefTotal(_PyThreadState_GET(), new_refcnt - refcnt); 153 # endif 154 #else 155 if (_Py_IsOwnedByCurrentThread(op)) { 156 uint32_t local = op->ob_ref_local; 157 Py_ssize_t refcnt = (Py_ssize_t)local + n; 158 # if PY_SSIZE_T_MAX > UINT32_MAX 159 if (refcnt > (Py_ssize_t)UINT32_MAX) { 160 // Make the object immortal if the 32-bit local reference count 161 // would overflow. 162 refcnt = _Py_IMMORTAL_REFCNT_LOCAL; 163 } 164 # endif 165 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, (uint32_t)refcnt); 166 } 167 else { 168 _Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT)); 169 } 170 # ifdef Py_REF_DEBUG 171 _Py_AddRefTotal(_PyThreadState_GET(), n); 172 # endif 173 #endif 174 // Although the ref count was increased by `n` (which may be greater than 1) 175 // it is only a single increment (i.e. addition) operation, so only 1 refcnt 176 // increment operation is counted. 177 _Py_INCREF_STAT_INC(); 178 } 179 #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) 180 181 // Checks if an object has a single, unique reference. If the caller holds a 182 // unique reference, it may be able to safely modify the object in-place. 183 static inline int 184 _PyObject_IsUniquelyReferenced(PyObject *ob) 185 { 186 #if !defined(Py_GIL_DISABLED) 187 return Py_REFCNT(ob) == 1; 188 #else 189 // NOTE: the entire ob_ref_shared field must be zero, including flags, to 190 // ensure that other threads cannot concurrently create new references to 191 // this object. 192 return (_Py_IsOwnedByCurrentThread(ob) && 193 _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 && 194 _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0); 195 #endif 196 } 197 198 PyAPI_FUNC(void) _Py_SetImmortal(PyObject *op); 199 PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); 200 201 // Makes an immortal object mortal again with the specified refcnt. Should only 202 // be used during runtime finalization. 203 static inline void _Py_SetMortal(PyObject *op, short refcnt) 204 { 205 if (op) { 206 assert(_Py_IsImmortal(op)); 207 #ifdef Py_GIL_DISABLED 208 op->ob_tid = _Py_UNOWNED_TID; 209 op->ob_ref_local = 0; 210 op->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); 211 #else 212 op->ob_refcnt = refcnt; 213 #endif 214 } 215 } 216 217 /* _Py_ClearImmortal() should only be used during runtime finalization. */ 218 static inline void _Py_ClearImmortal(PyObject *op) 219 { 220 if (op) { 221 _Py_SetMortal(op, 1); 222 Py_DECREF(op); 223 } 224 } 225 #define _Py_ClearImmortal(op) \ 226 do { \ 227 _Py_ClearImmortal(_PyObject_CAST(op)); \ 228 op = NULL; \ 229 } while (0) 230 231 #if !defined(Py_GIL_DISABLED) 232 static inline void 233 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) 234 { 235 if (_Py_IsImmortal(op)) { 236 _Py_DECREF_IMMORTAL_STAT_INC(); 237 return; 238 } 239 _Py_DECREF_STAT_INC(); 240 #ifdef Py_REF_DEBUG 241 _Py_DEC_REFTOTAL(PyInterpreterState_Get()); 242 #endif 243 if (--op->ob_refcnt != 0) { 244 assert(op->ob_refcnt > 0); 245 } 246 else { 247 #ifdef Py_TRACE_REFS 248 _Py_ForgetReference(op); 249 #endif 250 _PyReftracerTrack(op, PyRefTracer_DESTROY); 251 destruct(op); 252 } 253 } 254 255 static inline void 256 _Py_DECREF_NO_DEALLOC(PyObject *op) 257 { 258 if (_Py_IsImmortal(op)) { 259 _Py_DECREF_IMMORTAL_STAT_INC(); 260 return; 261 } 262 _Py_DECREF_STAT_INC(); 263 #ifdef Py_REF_DEBUG 264 _Py_DEC_REFTOTAL(PyInterpreterState_Get()); 265 #endif 266 op->ob_refcnt--; 267 #ifdef Py_DEBUG 268 if (op->ob_refcnt <= 0) { 269 _Py_FatalRefcountError("Expected a positive remaining refcount"); 270 } 271 #endif 272 } 273 274 #else 275 // TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build 276 static inline void 277 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) 278 { 279 Py_DECREF(op); 280 } 281 282 static inline void 283 _Py_DECREF_NO_DEALLOC(PyObject *op) 284 { 285 Py_DECREF(op); 286 } 287 288 static inline int 289 _Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) 290 { 291 return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; 292 } 293 294 static inline int 295 _Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) 296 { 297 return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; 298 } 299 300 // Merge the local and shared reference count fields and add `extra` to the 301 // refcount when merging. 302 Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); 303 #endif // !defined(Py_GIL_DISABLED) 304 305 #ifdef Py_REF_DEBUG 306 # undef _Py_DEC_REFTOTAL 307 #endif 308 309 310 extern int _PyType_CheckConsistency(PyTypeObject *type); 311 extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); 312 313 // Fast inlined version of PyType_HasFeature() 314 static inline int 315 _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { 316 return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); 317 } 318 319 extern void _PyType_InitCache(PyInterpreterState *interp); 320 321 extern PyStatus _PyObject_InitState(PyInterpreterState *interp); 322 extern void _PyObject_FiniState(PyInterpreterState *interp); 323 extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); 324 325 // Macros used for per-thread reference counting in the free threading build. 326 // They resolve to normal Py_INCREF/DECREF calls in the default build. 327 // 328 // The macros are used for only a few references that would otherwise cause 329 // scaling bottlenecks in the free threading build: 330 // - The reference from an object to `ob_type`. 331 // - The reference from a function to `func_code`. 332 // - The reference from a function to `func_globals` and `func_builtins`. 333 // 334 // It's safe, but not performant or necessary, to use these macros for other 335 // references to code, type, or dict objects. It's also safe to mix their 336 // usage with normal Py_INCREF/DECREF calls. 337 // 338 // See also Include/internal/pycore_dict.h for _Py_INCREF_DICT/_Py_DECREF_DICT. 339 #ifndef Py_GIL_DISABLED 340 # define _Py_INCREF_TYPE Py_INCREF 341 # define _Py_DECREF_TYPE Py_DECREF 342 # define _Py_INCREF_CODE Py_INCREF 343 # define _Py_DECREF_CODE Py_DECREF 344 #else 345 static inline void 346 _Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) 347 { 348 _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); 349 350 // The table index is `unique_id - 1` because 0 is not a valid unique id. 351 // Unsigned comparison so that `idx=-1` is handled by the "else". 352 size_t idx = (size_t)(unique_id - 1); 353 if (idx < (size_t)tstate->refcounts.size) { 354 # ifdef Py_REF_DEBUG 355 _Py_INCREF_IncRefTotal(); 356 # endif 357 _Py_INCREF_STAT_INC(); 358 tstate->refcounts.values[idx]++; 359 } 360 else { 361 // The slow path resizes the per-thread refcount array if necessary. 362 // It handles the unique_id=0 case to keep the inlinable function smaller. 363 _PyObject_ThreadIncrefSlow(obj, idx); 364 } 365 } 366 367 static inline void 368 _Py_INCREF_TYPE(PyTypeObject *type) 369 { 370 if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { 371 assert(_Py_IsImmortal(type)); 372 _Py_INCREF_IMMORTAL_STAT_INC(); 373 return; 374 } 375 376 // gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when 377 // _Py_INCREF_TYPE() is called on a statically allocated type. The 378 // _PyType_HasFeature check above ensures that the type is a heap type. 379 #if defined(__GNUC__) && __GNUC__ >= 11 380 # pragma GCC diagnostic push 381 # pragma GCC diagnostic ignored "-Warray-bounds" 382 #endif 383 _Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id); 384 #if defined(__GNUC__) && __GNUC__ >= 11 385 # pragma GCC diagnostic pop 386 #endif 387 } 388 389 static inline void 390 _Py_INCREF_CODE(PyCodeObject *co) 391 { 392 _Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id); 393 } 394 395 static inline void 396 _Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) 397 { 398 _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); 399 400 // The table index is `unique_id - 1` because 0 is not a valid unique id. 401 // Unsigned comparison so that `idx=-1` is handled by the "else". 402 size_t idx = (size_t)(unique_id - 1); 403 if (idx < (size_t)tstate->refcounts.size) { 404 # ifdef Py_REF_DEBUG 405 _Py_DECREF_DecRefTotal(); 406 # endif 407 _Py_DECREF_STAT_INC(); 408 tstate->refcounts.values[idx]--; 409 } 410 else { 411 // Directly decref the object if the id is not assigned or if 412 // per-thread refcounting has been disabled on this object. 413 Py_DECREF(obj); 414 } 415 } 416 417 static inline void 418 _Py_DECREF_TYPE(PyTypeObject *type) 419 { 420 if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { 421 assert(_Py_IsImmortal(type)); 422 _Py_DECREF_IMMORTAL_STAT_INC(); 423 return; 424 } 425 PyHeapTypeObject *ht = (PyHeapTypeObject *)type; 426 _Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id); 427 } 428 429 static inline void 430 _Py_DECREF_CODE(PyCodeObject *co) 431 { 432 _Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id); 433 } 434 #endif 435 436 #ifndef Py_GIL_DISABLED 437 #ifdef Py_REF_DEBUG 438 439 static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *op) 440 { 441 if (op->ob_refcnt <= 0) { 442 _Py_NegativeRefcount(filename, lineno, op); 443 } 444 _Py_DECREF_STAT_INC(); 445 assert(!_Py_IsStaticImmortal(op)); 446 if (!_Py_IsImmortal(op)) { 447 _Py_DECREF_DecRefTotal(); 448 } 449 if (--op->ob_refcnt == 0) { 450 _Py_Dealloc(op); 451 } 452 } 453 #define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(__FILE__, __LINE__, _PyObject_CAST(op)) 454 455 static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int lineno, PyObject *op, destructor destruct) 456 { 457 if (op->ob_refcnt <= 0) { 458 _Py_NegativeRefcount(filename, lineno, op); 459 } 460 _Py_DECREF_STAT_INC(); 461 assert(!_Py_IsStaticImmortal(op)); 462 if (!_Py_IsImmortal(op)) { 463 _Py_DECREF_DecRefTotal(); 464 } 465 if (--op->ob_refcnt == 0) { 466 #ifdef Py_TRACE_REFS 467 _Py_ForgetReference(op); 468 #endif 469 _PyReftracerTrack(op, PyRefTracer_DESTROY); 470 destruct(op); 471 } 472 } 473 #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) _Py_DECREF_MORTAL_SPECIALIZED(__FILE__, __LINE__, op, destruct) 474 475 #else 476 477 static inline void Py_DECREF_MORTAL(PyObject *op) 478 { 479 assert(!_Py_IsStaticImmortal(op)); 480 _Py_DECREF_STAT_INC(); 481 if (--op->ob_refcnt == 0) { 482 _Py_Dealloc(op); 483 } 484 } 485 #define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(_PyObject_CAST(op)) 486 487 static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruct) 488 { 489 assert(!_Py_IsStaticImmortal(op)); 490 _Py_DECREF_STAT_INC(); 491 if (--op->ob_refcnt == 0) { 492 _PyReftracerTrack(op, PyRefTracer_DESTROY); 493 destruct(op); 494 } 495 } 496 #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct) 497 498 #endif 499 #endif 500 501 /* Inline functions trading binary compatibility for speed: 502 _PyObject_Init() is the fast version of PyObject_Init(), and 503 _PyObject_InitVar() is the fast version of PyObject_InitVar(). 504 505 These inline functions must not be called with op=NULL. */ 506 static inline void 507 _PyObject_Init(PyObject *op, PyTypeObject *typeobj) 508 { 509 assert(op != NULL); 510 Py_SET_TYPE(op, typeobj); 511 assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); 512 _Py_INCREF_TYPE(typeobj); 513 _Py_NewReference(op); 514 } 515 516 static inline void 517 _PyObject_InitVar(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size) 518 { 519 assert(op != NULL); 520 assert(typeobj != &PyLong_Type); 521 _PyObject_Init((PyObject *)op, typeobj); 522 Py_SET_SIZE(op, size); 523 } 524 525 // Macros to accept any type for the parameter, and to automatically pass 526 // the filename and the filename (if NDEBUG is not defined) where the macro 527 // is called. 528 #ifdef NDEBUG 529 # define _PyObject_GC_TRACK(op) \ 530 _PyObject_GC_TRACK(_PyObject_CAST(op)) 531 # define _PyObject_GC_UNTRACK(op) \ 532 _PyObject_GC_UNTRACK(_PyObject_CAST(op)) 533 #else 534 # define _PyObject_GC_TRACK(op) \ 535 _PyObject_GC_TRACK(__FILE__, __LINE__, _PyObject_CAST(op)) 536 # define _PyObject_GC_UNTRACK(op) \ 537 _PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op)) 538 #endif 539 540 #ifdef Py_GIL_DISABLED 541 542 /* Tries to increment an object's reference count 543 * 544 * This is a specialized version of _Py_TryIncref that only succeeds if the 545 * object is immortal or local to this thread. It does not handle the case 546 * where the reference count modification requires an atomic operation. This 547 * allows call sites to specialize for the immortal/local case. 548 */ 549 static inline int 550 _Py_TryIncrefFast(PyObject *op) { 551 uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); 552 local += 1; 553 if (local == 0) { 554 // immortal 555 _Py_INCREF_IMMORTAL_STAT_INC(); 556 return 1; 557 } 558 if (_Py_IsOwnedByCurrentThread(op)) { 559 _Py_INCREF_STAT_INC(); 560 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); 561 #ifdef Py_REF_DEBUG 562 _Py_IncRefTotal(_PyThreadState_GET()); 563 #endif 564 return 1; 565 } 566 return 0; 567 } 568 569 static inline int 570 _Py_TryIncRefShared(PyObject *op) 571 { 572 Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); 573 for (;;) { 574 // If the shared refcount is zero and the object is either merged 575 // or may not have weak references, then we cannot incref it. 576 if (shared == 0 || shared == _Py_REF_MERGED) { 577 return 0; 578 } 579 580 if (_Py_atomic_compare_exchange_ssize( 581 &op->ob_ref_shared, 582 &shared, 583 shared + (1 << _Py_REF_SHARED_SHIFT))) { 584 #ifdef Py_REF_DEBUG 585 _Py_IncRefTotal(_PyThreadState_GET()); 586 #endif 587 _Py_INCREF_STAT_INC(); 588 return 1; 589 } 590 } 591 } 592 593 /* Tries to incref the object op and ensures that *src still points to it. */ 594 static inline int 595 _Py_TryIncrefCompare(PyObject **src, PyObject *op) 596 { 597 if (_Py_TryIncrefFast(op)) { 598 return 1; 599 } 600 if (!_Py_TryIncRefShared(op)) { 601 return 0; 602 } 603 if (op != _Py_atomic_load_ptr(src)) { 604 Py_DECREF(op); 605 return 0; 606 } 607 return 1; 608 } 609 610 /* Loads and increfs an object from ptr, which may contain a NULL value. 611 Safe with concurrent (atomic) updates to ptr. 612 NOTE: The writer must set maybe-weakref on the stored object! */ 613 static inline PyObject * 614 _Py_XGetRef(PyObject **ptr) 615 { 616 for (;;) { 617 PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); 618 if (value == NULL) { 619 return value; 620 } 621 if (_Py_TryIncrefCompare(ptr, value)) { 622 return value; 623 } 624 } 625 } 626 627 /* Attempts to loads and increfs an object from ptr. Returns NULL 628 on failure, which may be due to a NULL value or a concurrent update. */ 629 static inline PyObject * 630 _Py_TryXGetRef(PyObject **ptr) 631 { 632 PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); 633 if (value == NULL) { 634 return value; 635 } 636 if (_Py_TryIncrefCompare(ptr, value)) { 637 return value; 638 } 639 return NULL; 640 } 641 642 /* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF 643 on objects owned by a different thread. */ 644 static inline PyObject * 645 _Py_NewRefWithLock(PyObject *op) 646 { 647 if (_Py_TryIncrefFast(op)) { 648 return op; 649 } 650 #ifdef Py_REF_DEBUG 651 _Py_IncRefTotal(_PyThreadState_GET()); 652 #endif 653 _Py_INCREF_STAT_INC(); 654 for (;;) { 655 Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); 656 Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT); 657 if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) { 658 new_shared |= _Py_REF_MAYBE_WEAKREF; 659 } 660 if (_Py_atomic_compare_exchange_ssize( 661 &op->ob_ref_shared, 662 &shared, 663 new_shared)) { 664 return op; 665 } 666 } 667 } 668 669 static inline PyObject * 670 _Py_XNewRefWithLock(PyObject *obj) 671 { 672 if (obj == NULL) { 673 return NULL; 674 } 675 return _Py_NewRefWithLock(obj); 676 } 677 678 static inline void 679 _PyObject_SetMaybeWeakref(PyObject *op) 680 { 681 if (_Py_IsImmortal(op)) { 682 return; 683 } 684 for (;;) { 685 Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); 686 if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { 687 // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. 688 return; 689 } 690 if (_Py_atomic_compare_exchange_ssize( 691 &op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { 692 return; 693 } 694 } 695 } 696 697 extern PyAPI_FUNC(int) _PyObject_ResurrectEndSlow(PyObject *op); 698 #endif 699 700 // Temporarily resurrects an object during deallocation. The refcount is set 701 // to one. 702 static inline void 703 _PyObject_ResurrectStart(PyObject *op) 704 { 705 assert(Py_REFCNT(op) == 0); 706 #ifdef Py_REF_DEBUG 707 _Py_IncRefTotal(_PyThreadState_GET()); 708 #endif 709 #ifdef Py_GIL_DISABLED 710 _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_ThreadId()); 711 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 1); 712 _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); 713 #else 714 Py_SET_REFCNT(op, 1); 715 #endif 716 #ifdef Py_TRACE_REFS 717 _Py_ResurrectReference(op); 718 #endif 719 } 720 721 // Undoes an object resurrection by decrementing the refcount without calling 722 // _Py_Dealloc(). Returns 0 if the object is dead (the normal case), and 723 // deallocation should continue. Returns 1 if the object is still alive. 724 static inline int 725 _PyObject_ResurrectEnd(PyObject *op) 726 { 727 #ifdef Py_REF_DEBUG 728 _Py_DecRefTotal(_PyThreadState_GET()); 729 #endif 730 #ifndef Py_GIL_DISABLED 731 Py_SET_REFCNT(op, Py_REFCNT(op) - 1); 732 if (Py_REFCNT(op) == 0) { 733 # ifdef Py_TRACE_REFS 734 _Py_ForgetReference(op); 735 # endif 736 return 0; 737 } 738 return 1; 739 #else 740 uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); 741 Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); 742 if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) { 743 // Fast-path: object has a single refcount and is owned by this thread 744 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); 745 # ifdef Py_TRACE_REFS 746 _Py_ForgetReference(op); 747 # endif 748 return 0; 749 } 750 // Slow-path: object has a shared refcount or is not owned by this thread 751 return _PyObject_ResurrectEndSlow(op); 752 #endif 753 } 754 755 /* Tries to incref op and returns 1 if successful or 0 otherwise. */ 756 static inline int 757 _Py_TryIncref(PyObject *op) 758 { 759 #ifdef Py_GIL_DISABLED 760 return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op); 761 #else 762 if (Py_REFCNT(op) > 0) { 763 Py_INCREF(op); 764 return 1; 765 } 766 return 0; 767 #endif 768 } 769 770 #ifdef Py_REF_DEBUG 771 extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); 772 extern void _Py_FinalizeRefTotal(_PyRuntimeState *); 773 extern void _PyDebug_PrintTotalRefs(void); 774 #endif 775 776 #ifdef Py_TRACE_REFS 777 extern void _Py_AddToAllObjects(PyObject *op); 778 extern void _Py_PrintReferences(PyInterpreterState *, FILE *); 779 extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); 780 #endif 781 782 783 /* Return the *address* of the object's weaklist. The address may be 784 * dereferenced to get the current head of the weaklist. This is useful 785 * for iterating over the linked list of weakrefs, especially when the 786 * list is being modified externally (e.g. refs getting removed). 787 * 788 * The returned pointer should not be used to change the head of the list 789 * nor should it be used to add, remove, or swap any refs in the list. 790 * That is the sole responsibility of the code in weakrefobject.c. 791 */ 792 static inline PyObject ** 793 _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) 794 { 795 if (PyType_Check(op) && 796 ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { 797 PyInterpreterState *interp = _PyInterpreterState_GET(); 798 managed_static_type_state *state = _PyStaticType_GetState( 799 interp, (PyTypeObject *)op); 800 return _PyStaticType_GET_WEAKREFS_LISTPTR(state); 801 } 802 // Essentially _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(): 803 Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset; 804 return (PyObject **)((char *)op + offset); 805 } 806 807 /* This is a special case of _PyObject_GET_WEAKREFS_LISTPTR(). 808 * Only the most fundamental lookup path is used. 809 * Consequently, static types should not be used. 810 * 811 * For static builtin types the returned pointer will always point 812 * to a NULL tp_weaklist. This is fine for any deallocation cases, 813 * since static types are never deallocated and static builtin types 814 * are only finalized at the end of runtime finalization. 815 * 816 * If the weaklist for static types is actually needed then use 817 * _PyObject_GET_WEAKREFS_LISTPTR(). 818 */ 819 static inline PyWeakReference ** 820 _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op) 821 { 822 assert(!PyType_Check(op) || 823 ((PyTypeObject *)op)->tp_flags & Py_TPFLAGS_HEAPTYPE); 824 Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset; 825 return (PyWeakReference **)((char *)op + offset); 826 } 827 828 // Fast inlined version of PyType_IS_GC() 829 #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) 830 831 // Fast inlined version of PyObject_IS_GC() 832 static inline int 833 _PyObject_IS_GC(PyObject *obj) 834 { 835 PyTypeObject *type = Py_TYPE(obj); 836 return (_PyType_IS_GC(type) 837 && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); 838 } 839 840 // Fast inlined version of PyObject_Hash() 841 static inline Py_hash_t 842 _PyObject_HashFast(PyObject *op) 843 { 844 if (PyUnicode_CheckExact(op)) { 845 Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( 846 _PyASCIIObject_CAST(op)->hash); 847 if (hash != -1) { 848 return hash; 849 } 850 } 851 return PyObject_Hash(op); 852 } 853 854 static inline size_t 855 _PyType_PreHeaderSize(PyTypeObject *tp) 856 { 857 return ( 858 #ifndef Py_GIL_DISABLED 859 (size_t)_PyType_IS_GC(tp) * sizeof(PyGC_Head) + 860 #endif 861 (size_t)_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) 862 ); 863 } 864 865 void _PyObject_GC_Link(PyObject *op); 866 867 // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL)); 868 extern int _Py_CheckSlotResult( 869 PyObject *obj, 870 const char *slot_name, 871 int success); 872 873 // Test if a type supports weak references 874 static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { 875 return (type->tp_weaklistoffset != 0); 876 } 877 878 extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); 879 PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); 880 881 extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); 882 extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); 883 extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); 884 extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); 885 886 void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); 887 extern int _PyObject_StoreInstanceAttribute(PyObject *obj, 888 PyObject *name, PyObject *value); 889 extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, 890 PyObject **attr); 891 extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, 892 unsigned int *); 893 894 // Internal API to look for a name through the MRO. 895 // This stores a stack reference in out and returns the value of 896 // type->tp_version or zero if name is missing. It doesn't set an exception! 897 extern unsigned int 898 _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); 899 900 extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self, 901 PyObject *name, _PyStackRef *method); 902 903 // Cache the provided init method in the specialization cache of type if the 904 // provided type version matches the current version of the type. 905 // 906 // The cached value is borrowed and is only valid if guarded by a type 907 // version check. In free-threaded builds the init method must also use 908 // deferred reference counting. 909 // 910 // Returns 1 if the value was cached or 0 otherwise. 911 extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, 912 PyObject *init, 913 unsigned int tp_version); 914 915 #ifdef Py_GIL_DISABLED 916 # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) 917 # define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2) 918 #else 919 # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-3) 920 # define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) 921 #endif 922 923 typedef union { 924 PyDictObject *dict; 925 } PyManagedDictPointer; 926 927 static inline PyManagedDictPointer * 928 _PyObject_ManagedDictPointer(PyObject *obj) 929 { 930 assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); 931 return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); 932 } 933 934 static inline PyDictObject * 935 _PyObject_GetManagedDict(PyObject *obj) 936 { 937 PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj); 938 return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict); 939 } 940 941 static inline PyDictValues * 942 _PyObject_InlineValues(PyObject *obj) 943 { 944 PyTypeObject *tp = Py_TYPE(obj); 945 assert(tp->tp_basicsize > 0 && (size_t)tp->tp_basicsize % sizeof(PyObject *) == 0); 946 assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); 947 assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); 948 return (PyDictValues *)((char *)obj + tp->tp_basicsize); 949 } 950 951 extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); 952 extern int _PyObject_IsInstanceDictEmpty(PyObject *); 953 954 // Export for 'math' shared extension 955 PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); 956 PyAPI_FUNC(int) _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self); 957 958 // Calls the method named `attr` on `self`, but does not set an exception if 959 // the attribute does not exist. 960 PyAPI_FUNC(PyObject *) 961 _PyObject_MaybeCallSpecialNoArgs(PyObject *self, PyObject *attr); 962 963 PyAPI_FUNC(PyObject *) 964 _PyObject_MaybeCallSpecialOneArg(PyObject *self, PyObject *attr, PyObject *arg); 965 966 extern int _PyObject_IsAbstract(PyObject *); 967 968 PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); 969 extern PyObject* _PyObject_NextNotImplemented(PyObject *); 970 971 // Pickle support. 972 // Export for '_datetime' shared extension 973 PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); 974 975 /* C function call trampolines to mitigate bad function pointer casts. 976 * 977 * Typical native ABIs ignore additional arguments or fill in missing 978 * values with 0/NULL in function pointer cast. Compilers do not show 979 * warnings when a function pointer is explicitly casted to an 980 * incompatible type. 981 * 982 * Bad fpcasts are an issue in WebAssembly. WASM's indirect_call has strict 983 * function signature checks. Argument count, types, and return type must 984 * match. 985 * 986 * Third party code unintentionally rely on problematic fpcasts. The call 987 * trampoline mitigates common occurrences of bad fpcasts on Emscripten. 988 */ 989 #if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE)) 990 #define _PyCFunction_TrampolineCall(meth, self, args) \ 991 (meth)((self), (args)) 992 #define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ 993 (meth)((self), (args), (kw)) 994 #endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE 995 996 // Export these 2 symbols for '_pickle' shared extension 997 PyAPI_DATA(PyTypeObject) _PyNone_Type; 998 PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; 999 1000 // Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. 1001 // Export for the stable ABI. 1002 PyAPI_DATA(int) _Py_SwappedOp[]; 1003 1004 extern void _Py_GetConstant_Init(void); 1005 1006 enum _PyAnnotateFormat { 1007 _Py_ANNOTATE_FORMAT_VALUE = 1, 1008 _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS = 2, 1009 _Py_ANNOTATE_FORMAT_FORWARDREF = 3, 1010 _Py_ANNOTATE_FORMAT_STRING = 4, 1011 }; 1012 1013 int _PyObject_SetDict(PyObject *obj, PyObject *value); 1014 1015 #ifndef Py_GIL_DISABLED 1016 static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) 1017 { 1018 assert(!_Py_IsStaticImmortal(op)); 1019 op->ob_refcnt++; 1020 _Py_INCREF_STAT_INC(); 1021 #if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) 1022 if (!_Py_IsImmortal(op)) { 1023 _Py_INCREF_IncRefTotal(); 1024 } 1025 #endif 1026 } 1027 #endif 1028 1029 #ifdef __cplusplus 1030 } 1031 #endif 1032 #endif /* !Py_INTERNAL_OBJECT_H */
