Where Online Learning is simpler!
The C and C++ Include Header Files
cat -n /usr/include/python3.14/internal/pycore_stackref.h
1 #ifndef Py_INTERNAL_STACKREF_H 2 #define Py_INTERNAL_STACKREF_H 3 #ifdef __cplusplus 4 extern "C" { 5 #endif 6 7 #ifndef Py_BUILD_CORE 8 # error "this header requires Py_BUILD_CORE define" 9 #endif 10 11 #include "pycore_object.h" // Py_DECREF_MORTAL 12 #include "pycore_object_deferred.h" // _PyObject_HasDeferredRefcount() 13 14 #include <stdbool.h> // bool 15 16 17 /* 18 This file introduces a new API for handling references on the stack, called 19 _PyStackRef. This API is inspired by HPy. 20 21 There are 3 main operations, that convert _PyStackRef to PyObject* and 22 vice versa: 23 24 1. Borrow (discouraged) 25 2. Steal 26 3. New 27 28 Borrow means that the reference is converted without any change in ownership. 29 This is discouraged because it makes verification much harder. It also makes 30 unboxed integers harder in the future. 31 32 Steal means that ownership is transferred to something else. The total 33 number of references to the object stays the same. The old reference is no 34 longer valid. 35 36 New creates a new reference from the old reference. The old reference 37 is still valid. 38 39 All _PyStackRef must be operated on by the new reference operations: 40 41 1. DUP 42 2. CLOSE 43 44 DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old 45 reference. The old reference remains unchanged. 46 47 CLOSE is roughly equivalent to Py_DECREF. It destroys a reference. 48 49 Note that it is unsafe to borrow a _PyStackRef and then do normal 50 CPython refcounting operations on it! 51 */ 52 53 54 #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) 55 56 #define Py_TAG_BITS 0 57 58 PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref); 59 PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber); 60 PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber); 61 PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber); 62 extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref); 63 64 static const _PyStackRef PyStackRef_NULL = { .index = 0 }; 65 66 // Use the first 3 even numbers for None, True and False. 67 // Odd numbers are reserved for (tagged) integers 68 #define PyStackRef_None ((_PyStackRef){ .index = 2 } ) 69 #define PyStackRef_False ((_PyStackRef){ .index = 4 }) 70 #define PyStackRef_True ((_PyStackRef){ .index = 6 }) 71 72 #define INITIAL_STACKREF_INDEX 8 73 74 static inline int 75 PyStackRef_IsNull(_PyStackRef ref) 76 { 77 return ref.index == 0; 78 } 79 80 static inline int 81 PyStackRef_IsTrue(_PyStackRef ref) 82 { 83 return _Py_stackref_get_object(ref) == Py_True; 84 } 85 86 static inline int 87 PyStackRef_IsFalse(_PyStackRef ref) 88 { 89 return _Py_stackref_get_object(ref) == Py_False; 90 } 91 92 static inline int 93 PyStackRef_IsNone(_PyStackRef ref) 94 { 95 return _Py_stackref_get_object(ref) == Py_None; 96 } 97 98 static inline PyObject * 99 _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber) 100 { 101 assert((ref.index & 1) == 0); 102 _Py_stackref_record_borrow(ref, filename, linenumber); 103 return _Py_stackref_get_object(ref); 104 } 105 106 #define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__) 107 108 static inline PyObject * 109 _PyStackRef_AsPyObjectSteal(_PyStackRef ref, const char *filename, int linenumber) 110 { 111 return _Py_stackref_close(ref, filename, linenumber); 112 } 113 #define PyStackRef_AsPyObjectSteal(REF) _PyStackRef_AsPyObjectSteal((REF), __FILE__, __LINE__) 114 115 static inline _PyStackRef 116 _PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber) 117 { 118 Py_INCREF(obj); 119 return _Py_stackref_create(obj, filename, linenumber); 120 } 121 #define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__) 122 123 static inline _PyStackRef 124 _PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber) 125 { 126 return _Py_stackref_create(obj, filename, linenumber); 127 } 128 #define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) 129 130 static inline _PyStackRef 131 _PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber) 132 { 133 return _Py_stackref_create(obj, filename, linenumber); 134 } 135 #define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__) 136 137 static inline _PyStackRef 138 _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber) 139 { 140 assert(_Py_IsImmortal(obj)); 141 return _Py_stackref_create(obj, filename, linenumber); 142 } 143 #define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__) 144 145 static inline bool 146 PyStackRef_IsTaggedInt(_PyStackRef ref) 147 { 148 return (ref.index & 1) == 1; 149 } 150 151 static inline void 152 _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber) 153 { 154 if (PyStackRef_IsTaggedInt(ref)) { 155 return; 156 } 157 PyObject *obj = _Py_stackref_close(ref, filename, linenumber); 158 Py_DECREF(obj); 159 } 160 #define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__) 161 162 163 static inline void 164 _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber) 165 { 166 if (PyStackRef_IsNull(ref)) { 167 return; 168 } 169 _PyStackRef_CLOSE(ref, filename, linenumber); 170 } 171 #define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__) 172 173 static inline _PyStackRef 174 _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) 175 { 176 if (PyStackRef_IsTaggedInt(ref)) { 177 return ref; 178 } 179 else { 180 PyObject *obj = _Py_stackref_get_object(ref); 181 Py_INCREF(obj); 182 return _Py_stackref_create(obj, filename, linenumber); 183 } 184 } 185 #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) 186 187 extern void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber); 188 #define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__) 189 190 static inline _PyStackRef 191 PyStackRef_MakeHeapSafe(_PyStackRef ref) 192 { 193 return ref; 194 } 195 196 static inline _PyStackRef 197 PyStackRef_Borrow(_PyStackRef ref) 198 { 199 return PyStackRef_DUP(ref); 200 } 201 202 #define PyStackRef_CLEAR(REF) \ 203 do { \ 204 _PyStackRef *_tmp_op_ptr = &(REF); \ 205 _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ 206 *_tmp_op_ptr = PyStackRef_NULL; \ 207 PyStackRef_XCLOSE(_tmp_old_op); \ 208 } while (0) 209 210 static inline _PyStackRef 211 _PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber) 212 { 213 assert(!_Py_IsImmortal(obj)); 214 return _Py_stackref_create(obj, filename, linenumber); 215 } 216 #define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__) 217 218 static inline bool 219 PyStackRef_IsHeapSafe(_PyStackRef ref) 220 { 221 return true; 222 } 223 224 static inline _PyStackRef 225 _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linenumber) 226 { 227 assert(!_Py_IsStaticImmortal(obj)); 228 Py_INCREF(obj); 229 return _Py_stackref_create(obj, filename, linenumber); 230 } 231 #define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj), __FILE__, __LINE__) 232 233 #define PyStackRef_RefcountOnObject(REF) 1 234 235 extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b); 236 237 extern bool PyStackRef_IsTaggedInt(_PyStackRef ref); 238 239 extern intptr_t PyStackRef_UntagInt(_PyStackRef ref); 240 241 extern _PyStackRef PyStackRef_TagInt(intptr_t i); 242 243 extern bool 244 PyStackRef_IsNullOrInt(_PyStackRef ref); 245 246 #else 247 248 #define Py_INT_TAG 3 249 #define Py_TAG_REFCNT 1 250 251 static inline bool 252 PyStackRef_IsTaggedInt(_PyStackRef i) 253 { 254 return (i.bits & Py_INT_TAG) == Py_INT_TAG; 255 } 256 257 static inline _PyStackRef 258 PyStackRef_TagInt(intptr_t i) 259 { 260 assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i); 261 return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) }; 262 } 263 264 static inline intptr_t 265 PyStackRef_UntagInt(_PyStackRef i) 266 { 267 assert((i.bits & Py_INT_TAG) == Py_INT_TAG); 268 intptr_t val = (intptr_t)i.bits; 269 return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2); 270 } 271 272 273 #ifdef Py_GIL_DISABLED 274 275 #define Py_TAG_DEFERRED Py_TAG_REFCNT 276 277 #define Py_TAG_PTR ((uintptr_t)0) 278 #define Py_TAG_BITS ((uintptr_t)1) 279 280 281 static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED}; 282 #define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits) 283 #define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED }) 284 #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED }) 285 #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED }) 286 287 // Checks that mask out the deferred bit in the free threading build. 288 #define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None) 289 #define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True) 290 #define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False) 291 292 #define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref)) 293 294 static inline PyObject * 295 PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) 296 { 297 PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS))); 298 return cleared; 299 } 300 301 #define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) 302 303 static inline PyObject * 304 PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) 305 { 306 assert(!PyStackRef_IsDeferred(stackref)); 307 return (PyObject *)stackref.bits; 308 } 309 310 static inline PyObject * 311 PyStackRef_AsPyObjectSteal(_PyStackRef stackref) 312 { 313 assert(!PyStackRef_IsNull(stackref)); 314 if (PyStackRef_IsDeferred(stackref)) { 315 return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)); 316 } 317 return PyStackRef_AsPyObjectBorrow(stackref); 318 } 319 320 static inline _PyStackRef 321 _PyStackRef_FromPyObjectSteal(PyObject *obj) 322 { 323 assert(obj != NULL); 324 // Make sure we don't take an already tagged value. 325 assert(((uintptr_t)obj & Py_TAG_BITS) == 0); 326 return (_PyStackRef){ .bits = (uintptr_t)obj }; 327 } 328 # define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) 329 330 static inline _PyStackRef 331 PyStackRef_FromPyObjectBorrow(PyObject *obj) 332 { 333 assert(obj != NULL); 334 assert(((uintptr_t)obj & Py_TAG_BITS) == 0); 335 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; 336 } 337 338 static inline bool 339 PyStackRef_IsHeapSafe(_PyStackRef stackref) 340 { 341 if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsTaggedInt(stackref)) { 342 PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); 343 return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj); 344 } 345 return true; 346 } 347 348 static inline _PyStackRef 349 PyStackRef_MakeHeapSafe(_PyStackRef stackref) 350 { 351 if (PyStackRef_IsHeapSafe(stackref)) { 352 return stackref; 353 } 354 PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); 355 return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; 356 } 357 358 static inline _PyStackRef 359 PyStackRef_FromPyObjectStealMortal(PyObject *obj) 360 { 361 assert(obj != NULL); 362 assert(!_Py_IsImmortal(obj)); 363 // Make sure we don't take an already tagged value. 364 assert(((uintptr_t)obj & Py_TAG_BITS) == 0); 365 return (_PyStackRef){ .bits = (uintptr_t)obj }; 366 } 367 368 static inline _PyStackRef 369 PyStackRef_FromPyObjectNew(PyObject *obj) 370 { 371 // Make sure we don't take an already tagged value. 372 assert(((uintptr_t)obj & Py_TAG_BITS) == 0); 373 assert(obj != NULL); 374 if (_PyObject_HasDeferredRefcount(obj)) { 375 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; 376 } 377 else { 378 return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR }; 379 } 380 } 381 #define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) 382 383 static inline _PyStackRef 384 PyStackRef_FromPyObjectImmortal(PyObject *obj) 385 { 386 // Make sure we don't take an already tagged value. 387 assert(((uintptr_t)obj & Py_TAG_BITS) == 0); 388 assert(obj != NULL); 389 assert(_Py_IsImmortal(obj)); 390 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; 391 } 392 #define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj)) 393 394 #define PyStackRef_CLOSE(REF) \ 395 do { \ 396 _PyStackRef _close_tmp = (REF); \ 397 assert(!PyStackRef_IsNull(_close_tmp)); \ 398 if (!PyStackRef_IsDeferred(_close_tmp)) { \ 399 Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \ 400 } \ 401 } while (0) 402 403 static inline void 404 PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) 405 { 406 (void)destruct; 407 PyStackRef_CLOSE(ref); 408 } 409 410 static inline _PyStackRef 411 PyStackRef_DUP(_PyStackRef stackref) 412 { 413 assert(!PyStackRef_IsNull(stackref)); 414 if (PyStackRef_IsDeferred(stackref)) { 415 return stackref; 416 } 417 Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref)); 418 return stackref; 419 } 420 421 static inline _PyStackRef 422 PyStackRef_Borrow(_PyStackRef stackref) 423 { 424 return (_PyStackRef){ .bits = stackref.bits | Py_TAG_DEFERRED }; 425 } 426 427 // Convert a possibly deferred reference to a strong reference. 428 static inline _PyStackRef 429 PyStackRef_AsStrongReference(_PyStackRef stackref) 430 { 431 return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref)); 432 } 433 434 #define PyStackRef_XCLOSE(stackref) \ 435 do { \ 436 _PyStackRef _tmp = (stackref); \ 437 if (!PyStackRef_IsNull(_tmp)) { \ 438 PyStackRef_CLOSE(_tmp); \ 439 } \ 440 } while (0); 441 442 #define PyStackRef_CLEAR(op) \ 443 do { \ 444 _PyStackRef *_tmp_op_ptr = &(op); \ 445 _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ 446 if (!PyStackRef_IsNull(_tmp_old_op)) { \ 447 *_tmp_op_ptr = PyStackRef_NULL; \ 448 PyStackRef_CLOSE(_tmp_old_op); \ 449 } \ 450 } while (0) 451 452 #define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew 453 454 #else // Py_GIL_DISABLED 455 456 // With GIL 457 458 /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT 459 * as they can (must) have their reclamation deferred */ 460 461 #define Py_TAG_BITS 3 462 #if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT 463 # error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" 464 #endif 465 466 #define BITS_TO_PTR(REF) ((PyObject *)((REF).bits)) 467 #define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT))) 468 469 #define PyStackRef_NULL_BITS Py_TAG_REFCNT 470 static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS }; 471 472 #define PyStackRef_IsNull(ref) ((ref).bits == PyStackRef_NULL_BITS) 473 #define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT }) 474 #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT }) 475 #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT }) 476 477 #define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT)) 478 #define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT)) 479 #define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT)) 480 481 #ifdef Py_DEBUG 482 483 static inline void PyStackRef_CheckValid(_PyStackRef ref) { 484 assert(ref.bits != 0); 485 int tag = ref.bits & Py_TAG_BITS; 486 PyObject *obj = BITS_TO_PTR_MASKED(ref); 487 switch (tag) { 488 case 0: 489 /* Can be immortal if object was made immortal after reference came into existence */ 490 assert(!_Py_IsStaticImmortal(obj)); 491 break; 492 case Py_TAG_REFCNT: 493 break; 494 default: 495 assert(0); 496 } 497 } 498 499 #else 500 501 #define PyStackRef_CheckValid(REF) ((void)0) 502 503 #endif 504 505 #ifdef _WIN32 506 #define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_REFCNT) == 0) 507 #define PyStackRef_AsPyObjectBorrow BITS_TO_PTR_MASKED 508 #define PyStackRef_Borrow(REF) (_PyStackRef){ .bits = ((REF).bits) | Py_TAG_REFCNT}; 509 #else 510 /* Does this ref not have an embedded refcount and thus not refer to a declared immmortal object? */ 511 static inline int 512 PyStackRef_RefcountOnObject(_PyStackRef ref) 513 { 514 return (ref.bits & Py_TAG_REFCNT) == 0; 515 } 516 517 static inline PyObject * 518 PyStackRef_AsPyObjectBorrow(_PyStackRef ref) 519 { 520 assert(!PyStackRef_IsTaggedInt(ref)); 521 return BITS_TO_PTR_MASKED(ref); 522 } 523 524 static inline _PyStackRef 525 PyStackRef_Borrow(_PyStackRef ref) 526 { 527 return (_PyStackRef){ .bits = ref.bits | Py_TAG_REFCNT }; 528 } 529 #endif 530 531 static inline PyObject * 532 PyStackRef_AsPyObjectSteal(_PyStackRef ref) 533 { 534 if (PyStackRef_RefcountOnObject(ref)) { 535 return BITS_TO_PTR(ref); 536 } 537 else { 538 return Py_NewRef(BITS_TO_PTR_MASKED(ref)); 539 } 540 } 541 542 static inline _PyStackRef 543 PyStackRef_FromPyObjectSteal(PyObject *obj) 544 { 545 assert(obj != NULL); 546 #if SIZEOF_VOID_P > 4 547 unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; 548 #else 549 unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0; 550 #endif 551 _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag}); 552 PyStackRef_CheckValid(ref); 553 return ref; 554 } 555 556 static inline _PyStackRef 557 PyStackRef_FromPyObjectBorrow(PyObject *obj) 558 { 559 assert(obj != NULL); 560 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; 561 } 562 563 static inline _PyStackRef 564 PyStackRef_FromPyObjectStealMortal(PyObject *obj) 565 { 566 assert(obj != NULL); 567 assert(!_Py_IsImmortal(obj)); 568 _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) }); 569 PyStackRef_CheckValid(ref); 570 return ref; 571 } 572 573 static inline _PyStackRef 574 _PyStackRef_FromPyObjectNew(PyObject *obj) 575 { 576 assert(obj != NULL); 577 if (_Py_IsImmortal(obj)) { 578 return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT}; 579 } 580 _Py_INCREF_MORTAL(obj); 581 _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; 582 PyStackRef_CheckValid(ref); 583 return ref; 584 } 585 #define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj)) 586 587 static inline _PyStackRef 588 _PyStackRef_FromPyObjectNewMortal(PyObject *obj) 589 { 590 assert(obj != NULL); 591 _Py_INCREF_MORTAL(obj); 592 _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; 593 PyStackRef_CheckValid(ref); 594 return ref; 595 } 596 #define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj)) 597 598 /* Create a new reference from an object with an embedded reference count */ 599 static inline _PyStackRef 600 PyStackRef_FromPyObjectImmortal(PyObject *obj) 601 { 602 assert(_Py_IsImmortal(obj)); 603 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; 604 } 605 606 /* WARNING: This macro evaluates its argument more than once */ 607 #ifdef _WIN32 608 #define PyStackRef_DUP(REF) \ 609 (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF)) 610 #else 611 static inline _PyStackRef 612 PyStackRef_DUP(_PyStackRef ref) 613 { 614 assert(!PyStackRef_IsNull(ref)); 615 if (PyStackRef_RefcountOnObject(ref)) { 616 _Py_INCREF_MORTAL(BITS_TO_PTR(ref)); 617 } 618 return ref; 619 } 620 #endif 621 622 static inline bool 623 PyStackRef_IsHeapSafe(_PyStackRef ref) 624 { 625 return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); 626 } 627 628 static inline _PyStackRef 629 PyStackRef_MakeHeapSafe(_PyStackRef ref) 630 { 631 if (PyStackRef_IsHeapSafe(ref)) { 632 return ref; 633 } 634 PyObject *obj = BITS_TO_PTR_MASKED(ref); 635 Py_INCREF(obj); 636 ref.bits = (uintptr_t)obj; 637 PyStackRef_CheckValid(ref); 638 return ref; 639 } 640 641 #ifdef _WIN32 642 #define PyStackRef_CLOSE(REF) \ 643 do { \ 644 _PyStackRef _temp = (REF); \ 645 if (PyStackRef_RefcountOnObject(_temp)) Py_DECREF_MORTAL(BITS_TO_PTR(_temp)); \ 646 } while (0) 647 #else 648 static inline void 649 PyStackRef_CLOSE(_PyStackRef ref) 650 { 651 assert(!PyStackRef_IsNull(ref)); 652 if (PyStackRef_RefcountOnObject(ref)) { 653 Py_DECREF_MORTAL(BITS_TO_PTR(ref)); 654 } 655 } 656 #endif 657 658 static inline bool 659 PyStackRef_IsNullOrInt(_PyStackRef ref) 660 { 661 return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref); 662 } 663 664 static inline void 665 PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct) 666 { 667 assert(!PyStackRef_IsNull(ref)); 668 if (PyStackRef_RefcountOnObject(ref)) { 669 Py_DECREF_MORTAL_SPECIALIZED(BITS_TO_PTR(ref), destruct); 670 } 671 } 672 673 #ifdef _WIN32 674 #define PyStackRef_XCLOSE PyStackRef_CLOSE 675 #else 676 static inline void 677 PyStackRef_XCLOSE(_PyStackRef ref) 678 { 679 assert(ref.bits != 0); 680 if (PyStackRef_RefcountOnObject(ref)) { 681 assert(!PyStackRef_IsNull(ref)); 682 Py_DECREF_MORTAL(BITS_TO_PTR(ref)); 683 } 684 } 685 #endif 686 687 #define PyStackRef_CLEAR(REF) \ 688 do { \ 689 _PyStackRef *_tmp_op_ptr = &(REF); \ 690 _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ 691 *_tmp_op_ptr = PyStackRef_NULL; \ 692 PyStackRef_XCLOSE(_tmp_old_op); \ 693 } while (0) 694 695 696 #endif // Py_GIL_DISABLED 697 698 // Note: this is a macro because MSVC (Windows) has trouble inlining it. 699 700 #define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT))) 701 702 703 #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) 704 705 #define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref)) 706 707 // Converts a PyStackRef back to a PyObject *, converting the 708 // stackref to a new reference. 709 #define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)) 710 711 // StackRef type checks 712 713 static inline bool 714 PyStackRef_GenCheck(_PyStackRef stackref) 715 { 716 return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref)); 717 } 718 719 static inline bool 720 PyStackRef_BoolCheck(_PyStackRef stackref) 721 { 722 return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref)); 723 } 724 725 static inline bool 726 PyStackRef_LongCheck(_PyStackRef stackref) 727 { 728 return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref)); 729 } 730 731 static inline bool 732 PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref) 733 { 734 return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref)); 735 } 736 737 static inline bool 738 PyStackRef_CodeCheck(_PyStackRef stackref) 739 { 740 return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref)); 741 } 742 743 static inline bool 744 PyStackRef_FunctionCheck(_PyStackRef stackref) 745 { 746 return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref)); 747 } 748 749 static inline void 750 _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) 751 { 752 #ifdef Py_GIL_DISABLED 753 _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; 754 ref->next = tstate_impl->c_stack_refs; 755 tstate_impl->c_stack_refs = ref; 756 #endif 757 ref->ref = PyStackRef_NULL; 758 } 759 760 static inline void 761 _PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj) 762 { 763 _PyThreadState_PushCStackRef(tstate, ref); 764 ref->ref = PyStackRef_FromPyObjectNew(obj); 765 } 766 767 static inline void 768 _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) 769 { 770 #ifdef Py_GIL_DISABLED 771 _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; 772 assert(tstate_impl->c_stack_refs == ref); 773 tstate_impl->c_stack_refs = ref->next; 774 #endif 775 PyStackRef_XCLOSE(ref->ref); 776 } 777 778 static inline _PyStackRef 779 _PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref) 780 { 781 #ifdef Py_GIL_DISABLED 782 _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; 783 assert(tstate_impl->c_stack_refs == ref); 784 tstate_impl->c_stack_refs = ref->next; 785 #endif 786 return ref->ref; 787 } 788 789 #ifdef Py_GIL_DISABLED 790 791 static inline int 792 _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) 793 { 794 if (_PyObject_HasDeferredRefcount(op)) { 795 *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_DEFERRED }; 796 return 1; 797 } 798 if (_Py_TryIncrefCompare(src, op)) { 799 *out = PyStackRef_FromPyObjectSteal(op); 800 return 1; 801 } 802 return 0; 803 } 804 805 static inline int 806 _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) 807 { 808 PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src)); 809 if (op == NULL) { 810 *out = PyStackRef_NULL; 811 return 1; 812 } 813 return _Py_TryIncrefCompareStackRef(src, op, out); 814 } 815 816 #endif 817 818 #define PyStackRef_XSETREF(dst, src) \ 819 do { \ 820 _PyStackRef _tmp_dst_ref = (dst); \ 821 (dst) = (src); \ 822 PyStackRef_XCLOSE(_tmp_dst_ref); \ 823 } while(0) 824 825 // Like Py_VISIT but for _PyStackRef fields 826 #define _Py_VISIT_STACKREF(ref) \ 827 do { \ 828 if (!PyStackRef_IsNullOrInt(ref)) { \ 829 int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \ 830 if (vret) \ 831 return vret; \ 832 } \ 833 } while (0) 834 835 #ifdef __cplusplus 836 } 837 #endif 838 #endif /* !Py_INTERNAL_STACKREF_H */
