Where Online Learning is simpler!
The C and C++ Include Header Files
cat -n /usr/include/python3.14/cpython/pyatomic_msc.h
1 // This is the implementation of Python atomic operations for MSVC if the 2 // compiler does not support C11 or C++11 atomics. 3 // 4 // MSVC intrinsics are defined on char, short, long, __int64, and pointer 5 // types. Note that long and int are both 32-bits even on 64-bit Windows, 6 // so operations on int are cast to long. 7 // 8 // The volatile keyword has additional memory ordering semantics on MSVC. On 9 // x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64, 10 // volatile accesses behave like C11's memory_order_relaxed. 11 12 #ifndef Py_ATOMIC_MSC_H 13 # error "this header file must not be included directly" 14 #endif 15 16 #include <intrin.h> 17 18 #define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \ 19 Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE)) 20 21 22 // --- _Py_atomic_add -------------------------------------------------------- 23 24 static inline int8_t 25 _Py_atomic_add_int8(int8_t *obj, int8_t value) 26 { 27 _Py_atomic_ASSERT_ARG_TYPE(char); 28 return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); 29 } 30 31 static inline int16_t 32 _Py_atomic_add_int16(int16_t *obj, int16_t value) 33 { 34 _Py_atomic_ASSERT_ARG_TYPE(short); 35 return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); 36 } 37 38 static inline int32_t 39 _Py_atomic_add_int32(int32_t *obj, int32_t value) 40 { 41 _Py_atomic_ASSERT_ARG_TYPE(long); 42 return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); 43 } 44 45 static inline int64_t 46 _Py_atomic_add_int64(int64_t *obj, int64_t value) 47 { 48 #if defined(_M_X64) || defined(_M_ARM64) 49 _Py_atomic_ASSERT_ARG_TYPE(__int64); 50 return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); 51 #else 52 int64_t old_value = _Py_atomic_load_int64_relaxed(obj); 53 for (;;) { 54 int64_t new_value = old_value + value; 55 if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) { 56 return old_value; 57 } 58 } 59 #endif 60 } 61 62 63 static inline uint8_t 64 _Py_atomic_add_uint8(uint8_t *obj, uint8_t value) 65 { 66 return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value); 67 } 68 69 static inline uint16_t 70 _Py_atomic_add_uint16(uint16_t *obj, uint16_t value) 71 { 72 return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value); 73 } 74 75 static inline uint32_t 76 _Py_atomic_add_uint32(uint32_t *obj, uint32_t value) 77 { 78 return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); 79 } 80 81 static inline int 82 _Py_atomic_add_int(int *obj, int value) 83 { 84 _Py_atomic_ASSERT_ARG_TYPE(int32_t); 85 return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); 86 } 87 88 static inline unsigned int 89 _Py_atomic_add_uint(unsigned int *obj, unsigned int value) 90 { 91 _Py_atomic_ASSERT_ARG_TYPE(int32_t); 92 return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); 93 } 94 95 static inline uint64_t 96 _Py_atomic_add_uint64(uint64_t *obj, uint64_t value) 97 { 98 return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); 99 } 100 101 static inline intptr_t 102 _Py_atomic_add_intptr(intptr_t *obj, intptr_t value) 103 { 104 #if SIZEOF_VOID_P == 8 105 _Py_atomic_ASSERT_ARG_TYPE(int64_t); 106 return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); 107 #else 108 _Py_atomic_ASSERT_ARG_TYPE(int32_t); 109 return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); 110 #endif 111 } 112 113 static inline uintptr_t 114 _Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) 115 { 116 _Py_atomic_ASSERT_ARG_TYPE(intptr_t); 117 return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); 118 } 119 120 static inline Py_ssize_t 121 _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) 122 { 123 _Py_atomic_ASSERT_ARG_TYPE(intptr_t); 124 return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); 125 } 126 127 128 // --- _Py_atomic_compare_exchange ------------------------------------------- 129 130 static inline int 131 _Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value) 132 { 133 _Py_atomic_ASSERT_ARG_TYPE(char); 134 int8_t initial = (int8_t)_InterlockedCompareExchange8( 135 (volatile char *)obj, 136 (char)value, 137 (char)*expected); 138 if (initial == *expected) { 139 return 1; 140 } 141 *expected = initial; 142 return 0; 143 } 144 145 static inline int 146 _Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value) 147 { 148 _Py_atomic_ASSERT_ARG_TYPE(short); 149 int16_t initial = (int16_t)_InterlockedCompareExchange16( 150 (volatile short *)obj, 151 (short)value, 152 (short)*expected); 153 if (initial == *expected) { 154 return 1; 155 } 156 *expected = initial; 157 return 0; 158 } 159 160 static inline int 161 _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value) 162 { 163 _Py_atomic_ASSERT_ARG_TYPE(long); 164 int32_t initial = (int32_t)_InterlockedCompareExchange( 165 (volatile long *)obj, 166 (long)value, 167 (long)*expected); 168 if (initial == *expected) { 169 return 1; 170 } 171 *expected = initial; 172 return 0; 173 } 174 175 static inline int 176 _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value) 177 { 178 _Py_atomic_ASSERT_ARG_TYPE(__int64); 179 int64_t initial = (int64_t)_InterlockedCompareExchange64( 180 (volatile __int64 *)obj, 181 (__int64)value, 182 (__int64)*expected); 183 if (initial == *expected) { 184 return 1; 185 } 186 *expected = initial; 187 return 0; 188 } 189 190 static inline int 191 _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value) 192 { 193 void *initial = _InterlockedCompareExchangePointer( 194 (void**)obj, 195 value, 196 *(void**)expected); 197 if (initial == *(void**)expected) { 198 return 1; 199 } 200 *(void**)expected = initial; 201 return 0; 202 } 203 204 205 static inline int 206 _Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value) 207 { 208 return _Py_atomic_compare_exchange_int8((int8_t *)obj, 209 (int8_t *)expected, 210 (int8_t)value); 211 } 212 213 static inline int 214 _Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value) 215 { 216 return _Py_atomic_compare_exchange_int16((int16_t *)obj, 217 (int16_t *)expected, 218 (int16_t)value); 219 } 220 221 static inline int 222 _Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value) 223 { 224 return _Py_atomic_compare_exchange_int32((int32_t *)obj, 225 (int32_t *)expected, 226 (int32_t)value); 227 } 228 229 static inline int 230 _Py_atomic_compare_exchange_int(int *obj, int *expected, int value) 231 { 232 _Py_atomic_ASSERT_ARG_TYPE(int32_t); 233 return _Py_atomic_compare_exchange_int32((int32_t *)obj, 234 (int32_t *)expected, 235 (int32_t)value); 236 } 237 238 static inline int 239 _Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value) 240 { 241 _Py_atomic_ASSERT_ARG_TYPE(int32_t); 242 return _Py_atomic_compare_exchange_int32((int32_t *)obj, 243 (int32_t *)expected, 244 (int32_t)value); 245 } 246 247 static inline int 248 _Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value) 249 { 250 return _Py_atomic_compare_exchange_int64((int64_t *)obj, 251 (int64_t *)expected, 252 (int64_t)value); 253 } 254 255 static inline int 256 _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value) 257 { 258 _Py_atomic_ASSERT_ARG_TYPE(void*); 259 return _Py_atomic_compare_exchange_ptr((void**)obj, 260 (void**)expected, 261 (void*)value); 262 } 263 264 static inline int 265 _Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value) 266 { 267 _Py_atomic_ASSERT_ARG_TYPE(void*); 268 return _Py_atomic_compare_exchange_ptr((void**)obj, 269 (void**)expected, 270 (void*)value); 271 } 272 273 static inline int 274 _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value) 275 { 276 _Py_atomic_ASSERT_ARG_TYPE(void*); 277 return _Py_atomic_compare_exchange_ptr((void**)obj, 278 (void**)expected, 279 (void*)value); 280 } 281 282 283 // --- _Py_atomic_exchange --------------------------------------------------- 284 285 static inline int8_t 286 _Py_atomic_exchange_int8(int8_t *obj, int8_t value) 287 { 288 _Py_atomic_ASSERT_ARG_TYPE(char); 289 return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value); 290 } 291 292 static inline int16_t 293 _Py_atomic_exchange_int16(int16_t *obj, int16_t value) 294 { 295 _Py_atomic_ASSERT_ARG_TYPE(short); 296 return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value); 297 } 298 299 static inline int32_t 300 _Py_atomic_exchange_int32(int32_t *obj, int32_t value) 301 { 302 _Py_atomic_ASSERT_ARG_TYPE(long); 303 return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value); 304 } 305 306 static inline int64_t 307 _Py_atomic_exchange_int64(int64_t *obj, int64_t value) 308 { 309 #if defined(_M_X64) || defined(_M_ARM64) 310 _Py_atomic_ASSERT_ARG_TYPE(__int64); 311 return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value); 312 #else 313 int64_t old_value = _Py_atomic_load_int64_relaxed(obj); 314 for (;;) { 315 if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) { 316 return old_value; 317 } 318 } 319 #endif 320 } 321 322 static inline void* 323 _Py_atomic_exchange_ptr(void *obj, void *value) 324 { 325 return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value); 326 } 327 328 329 static inline uint8_t 330 _Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) 331 { 332 return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj, 333 (int8_t)value); 334 } 335 336 static inline uint16_t 337 _Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) 338 { 339 return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj, 340 (int16_t)value); 341 } 342 343 static inline uint32_t 344 _Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) 345 { 346 return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj, 347 (int32_t)value); 348 } 349 350 static inline int 351 _Py_atomic_exchange_int(int *obj, int value) 352 { 353 _Py_atomic_ASSERT_ARG_TYPE(int32_t); 354 return (int)_Py_atomic_exchange_int32((int32_t *)obj, 355 (int32_t)value); 356 } 357 358 static inline unsigned int 359 _Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) 360 { 361 _Py_atomic_ASSERT_ARG_TYPE(int32_t); 362 return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj, 363 (int32_t)value); 364 } 365 366 static inline uint64_t 367 _Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) 368 { 369 return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj, 370 (int64_t)value); 371 } 372 373 static inline intptr_t 374 _Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) 375 { 376 _Py_atomic_ASSERT_ARG_TYPE(void*); 377 return (intptr_t)_Py_atomic_exchange_ptr((void**)obj, 378 (void*)value); 379 } 380 381 static inline uintptr_t 382 _Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) 383 { 384 _Py_atomic_ASSERT_ARG_TYPE(void*); 385 return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj, 386 (void*)value); 387 } 388 389 static inline Py_ssize_t 390 _Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) 391 { 392 _Py_atomic_ASSERT_ARG_TYPE(void*); 393 return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj, 394 (void*)value); 395 } 396 397 398 // --- _Py_atomic_and -------------------------------------------------------- 399 400 static inline uint8_t 401 _Py_atomic_and_uint8(uint8_t *obj, uint8_t value) 402 { 403 _Py_atomic_ASSERT_ARG_TYPE(char); 404 return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value); 405 } 406 407 static inline uint16_t 408 _Py_atomic_and_uint16(uint16_t *obj, uint16_t value) 409 { 410 _Py_atomic_ASSERT_ARG_TYPE(short); 411 return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value); 412 } 413 414 static inline uint32_t 415 _Py_atomic_and_uint32(uint32_t *obj, uint32_t value) 416 { 417 _Py_atomic_ASSERT_ARG_TYPE(long); 418 return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value); 419 } 420 421 static inline uint64_t 422 _Py_atomic_and_uint64(uint64_t *obj, uint64_t value) 423 { 424 #if defined(_M_X64) || defined(_M_ARM64) 425 _Py_atomic_ASSERT_ARG_TYPE(__int64); 426 return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value); 427 #else 428 uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); 429 for (;;) { 430 uint64_t new_value = old_value & value; 431 if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { 432 return old_value; 433 } 434 } 435 #endif 436 } 437 438 static inline uintptr_t 439 _Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) 440 { 441 #if SIZEOF_VOID_P == 8 442 _Py_atomic_ASSERT_ARG_TYPE(uint64_t); 443 return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj, 444 (uint64_t)value); 445 #else 446 _Py_atomic_ASSERT_ARG_TYPE(uint32_t); 447 return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj, 448 (uint32_t)value); 449 #endif 450 } 451 452 453 // --- _Py_atomic_or --------------------------------------------------------- 454 455 static inline uint8_t 456 _Py_atomic_or_uint8(uint8_t *obj, uint8_t value) 457 { 458 _Py_atomic_ASSERT_ARG_TYPE(char); 459 return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value); 460 } 461 462 static inline uint16_t 463 _Py_atomic_or_uint16(uint16_t *obj, uint16_t value) 464 { 465 _Py_atomic_ASSERT_ARG_TYPE(short); 466 return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value); 467 } 468 469 static inline uint32_t 470 _Py_atomic_or_uint32(uint32_t *obj, uint32_t value) 471 { 472 _Py_atomic_ASSERT_ARG_TYPE(long); 473 return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value); 474 } 475 476 static inline uint64_t 477 _Py_atomic_or_uint64(uint64_t *obj, uint64_t value) 478 { 479 #if defined(_M_X64) || defined(_M_ARM64) 480 _Py_atomic_ASSERT_ARG_TYPE(__int64); 481 return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value); 482 #else 483 uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); 484 for (;;) { 485 uint64_t new_value = old_value | value; 486 if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { 487 return old_value; 488 } 489 } 490 #endif 491 } 492 493 494 static inline uintptr_t 495 _Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) 496 { 497 #if SIZEOF_VOID_P == 8 498 _Py_atomic_ASSERT_ARG_TYPE(uint64_t); 499 return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj, 500 (uint64_t)value); 501 #else 502 _Py_atomic_ASSERT_ARG_TYPE(uint32_t); 503 return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj, 504 (uint32_t)value); 505 #endif 506 } 507 508 509 // --- _Py_atomic_load ------------------------------------------------------- 510 511 static inline uint8_t 512 _Py_atomic_load_uint8(const uint8_t *obj) 513 { 514 #if defined(_M_X64) || defined(_M_IX86) 515 return *(volatile uint8_t *)obj; 516 #elif defined(_M_ARM64) 517 return (uint8_t)__ldar8((unsigned __int8 volatile *)obj); 518 #else 519 # error "no implementation of _Py_atomic_load_uint8" 520 #endif 521 } 522 523 static inline uint16_t 524 _Py_atomic_load_uint16(const uint16_t *obj) 525 { 526 #if defined(_M_X64) || defined(_M_IX86) 527 return *(volatile uint16_t *)obj; 528 #elif defined(_M_ARM64) 529 return (uint16_t)__ldar16((unsigned __int16 volatile *)obj); 530 #else 531 # error "no implementation of _Py_atomic_load_uint16" 532 #endif 533 } 534 535 static inline uint32_t 536 _Py_atomic_load_uint32(const uint32_t *obj) 537 { 538 #if defined(_M_X64) || defined(_M_IX86) 539 return *(volatile uint32_t *)obj; 540 #elif defined(_M_ARM64) 541 return (uint32_t)__ldar32((unsigned __int32 volatile *)obj); 542 #else 543 # error "no implementation of _Py_atomic_load_uint32" 544 #endif 545 } 546 547 static inline uint64_t 548 _Py_atomic_load_uint64(const uint64_t *obj) 549 { 550 #if defined(_M_X64) || defined(_M_IX86) 551 return *(volatile uint64_t *)obj; 552 #elif defined(_M_ARM64) 553 return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); 554 #else 555 # error "no implementation of _Py_atomic_load_uint64" 556 #endif 557 } 558 559 static inline int8_t 560 _Py_atomic_load_int8(const int8_t *obj) 561 { 562 return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj); 563 } 564 565 static inline int16_t 566 _Py_atomic_load_int16(const int16_t *obj) 567 { 568 return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj); 569 } 570 571 static inline int32_t 572 _Py_atomic_load_int32(const int32_t *obj) 573 { 574 return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj); 575 } 576 577 static inline int 578 _Py_atomic_load_int(const int *obj) 579 { 580 _Py_atomic_ASSERT_ARG_TYPE(uint32_t); 581 return (int)_Py_atomic_load_uint32((uint32_t *)obj); 582 } 583 584 static inline unsigned int 585 _Py_atomic_load_uint(const unsigned int *obj) 586 { 587 _Py_atomic_ASSERT_ARG_TYPE(uint32_t); 588 return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj); 589 } 590 591 static inline int64_t 592 _Py_atomic_load_int64(const int64_t *obj) 593 { 594 return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj); 595 } 596 597 static inline void* 598 _Py_atomic_load_ptr(const void *obj) 599 { 600 #if SIZEOF_VOID_P == 8 601 return (void*)_Py_atomic_load_uint64((const uint64_t *)obj); 602 #else 603 return (void*)_Py_atomic_load_uint32((const uint32_t *)obj); 604 #endif 605 } 606 607 static inline intptr_t 608 _Py_atomic_load_intptr(const intptr_t *obj) 609 { 610 _Py_atomic_ASSERT_ARG_TYPE(void*); 611 return (intptr_t)_Py_atomic_load_ptr((void*)obj); 612 } 613 614 static inline uintptr_t 615 _Py_atomic_load_uintptr(const uintptr_t *obj) 616 { 617 _Py_atomic_ASSERT_ARG_TYPE(void*); 618 return (uintptr_t)_Py_atomic_load_ptr((void*)obj); 619 } 620 621 static inline Py_ssize_t 622 _Py_atomic_load_ssize(const Py_ssize_t *obj) 623 { 624 _Py_atomic_ASSERT_ARG_TYPE(void*); 625 return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj); 626 } 627 628 629 // --- _Py_atomic_load_relaxed ----------------------------------------------- 630 631 static inline int 632 _Py_atomic_load_int_relaxed(const int *obj) 633 { 634 return *(volatile int *)obj; 635 } 636 637 static inline char 638 _Py_atomic_load_char_relaxed(const char *obj) 639 { 640 return *(volatile char *)obj; 641 } 642 643 static inline unsigned char 644 _Py_atomic_load_uchar_relaxed(const unsigned char *obj) 645 { 646 return *(volatile unsigned char *)obj; 647 } 648 649 static inline short 650 _Py_atomic_load_short_relaxed(const short *obj) 651 { 652 return *(volatile short *)obj; 653 } 654 655 static inline unsigned short 656 _Py_atomic_load_ushort_relaxed(const unsigned short *obj) 657 { 658 return *(volatile unsigned short *)obj; 659 } 660 661 static inline long 662 _Py_atomic_load_long_relaxed(const long *obj) 663 { 664 return *(volatile long *)obj; 665 } 666 667 static inline float 668 _Py_atomic_load_float_relaxed(const float *obj) 669 { 670 return *(volatile float *)obj; 671 } 672 673 static inline double 674 _Py_atomic_load_double_relaxed(const double *obj) 675 { 676 return *(volatile double *)obj; 677 } 678 679 static inline int8_t 680 _Py_atomic_load_int8_relaxed(const int8_t *obj) 681 { 682 return *(volatile int8_t *)obj; 683 } 684 685 static inline int16_t 686 _Py_atomic_load_int16_relaxed(const int16_t *obj) 687 { 688 return *(volatile int16_t *)obj; 689 } 690 691 static inline int32_t 692 _Py_atomic_load_int32_relaxed(const int32_t *obj) 693 { 694 return *(volatile int32_t *)obj; 695 } 696 697 static inline int64_t 698 _Py_atomic_load_int64_relaxed(const int64_t *obj) 699 { 700 return *(volatile int64_t *)obj; 701 } 702 703 static inline intptr_t 704 _Py_atomic_load_intptr_relaxed(const intptr_t *obj) 705 { 706 return *(volatile intptr_t *)obj; 707 } 708 709 static inline uint8_t 710 _Py_atomic_load_uint8_relaxed(const uint8_t *obj) 711 { 712 return *(volatile uint8_t *)obj; 713 } 714 715 static inline uint16_t 716 _Py_atomic_load_uint16_relaxed(const uint16_t *obj) 717 { 718 return *(volatile uint16_t *)obj; 719 } 720 721 static inline uint32_t 722 _Py_atomic_load_uint32_relaxed(const uint32_t *obj) 723 { 724 return *(volatile uint32_t *)obj; 725 } 726 727 static inline uint64_t 728 _Py_atomic_load_uint64_relaxed(const uint64_t *obj) 729 { 730 return *(volatile uint64_t *)obj; 731 } 732 733 static inline uintptr_t 734 _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) 735 { 736 return *(volatile uintptr_t *)obj; 737 } 738 739 static inline unsigned int 740 _Py_atomic_load_uint_relaxed(const unsigned int *obj) 741 { 742 return *(volatile unsigned int *)obj; 743 } 744 745 static inline Py_ssize_t 746 _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) 747 { 748 return *(volatile Py_ssize_t *)obj; 749 } 750 751 static inline void* 752 _Py_atomic_load_ptr_relaxed(const void *obj) 753 { 754 return *(void * volatile *)obj; 755 } 756 757 static inline unsigned long long 758 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) 759 { 760 return *(volatile unsigned long long *)obj; 761 } 762 763 static inline long long 764 _Py_atomic_load_llong_relaxed(const long long *obj) 765 { 766 return *(volatile long long *)obj; 767 } 768 769 770 // --- _Py_atomic_store ------------------------------------------------------ 771 772 static inline void 773 _Py_atomic_store_int(int *obj, int value) 774 { 775 (void)_Py_atomic_exchange_int(obj, value); 776 } 777 778 static inline void 779 _Py_atomic_store_int8(int8_t *obj, int8_t value) 780 { 781 (void)_Py_atomic_exchange_int8(obj, value); 782 } 783 784 static inline void 785 _Py_atomic_store_int16(int16_t *obj, int16_t value) 786 { 787 (void)_Py_atomic_exchange_int16(obj, value); 788 } 789 790 static inline void 791 _Py_atomic_store_int32(int32_t *obj, int32_t value) 792 { 793 (void)_Py_atomic_exchange_int32(obj, value); 794 } 795 796 static inline void 797 _Py_atomic_store_int64(int64_t *obj, int64_t value) 798 { 799 (void)_Py_atomic_exchange_int64(obj, value); 800 } 801 802 static inline void 803 _Py_atomic_store_intptr(intptr_t *obj, intptr_t value) 804 { 805 (void)_Py_atomic_exchange_intptr(obj, value); 806 } 807 808 static inline void 809 _Py_atomic_store_uint8(uint8_t *obj, uint8_t value) 810 { 811 (void)_Py_atomic_exchange_uint8(obj, value); 812 } 813 814 static inline void 815 _Py_atomic_store_uint16(uint16_t *obj, uint16_t value) 816 { 817 (void)_Py_atomic_exchange_uint16(obj, value); 818 } 819 820 static inline void 821 _Py_atomic_store_uint32(uint32_t *obj, uint32_t value) 822 { 823 (void)_Py_atomic_exchange_uint32(obj, value); 824 } 825 826 static inline void 827 _Py_atomic_store_uint64(uint64_t *obj, uint64_t value) 828 { 829 (void)_Py_atomic_exchange_uint64(obj, value); 830 } 831 832 static inline void 833 _Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) 834 { 835 (void)_Py_atomic_exchange_uintptr(obj, value); 836 } 837 838 static inline void 839 _Py_atomic_store_uint(unsigned int *obj, unsigned int value) 840 { 841 (void)_Py_atomic_exchange_uint(obj, value); 842 } 843 844 static inline void 845 _Py_atomic_store_ptr(void *obj, void *value) 846 { 847 (void)_Py_atomic_exchange_ptr(obj, value); 848 } 849 850 static inline void 851 _Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) 852 { 853 (void)_Py_atomic_exchange_ssize(obj, value); 854 } 855 856 857 // --- _Py_atomic_store_relaxed ---------------------------------------------- 858 859 static inline void 860 _Py_atomic_store_int_relaxed(int *obj, int value) 861 { 862 *(volatile int *)obj = value; 863 } 864 865 static inline void 866 _Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) 867 { 868 *(volatile int8_t *)obj = value; 869 } 870 871 static inline void 872 _Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) 873 { 874 *(volatile int16_t *)obj = value; 875 } 876 877 static inline void 878 _Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) 879 { 880 *(volatile int32_t *)obj = value; 881 } 882 883 static inline void 884 _Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) 885 { 886 *(volatile int64_t *)obj = value; 887 } 888 889 static inline void 890 _Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) 891 { 892 *(volatile intptr_t *)obj = value; 893 } 894 895 static inline void 896 _Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) 897 { 898 *(volatile uint8_t *)obj = value; 899 } 900 901 static inline void 902 _Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) 903 { 904 *(volatile uint16_t *)obj = value; 905 } 906 907 static inline void 908 _Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) 909 { 910 *(volatile uint32_t *)obj = value; 911 } 912 913 static inline void 914 _Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) 915 { 916 *(volatile uint64_t *)obj = value; 917 } 918 919 static inline void 920 _Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) 921 { 922 *(volatile uintptr_t *)obj = value; 923 } 924 925 static inline void 926 _Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) 927 { 928 *(volatile unsigned int *)obj = value; 929 } 930 931 static inline void 932 _Py_atomic_store_ptr_relaxed(void *obj, void* value) 933 { 934 *(void * volatile *)obj = value; 935 } 936 937 static inline void 938 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) 939 { 940 *(volatile Py_ssize_t *)obj = value; 941 } 942 943 static inline void 944 _Py_atomic_store_ullong_relaxed(unsigned long long *obj, 945 unsigned long long value) 946 { 947 *(volatile unsigned long long *)obj = value; 948 } 949 950 static inline void 951 _Py_atomic_store_char_relaxed(char *obj, char value) 952 { 953 *(volatile char *)obj = value; 954 } 955 956 static inline void 957 _Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) 958 { 959 *(volatile unsigned char *)obj = value; 960 } 961 962 static inline void 963 _Py_atomic_store_short_relaxed(short *obj, short value) 964 { 965 *(volatile short *)obj = value; 966 } 967 968 static inline void 969 _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) 970 { 971 *(volatile unsigned short *)obj = value; 972 } 973 974 static inline void 975 _Py_atomic_store_long_relaxed(long *obj, long value) 976 { 977 *(volatile long *)obj = value; 978 } 979 980 static inline void 981 _Py_atomic_store_float_relaxed(float *obj, float value) 982 { 983 *(volatile float *)obj = value; 984 } 985 986 static inline void 987 _Py_atomic_store_double_relaxed(double *obj, double value) 988 { 989 *(volatile double *)obj = value; 990 } 991 992 static inline void 993 _Py_atomic_store_llong_relaxed(long long *obj, long long value) 994 { 995 *(volatile long long *)obj = value; 996 } 997 998 999 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ 1000 1001 static inline void * 1002 _Py_atomic_load_ptr_acquire(const void *obj) 1003 { 1004 #if defined(_M_X64) || defined(_M_IX86) 1005 return *(void * volatile *)obj; 1006 #elif defined(_M_ARM64) 1007 return (void *)__ldar64((unsigned __int64 volatile *)obj); 1008 #else 1009 # error "no implementation of _Py_atomic_load_ptr_acquire" 1010 #endif 1011 } 1012 1013 static inline uintptr_t 1014 _Py_atomic_load_uintptr_acquire(const uintptr_t *obj) 1015 { 1016 #if defined(_M_X64) || defined(_M_IX86) 1017 return *(uintptr_t volatile *)obj; 1018 #elif defined(_M_ARM64) 1019 return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); 1020 #else 1021 # error "no implementation of _Py_atomic_load_uintptr_acquire" 1022 #endif 1023 } 1024 1025 static inline void 1026 _Py_atomic_store_ptr_release(void *obj, void *value) 1027 { 1028 #if defined(_M_X64) || defined(_M_IX86) 1029 *(void * volatile *)obj = value; 1030 #elif defined(_M_ARM64) 1031 __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); 1032 #else 1033 # error "no implementation of _Py_atomic_store_ptr_release" 1034 #endif 1035 } 1036 1037 static inline void 1038 _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) 1039 { 1040 #if defined(_M_X64) || defined(_M_IX86) 1041 *(uintptr_t volatile *)obj = value; 1042 #elif defined(_M_ARM64) 1043 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); 1044 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); 1045 #else 1046 # error "no implementation of _Py_atomic_store_uintptr_release" 1047 #endif 1048 } 1049 1050 static inline void 1051 _Py_atomic_store_int_release(int *obj, int value) 1052 { 1053 #if defined(_M_X64) || defined(_M_IX86) 1054 *(int volatile *)obj = value; 1055 #elif defined(_M_ARM64) 1056 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); 1057 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); 1058 #else 1059 # error "no implementation of _Py_atomic_store_int_release" 1060 #endif 1061 } 1062 1063 static inline void 1064 _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) 1065 { 1066 #if defined(_M_X64) || defined(_M_IX86) 1067 *(volatile unsigned int *)obj = value; 1068 #elif defined(_M_ARM64) 1069 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); 1070 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); 1071 #else 1072 # error "no implementation of _Py_atomic_store_uint_release" 1073 #endif 1074 } 1075 1076 static inline void 1077 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) 1078 { 1079 #if defined(_M_X64) || defined(_M_IX86) 1080 *(Py_ssize_t volatile *)obj = value; 1081 #elif defined(_M_ARM64) 1082 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); 1083 #else 1084 # error "no implementation of _Py_atomic_store_ssize_release" 1085 #endif 1086 } 1087 1088 static inline int 1089 _Py_atomic_load_int_acquire(const int *obj) 1090 { 1091 #if defined(_M_X64) || defined(_M_IX86) 1092 return *(int volatile *)obj; 1093 #elif defined(_M_ARM64) 1094 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); 1095 return (int)__ldar32((unsigned __int32 volatile *)obj); 1096 #else 1097 # error "no implementation of _Py_atomic_load_int_acquire" 1098 #endif 1099 } 1100 1101 static inline void 1102 _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) 1103 { 1104 #if defined(_M_X64) || defined(_M_IX86) 1105 *(uint32_t volatile *)obj = value; 1106 #elif defined(_M_ARM64) 1107 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); 1108 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); 1109 #else 1110 # error "no implementation of _Py_atomic_store_uint32_release" 1111 #endif 1112 } 1113 1114 static inline void 1115 _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) 1116 { 1117 #if defined(_M_X64) || defined(_M_IX86) 1118 *(uint64_t volatile *)obj = value; 1119 #elif defined(_M_ARM64) 1120 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); 1121 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); 1122 #else 1123 # error "no implementation of _Py_atomic_store_uint64_release" 1124 #endif 1125 } 1126 1127 static inline uint64_t 1128 _Py_atomic_load_uint64_acquire(const uint64_t *obj) 1129 { 1130 #if defined(_M_X64) || defined(_M_IX86) 1131 return *(uint64_t volatile *)obj; 1132 #elif defined(_M_ARM64) 1133 _Py_atomic_ASSERT_ARG_TYPE(__int64); 1134 return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); 1135 #else 1136 # error "no implementation of _Py_atomic_load_uint64_acquire" 1137 #endif 1138 } 1139 1140 static inline uint32_t 1141 _Py_atomic_load_uint32_acquire(const uint32_t *obj) 1142 { 1143 #if defined(_M_X64) || defined(_M_IX86) 1144 return *(uint32_t volatile *)obj; 1145 #elif defined(_M_ARM64) 1146 return (uint32_t)__ldar32((uint32_t volatile *)obj); 1147 #else 1148 # error "no implementation of _Py_atomic_load_uint32_acquire" 1149 #endif 1150 } 1151 1152 static inline Py_ssize_t 1153 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) 1154 { 1155 #if defined(_M_X64) || defined(_M_IX86) 1156 return *(Py_ssize_t volatile *)obj; 1157 #elif defined(_M_ARM64) 1158 return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); 1159 #else 1160 # error "no implementation of _Py_atomic_load_ssize_acquire" 1161 #endif 1162 } 1163 1164 // --- _Py_atomic_fence ------------------------------------------------------ 1165 1166 static inline void 1167 _Py_atomic_fence_seq_cst(void) 1168 { 1169 #if defined(_M_ARM64) 1170 __dmb(_ARM64_BARRIER_ISH); 1171 #elif defined(_M_X64) 1172 __faststorefence(); 1173 #elif defined(_M_IX86) 1174 _mm_mfence(); 1175 #else 1176 # error "no implementation of _Py_atomic_fence_seq_cst" 1177 #endif 1178 } 1179 1180 static inline void 1181 _Py_atomic_fence_acquire(void) 1182 { 1183 #if defined(_M_ARM64) 1184 __dmb(_ARM64_BARRIER_ISHLD); 1185 #elif defined(_M_X64) || defined(_M_IX86) 1186 _ReadBarrier(); 1187 #else 1188 # error "no implementation of _Py_atomic_fence_acquire" 1189 #endif 1190 } 1191 1192 static inline void 1193 _Py_atomic_fence_release(void) 1194 { 1195 #if defined(_M_ARM64) 1196 __dmb(_ARM64_BARRIER_ISH); 1197 #elif defined(_M_X64) || defined(_M_IX86) 1198 _ReadWriteBarrier(); 1199 #else 1200 # error "no implementation of _Py_atomic_fence_release" 1201 #endif 1202 } 1203 1204 #undef _Py_atomic_ASSERT_ARG_TYPE
