Where Online Learning is simpler!

The C and C++ Include Header Files

cat -n /usr/include/python3.14/cpython/pyatomic_std.h

     1	// This is the implementation of Python atomic operations using C++11 or C11
     2	// atomics. Note that the pyatomic_gcc.h implementation is preferred for GCC
     3	// compatible compilers, even if they support C++11 atomics.
     4	
     5	#ifndef Py_ATOMIC_STD_H
     6	#  error "this header file must not be included directly"
     7	#endif
     8	
     9	#ifdef __cplusplus
    10	extern "C++" {
    11	#  include <atomic>
    12	}
    13	#  define _Py_USING_STD using namespace std
    14	#  define _Atomic(tp) atomic<tp>
    15	#else
    16	#  define  _Py_USING_STD
    17	#  include <stdatomic.h>
    18	#endif
    19	
    20	
    21	// --- _Py_atomic_add --------------------------------------------------------
    22	
    23	static inline int
    24	_Py_atomic_add_int(int *obj, int value)
    25	{
    26	    _Py_USING_STD;
    27	    return atomic_fetch_add((_Atomic(int)*)obj, value);
    28	}
    29	
    30	static inline int8_t
    31	_Py_atomic_add_int8(int8_t *obj, int8_t value)
    32	{
    33	    _Py_USING_STD;
    34	    return atomic_fetch_add((_Atomic(int8_t)*)obj, value);
    35	}
    36	
    37	static inline int16_t
    38	_Py_atomic_add_int16(int16_t *obj, int16_t value)
    39	{
    40	    _Py_USING_STD;
    41	    return atomic_fetch_add((_Atomic(int16_t)*)obj, value);
    42	}
    43	
    44	static inline int32_t
    45	_Py_atomic_add_int32(int32_t *obj, int32_t value)
    46	{
    47	    _Py_USING_STD;
    48	    return atomic_fetch_add((_Atomic(int32_t)*)obj, value);
    49	}
    50	
    51	static inline int64_t
    52	_Py_atomic_add_int64(int64_t *obj, int64_t value)
    53	{
    54	    _Py_USING_STD;
    55	    return atomic_fetch_add((_Atomic(int64_t)*)obj, value);
    56	}
    57	
    58	static inline intptr_t
    59	_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
    60	{
    61	    _Py_USING_STD;
    62	    return atomic_fetch_add((_Atomic(intptr_t)*)obj, value);
    63	}
    64	
    65	static inline unsigned int
    66	_Py_atomic_add_uint(unsigned int *obj, unsigned int value)
    67	{
    68	    _Py_USING_STD;
    69	    return atomic_fetch_add((_Atomic(unsigned int)*)obj, value);
    70	}
    71	
    72	static inline uint8_t
    73	_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
    74	{
    75	    _Py_USING_STD;
    76	    return atomic_fetch_add((_Atomic(uint8_t)*)obj, value);
    77	}
    78	
    79	static inline uint16_t
    80	_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
    81	{
    82	    _Py_USING_STD;
    83	    return atomic_fetch_add((_Atomic(uint16_t)*)obj, value);
    84	}
    85	
    86	static inline uint32_t
    87	_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
    88	{
    89	    _Py_USING_STD;
    90	    return atomic_fetch_add((_Atomic(uint32_t)*)obj, value);
    91	}
    92	
    93	static inline uint64_t
    94	_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
    95	{
    96	    _Py_USING_STD;
    97	    return atomic_fetch_add((_Atomic(uint64_t)*)obj, value);
    98	}
    99	
   100	static inline uintptr_t
   101	_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
   102	{
   103	    _Py_USING_STD;
   104	    return atomic_fetch_add((_Atomic(uintptr_t)*)obj, value);
   105	}
   106	
   107	static inline Py_ssize_t
   108	_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
   109	{
   110	    _Py_USING_STD;
   111	    return atomic_fetch_add((_Atomic(Py_ssize_t)*)obj, value);
   112	}
   113	
   114	
   115	// --- _Py_atomic_compare_exchange -------------------------------------------
   116	
   117	static inline int
   118	_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired)
   119	{
   120	    _Py_USING_STD;
   121	    return atomic_compare_exchange_strong((_Atomic(int)*)obj,
   122	                                          expected, desired);
   123	}
   124	
   125	static inline int
   126	_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired)
   127	{
   128	    _Py_USING_STD;
   129	    return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj,
   130	                                          expected, desired);
   131	}
   132	
   133	static inline int
   134	_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired)
   135	{
   136	    _Py_USING_STD;
   137	    return atomic_compare_exchange_strong((_Atomic(int16_t)*)obj,
   138	                                          expected, desired);
   139	}
   140	
   141	static inline int
   142	_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired)
   143	{
   144	    _Py_USING_STD;
   145	    return atomic_compare_exchange_strong((_Atomic(int32_t)*)obj,
   146	                                          expected, desired);
   147	}
   148	
   149	static inline int
   150	_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired)
   151	{
   152	    _Py_USING_STD;
   153	    return atomic_compare_exchange_strong((_Atomic(int64_t)*)obj,
   154	                                          expected, desired);
   155	}
   156	
   157	static inline int
   158	_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired)
   159	{
   160	    _Py_USING_STD;
   161	    return atomic_compare_exchange_strong((_Atomic(intptr_t)*)obj,
   162	                                          expected, desired);
   163	}
   164	
   165	static inline int
   166	_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired)
   167	{
   168	    _Py_USING_STD;
   169	    return atomic_compare_exchange_strong((_Atomic(unsigned int)*)obj,
   170	                                          expected, desired);
   171	}
   172	
   173	static inline int
   174	_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired)
   175	{
   176	    _Py_USING_STD;
   177	    return atomic_compare_exchange_strong((_Atomic(uint8_t)*)obj,
   178	                                          expected, desired);
   179	}
   180	
   181	static inline int
   182	_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired)
   183	{
   184	    _Py_USING_STD;
   185	    return atomic_compare_exchange_strong((_Atomic(uint16_t)*)obj,
   186	                                          expected, desired);
   187	}
   188	
   189	static inline int
   190	_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired)
   191	{
   192	    _Py_USING_STD;
   193	    return atomic_compare_exchange_strong((_Atomic(uint32_t)*)obj,
   194	                                          expected, desired);
   195	}
   196	
   197	static inline int
   198	_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired)
   199	{
   200	    _Py_USING_STD;
   201	    return atomic_compare_exchange_strong((_Atomic(uint64_t)*)obj,
   202	                                          expected, desired);
   203	}
   204	
   205	static inline int
   206	_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired)
   207	{
   208	    _Py_USING_STD;
   209	    return atomic_compare_exchange_strong((_Atomic(uintptr_t)*)obj,
   210	                                          expected, desired);
   211	}
   212	
   213	static inline int
   214	_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired)
   215	{
   216	    _Py_USING_STD;
   217	    return atomic_compare_exchange_strong((_Atomic(Py_ssize_t)*)obj,
   218	                                          expected, desired);
   219	}
   220	
   221	static inline int
   222	_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired)
   223	{
   224	    _Py_USING_STD;
   225	    return atomic_compare_exchange_strong((_Atomic(void *)*)obj,
   226	                                          (void **)expected, desired);
   227	}
   228	
   229	
   230	// --- _Py_atomic_exchange ---------------------------------------------------
   231	
   232	static inline int
   233	_Py_atomic_exchange_int(int *obj, int value)
   234	{
   235	    _Py_USING_STD;
   236	    return atomic_exchange((_Atomic(int)*)obj, value);
   237	}
   238	
   239	static inline int8_t
   240	_Py_atomic_exchange_int8(int8_t *obj, int8_t value)
   241	{
   242	    _Py_USING_STD;
   243	    return atomic_exchange((_Atomic(int8_t)*)obj, value);
   244	}
   245	
   246	static inline int16_t
   247	_Py_atomic_exchange_int16(int16_t *obj, int16_t value)
   248	{
   249	    _Py_USING_STD;
   250	    return atomic_exchange((_Atomic(int16_t)*)obj, value);
   251	}
   252	
   253	static inline int32_t
   254	_Py_atomic_exchange_int32(int32_t *obj, int32_t value)
   255	{
   256	    _Py_USING_STD;
   257	    return atomic_exchange((_Atomic(int32_t)*)obj, value);
   258	}
   259	
   260	static inline int64_t
   261	_Py_atomic_exchange_int64(int64_t *obj, int64_t value)
   262	{
   263	    _Py_USING_STD;
   264	    return atomic_exchange((_Atomic(int64_t)*)obj, value);
   265	}
   266	
   267	static inline intptr_t
   268	_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
   269	{
   270	    _Py_USING_STD;
   271	    return atomic_exchange((_Atomic(intptr_t)*)obj, value);
   272	}
   273	
   274	static inline unsigned int
   275	_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
   276	{
   277	    _Py_USING_STD;
   278	    return atomic_exchange((_Atomic(unsigned int)*)obj, value);
   279	}
   280	
   281	static inline uint8_t
   282	_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
   283	{
   284	    _Py_USING_STD;
   285	    return atomic_exchange((_Atomic(uint8_t)*)obj, value);
   286	}
   287	
   288	static inline uint16_t
   289	_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
   290	{
   291	    _Py_USING_STD;
   292	    return atomic_exchange((_Atomic(uint16_t)*)obj, value);
   293	}
   294	
   295	static inline uint32_t
   296	_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
   297	{
   298	    _Py_USING_STD;
   299	    return atomic_exchange((_Atomic(uint32_t)*)obj, value);
   300	}
   301	
   302	static inline uint64_t
   303	_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
   304	{
   305	    _Py_USING_STD;
   306	    return atomic_exchange((_Atomic(uint64_t)*)obj, value);
   307	}
   308	
   309	static inline uintptr_t
   310	_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
   311	{
   312	    _Py_USING_STD;
   313	    return atomic_exchange((_Atomic(uintptr_t)*)obj, value);
   314	}
   315	
   316	static inline Py_ssize_t
   317	_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
   318	{
   319	    _Py_USING_STD;
   320	    return atomic_exchange((_Atomic(Py_ssize_t)*)obj, value);
   321	}
   322	
   323	static inline void*
   324	_Py_atomic_exchange_ptr(void *obj, void *value)
   325	{
   326	    _Py_USING_STD;
   327	    return atomic_exchange((_Atomic(void *)*)obj, value);
   328	}
   329	
   330	
   331	// --- _Py_atomic_and --------------------------------------------------------
   332	
   333	static inline uint8_t
   334	_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
   335	{
   336	    _Py_USING_STD;
   337	    return atomic_fetch_and((_Atomic(uint8_t)*)obj, value);
   338	}
   339	
   340	static inline uint16_t
   341	_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
   342	{
   343	    _Py_USING_STD;
   344	    return atomic_fetch_and((_Atomic(uint16_t)*)obj, value);
   345	}
   346	
   347	static inline uint32_t
   348	_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
   349	{
   350	    _Py_USING_STD;
   351	    return atomic_fetch_and((_Atomic(uint32_t)*)obj, value);
   352	}
   353	
   354	static inline uint64_t
   355	_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
   356	{
   357	    _Py_USING_STD;
   358	    return atomic_fetch_and((_Atomic(uint64_t)*)obj, value);
   359	}
   360	
   361	static inline uintptr_t
   362	_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
   363	{
   364	    _Py_USING_STD;
   365	    return atomic_fetch_and((_Atomic(uintptr_t)*)obj, value);
   366	}
   367	
   368	
   369	// --- _Py_atomic_or ---------------------------------------------------------
   370	
   371	static inline uint8_t
   372	_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
   373	{
   374	    _Py_USING_STD;
   375	    return atomic_fetch_or((_Atomic(uint8_t)*)obj, value);
   376	}
   377	
   378	static inline uint16_t
   379	_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
   380	{
   381	    _Py_USING_STD;
   382	    return atomic_fetch_or((_Atomic(uint16_t)*)obj, value);
   383	}
   384	
   385	static inline uint32_t
   386	_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
   387	{
   388	    _Py_USING_STD;
   389	    return atomic_fetch_or((_Atomic(uint32_t)*)obj, value);
   390	}
   391	
   392	static inline uint64_t
   393	_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
   394	{
   395	    _Py_USING_STD;
   396	    return atomic_fetch_or((_Atomic(uint64_t)*)obj, value);
   397	}
   398	
   399	static inline uintptr_t
   400	_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
   401	{
   402	    _Py_USING_STD;
   403	    return atomic_fetch_or((_Atomic(uintptr_t)*)obj, value);
   404	}
   405	
   406	
   407	// --- _Py_atomic_load -------------------------------------------------------
   408	
   409	static inline int
   410	_Py_atomic_load_int(const int *obj)
   411	{
   412	    _Py_USING_STD;
   413	    return atomic_load((const _Atomic(int)*)obj);
   414	}
   415	
   416	static inline int8_t
   417	_Py_atomic_load_int8(const int8_t *obj)
   418	{
   419	    _Py_USING_STD;
   420	    return atomic_load((const _Atomic(int8_t)*)obj);
   421	}
   422	
   423	static inline int16_t
   424	_Py_atomic_load_int16(const int16_t *obj)
   425	{
   426	    _Py_USING_STD;
   427	    return atomic_load((const _Atomic(int16_t)*)obj);
   428	}
   429	
   430	static inline int32_t
   431	_Py_atomic_load_int32(const int32_t *obj)
   432	{
   433	    _Py_USING_STD;
   434	    return atomic_load((const _Atomic(int32_t)*)obj);
   435	}
   436	
   437	static inline int64_t
   438	_Py_atomic_load_int64(const int64_t *obj)
   439	{
   440	    _Py_USING_STD;
   441	    return atomic_load((const _Atomic(int64_t)*)obj);
   442	}
   443	
   444	static inline intptr_t
   445	_Py_atomic_load_intptr(const intptr_t *obj)
   446	{
   447	    _Py_USING_STD;
   448	    return atomic_load((const _Atomic(intptr_t)*)obj);
   449	}
   450	
   451	static inline uint8_t
   452	_Py_atomic_load_uint8(const uint8_t *obj)
   453	{
   454	    _Py_USING_STD;
   455	    return atomic_load((const _Atomic(uint8_t)*)obj);
   456	}
   457	
   458	static inline uint16_t
   459	_Py_atomic_load_uint16(const uint16_t *obj)
   460	{
   461	    _Py_USING_STD;
   462	    return atomic_load((const _Atomic(uint16_t)*)obj);
   463	}
   464	
   465	static inline uint32_t
   466	_Py_atomic_load_uint32(const uint32_t *obj)
   467	{
   468	    _Py_USING_STD;
   469	    return atomic_load((const _Atomic(uint32_t)*)obj);
   470	}
   471	
   472	static inline uint64_t
   473	_Py_atomic_load_uint64(const uint64_t *obj)
   474	{
   475	    _Py_USING_STD;
   476	    return atomic_load((const _Atomic(uint64_t)*)obj);
   477	}
   478	
   479	static inline uintptr_t
   480	_Py_atomic_load_uintptr(const uintptr_t *obj)
   481	{
   482	    _Py_USING_STD;
   483	    return atomic_load((const _Atomic(uintptr_t)*)obj);
   484	}
   485	
   486	static inline unsigned int
   487	_Py_atomic_load_uint(const unsigned int *obj)
   488	{
   489	    _Py_USING_STD;
   490	    return atomic_load((const _Atomic(unsigned int)*)obj);
   491	}
   492	
   493	static inline Py_ssize_t
   494	_Py_atomic_load_ssize(const Py_ssize_t *obj)
   495	{
   496	    _Py_USING_STD;
   497	    return atomic_load((const _Atomic(Py_ssize_t)*)obj);
   498	}
   499	
   500	static inline void*
   501	_Py_atomic_load_ptr(const void *obj)
   502	{
   503	    _Py_USING_STD;
   504	    return atomic_load((const _Atomic(void*)*)obj);
   505	}
   506	
   507	
   508	// --- _Py_atomic_load_relaxed -----------------------------------------------
   509	
   510	static inline int
   511	_Py_atomic_load_int_relaxed(const int *obj)
   512	{
   513	    _Py_USING_STD;
   514	    return atomic_load_explicit((const _Atomic(int)*)obj,
   515	                                memory_order_relaxed);
   516	}
   517	
   518	static inline char
   519	_Py_atomic_load_char_relaxed(const char *obj)
   520	{
   521	    _Py_USING_STD;
   522	    return atomic_load_explicit((const _Atomic(char)*)obj,
   523	                                memory_order_relaxed);
   524	}
   525	
   526	static inline unsigned char
   527	_Py_atomic_load_uchar_relaxed(const unsigned char *obj)
   528	{
   529	    _Py_USING_STD;
   530	    return atomic_load_explicit((const _Atomic(unsigned char)*)obj,
   531	                                memory_order_relaxed);
   532	}
   533	
   534	static inline short
   535	_Py_atomic_load_short_relaxed(const short *obj)
   536	{
   537	    _Py_USING_STD;
   538	    return atomic_load_explicit((const _Atomic(short)*)obj,
   539	                                memory_order_relaxed);
   540	}
   541	
   542	static inline unsigned short
   543	_Py_atomic_load_ushort_relaxed(const unsigned short *obj)
   544	{
   545	    _Py_USING_STD;
   546	    return atomic_load_explicit((const _Atomic(unsigned short)*)obj,
   547	                                memory_order_relaxed);
   548	}
   549	
   550	static inline long
   551	_Py_atomic_load_long_relaxed(const long *obj)
   552	{
   553	    _Py_USING_STD;
   554	    return atomic_load_explicit((const _Atomic(long)*)obj,
   555	                                memory_order_relaxed);
   556	}
   557	
   558	static inline float
   559	_Py_atomic_load_float_relaxed(const float *obj)
   560	{
   561	    _Py_USING_STD;
   562	    return atomic_load_explicit((const _Atomic(float)*)obj,
   563	                                memory_order_relaxed);
   564	}
   565	
   566	static inline double
   567	_Py_atomic_load_double_relaxed(const double *obj)
   568	{
   569	    _Py_USING_STD;
   570	    return atomic_load_explicit((const _Atomic(double)*)obj,
   571	                                memory_order_relaxed);
   572	}
   573	
   574	static inline int8_t
   575	_Py_atomic_load_int8_relaxed(const int8_t *obj)
   576	{
   577	    _Py_USING_STD;
   578	    return atomic_load_explicit((const _Atomic(int8_t)*)obj,
   579	                                memory_order_relaxed);
   580	}
   581	
   582	static inline int16_t
   583	_Py_atomic_load_int16_relaxed(const int16_t *obj)
   584	{
   585	    _Py_USING_STD;
   586	    return atomic_load_explicit((const _Atomic(int16_t)*)obj,
   587	                                memory_order_relaxed);
   588	}
   589	
   590	static inline int32_t
   591	_Py_atomic_load_int32_relaxed(const int32_t *obj)
   592	{
   593	    _Py_USING_STD;
   594	    return atomic_load_explicit((const _Atomic(int32_t)*)obj,
   595	                                memory_order_relaxed);
   596	}
   597	
   598	static inline int64_t
   599	_Py_atomic_load_int64_relaxed(const int64_t *obj)
   600	{
   601	    _Py_USING_STD;
   602	    return atomic_load_explicit((const _Atomic(int64_t)*)obj,
   603	                                memory_order_relaxed);
   604	}
   605	
   606	static inline intptr_t
   607	_Py_atomic_load_intptr_relaxed(const intptr_t *obj)
   608	{
   609	    _Py_USING_STD;
   610	    return atomic_load_explicit((const _Atomic(intptr_t)*)obj,
   611	                                memory_order_relaxed);
   612	}
   613	
   614	static inline uint8_t
   615	_Py_atomic_load_uint8_relaxed(const uint8_t *obj)
   616	{
   617	    _Py_USING_STD;
   618	    return atomic_load_explicit((const _Atomic(uint8_t)*)obj,
   619	                                memory_order_relaxed);
   620	}
   621	
   622	static inline uint16_t
   623	_Py_atomic_load_uint16_relaxed(const uint16_t *obj)
   624	{
   625	    _Py_USING_STD;
   626	    return atomic_load_explicit((const _Atomic(uint16_t)*)obj,
   627	                                memory_order_relaxed);
   628	}
   629	
   630	static inline uint32_t
   631	_Py_atomic_load_uint32_relaxed(const uint32_t *obj)
   632	{
   633	    _Py_USING_STD;
   634	    return atomic_load_explicit((const _Atomic(uint32_t)*)obj,
   635	                                memory_order_relaxed);
   636	}
   637	
   638	static inline uint64_t
   639	_Py_atomic_load_uint64_relaxed(const uint64_t *obj)
   640	{
   641	    _Py_USING_STD;
   642	    return atomic_load_explicit((const _Atomic(uint64_t)*)obj,
   643	                                memory_order_relaxed);
   644	}
   645	
   646	static inline uintptr_t
   647	_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
   648	{
   649	    _Py_USING_STD;
   650	    return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
   651	                                memory_order_relaxed);
   652	}
   653	
   654	static inline unsigned int
   655	_Py_atomic_load_uint_relaxed(const unsigned int *obj)
   656	{
   657	    _Py_USING_STD;
   658	    return atomic_load_explicit((const _Atomic(unsigned int)*)obj,
   659	                                memory_order_relaxed);
   660	}
   661	
   662	static inline Py_ssize_t
   663	_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
   664	{
   665	    _Py_USING_STD;
   666	    return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj,
   667	                                memory_order_relaxed);
   668	}
   669	
   670	static inline void*
   671	_Py_atomic_load_ptr_relaxed(const void *obj)
   672	{
   673	    _Py_USING_STD;
   674	    return atomic_load_explicit((const _Atomic(void*)*)obj,
   675	                                memory_order_relaxed);
   676	}
   677	
   678	static inline unsigned long long
   679	_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
   680	{
   681	    _Py_USING_STD;
   682	    return atomic_load_explicit((const _Atomic(unsigned long long)*)obj,
   683	                                memory_order_relaxed);
   684	}
   685	
   686	static inline long long
   687	_Py_atomic_load_llong_relaxed(const long long *obj)
   688	{
   689	    _Py_USING_STD;
   690	    return atomic_load_explicit((const _Atomic(long long)*)obj,
   691	                                memory_order_relaxed);
   692	}
   693	
   694	
   695	// --- _Py_atomic_store ------------------------------------------------------
   696	
   697	static inline void
   698	_Py_atomic_store_int(int *obj, int value)
   699	{
   700	    _Py_USING_STD;
   701	    atomic_store((_Atomic(int)*)obj, value);
   702	}
   703	
   704	static inline void
   705	_Py_atomic_store_int8(int8_t *obj, int8_t value)
   706	{
   707	    _Py_USING_STD;
   708	    atomic_store((_Atomic(int8_t)*)obj, value);
   709	}
   710	
   711	static inline void
   712	_Py_atomic_store_int16(int16_t *obj, int16_t value)
   713	{
   714	    _Py_USING_STD;
   715	    atomic_store((_Atomic(int16_t)*)obj, value);
   716	}
   717	
   718	static inline void
   719	_Py_atomic_store_int32(int32_t *obj, int32_t value)
   720	{
   721	    _Py_USING_STD;
   722	    atomic_store((_Atomic(int32_t)*)obj, value);
   723	}
   724	
   725	static inline void
   726	_Py_atomic_store_int64(int64_t *obj, int64_t value)
   727	{
   728	    _Py_USING_STD;
   729	    atomic_store((_Atomic(int64_t)*)obj, value);
   730	}
   731	
   732	static inline void
   733	_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
   734	{
   735	    _Py_USING_STD;
   736	    atomic_store((_Atomic(intptr_t)*)obj, value);
   737	}
   738	
   739	static inline void
   740	_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
   741	{
   742	    _Py_USING_STD;
   743	    atomic_store((_Atomic(uint8_t)*)obj, value);
   744	}
   745	
   746	static inline void
   747	_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
   748	{
   749	    _Py_USING_STD;
   750	    atomic_store((_Atomic(uint16_t)*)obj, value);
   751	}
   752	
   753	static inline void
   754	_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
   755	{
   756	    _Py_USING_STD;
   757	    atomic_store((_Atomic(uint32_t)*)obj, value);
   758	}
   759	
   760	static inline void
   761	_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
   762	{
   763	    _Py_USING_STD;
   764	    atomic_store((_Atomic(uint64_t)*)obj, value);
   765	}
   766	
   767	static inline void
   768	_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
   769	{
   770	    _Py_USING_STD;
   771	    atomic_store((_Atomic(uintptr_t)*)obj, value);
   772	}
   773	
   774	static inline void
   775	_Py_atomic_store_uint(unsigned int *obj, unsigned int value)
   776	{
   777	    _Py_USING_STD;
   778	    atomic_store((_Atomic(unsigned int)*)obj, value);
   779	}
   780	
   781	static inline void
   782	_Py_atomic_store_ptr(void *obj, void *value)
   783	{
   784	    _Py_USING_STD;
   785	    atomic_store((_Atomic(void*)*)obj, value);
   786	}
   787	
   788	static inline void
   789	_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
   790	{
   791	    _Py_USING_STD;
   792	    atomic_store((_Atomic(Py_ssize_t)*)obj, value);
   793	}
   794	
   795	
   796	// --- _Py_atomic_store_relaxed ----------------------------------------------
   797	
   798	static inline void
   799	_Py_atomic_store_int_relaxed(int *obj, int value)
   800	{
   801	    _Py_USING_STD;
   802	    atomic_store_explicit((_Atomic(int)*)obj, value,
   803	                          memory_order_relaxed);
   804	}
   805	
   806	static inline void
   807	_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
   808	{
   809	    _Py_USING_STD;
   810	    atomic_store_explicit((_Atomic(int8_t)*)obj, value,
   811	                          memory_order_relaxed);
   812	}
   813	
   814	static inline void
   815	_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
   816	{
   817	    _Py_USING_STD;
   818	    atomic_store_explicit((_Atomic(int16_t)*)obj, value,
   819	                          memory_order_relaxed);
   820	}
   821	
   822	static inline void
   823	_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
   824	{
   825	    _Py_USING_STD;
   826	    atomic_store_explicit((_Atomic(int32_t)*)obj, value,
   827	                          memory_order_relaxed);
   828	}
   829	
   830	static inline void
   831	_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
   832	{
   833	    _Py_USING_STD;
   834	    atomic_store_explicit((_Atomic(int64_t)*)obj, value,
   835	                          memory_order_relaxed);
   836	}
   837	
   838	static inline void
   839	_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
   840	{
   841	    _Py_USING_STD;
   842	    atomic_store_explicit((_Atomic(intptr_t)*)obj, value,
   843	                          memory_order_relaxed);
   844	}
   845	
   846	static inline void
   847	_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
   848	{
   849	    _Py_USING_STD;
   850	    atomic_store_explicit((_Atomic(uint8_t)*)obj, value,
   851	                          memory_order_relaxed);
   852	}
   853	
   854	static inline void
   855	_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
   856	{
   857	    _Py_USING_STD;
   858	    atomic_store_explicit((_Atomic(uint16_t)*)obj, value,
   859	                          memory_order_relaxed);
   860	}
   861	
   862	static inline void
   863	_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
   864	{
   865	    _Py_USING_STD;
   866	    atomic_store_explicit((_Atomic(uint32_t)*)obj, value,
   867	                          memory_order_relaxed);
   868	}
   869	
   870	static inline void
   871	_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
   872	{
   873	    _Py_USING_STD;
   874	    atomic_store_explicit((_Atomic(uint64_t)*)obj, value,
   875	                          memory_order_relaxed);
   876	}
   877	
   878	static inline void
   879	_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
   880	{
   881	    _Py_USING_STD;
   882	    atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
   883	                          memory_order_relaxed);
   884	}
   885	
   886	static inline void
   887	_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
   888	{
   889	    _Py_USING_STD;
   890	    atomic_store_explicit((_Atomic(unsigned int)*)obj, value,
   891	                          memory_order_relaxed);
   892	}
   893	
   894	static inline void
   895	_Py_atomic_store_ptr_relaxed(void *obj, void *value)
   896	{
   897	    _Py_USING_STD;
   898	    atomic_store_explicit((_Atomic(void*)*)obj, value,
   899	                          memory_order_relaxed);
   900	}
   901	
   902	static inline void
   903	_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
   904	{
   905	    _Py_USING_STD;
   906	    atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value,
   907	                          memory_order_relaxed);
   908	}
   909	
   910	static inline void
   911	_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
   912	                                unsigned long long value)
   913	{
   914	    _Py_USING_STD;
   915	    atomic_store_explicit((_Atomic(unsigned long long)*)obj, value,
   916	                          memory_order_relaxed);
   917	}
   918	
   919	static inline void
   920	_Py_atomic_store_char_relaxed(char *obj, char value)
   921	{
   922	    _Py_USING_STD;
   923	    atomic_store_explicit((_Atomic(char)*)obj, value,
   924	                          memory_order_relaxed);
   925	}
   926	
   927	static inline void
   928	_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value)
   929	{
   930	    _Py_USING_STD;
   931	    atomic_store_explicit((_Atomic(unsigned char)*)obj, value,
   932	                          memory_order_relaxed);
   933	}
   934	
   935	static inline void
   936	_Py_atomic_store_short_relaxed(short *obj, short value)
   937	{
   938	    _Py_USING_STD;
   939	    atomic_store_explicit((_Atomic(short)*)obj, value,
   940	                          memory_order_relaxed);
   941	}
   942	
   943	static inline void
   944	_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value)
   945	{
   946	    _Py_USING_STD;
   947	    atomic_store_explicit((_Atomic(unsigned short)*)obj, value,
   948	                          memory_order_relaxed);
   949	}
   950	
   951	static inline void
   952	_Py_atomic_store_long_relaxed(long *obj, long value)
   953	{
   954	    _Py_USING_STD;
   955	    atomic_store_explicit((_Atomic(long)*)obj, value,
   956	                          memory_order_relaxed);
   957	}
   958	
   959	static inline void
   960	_Py_atomic_store_float_relaxed(float *obj, float value)
   961	{
   962	    _Py_USING_STD;
   963	    atomic_store_explicit((_Atomic(float)*)obj, value,
   964	                          memory_order_relaxed);
   965	}
   966	
   967	static inline void
   968	_Py_atomic_store_double_relaxed(double *obj, double value)
   969	{
   970	    _Py_USING_STD;
   971	    atomic_store_explicit((_Atomic(double)*)obj, value,
   972	                          memory_order_relaxed);
   973	}
   974	
   975	static inline void
   976	_Py_atomic_store_llong_relaxed(long long *obj, long long value)
   977	{
   978	    _Py_USING_STD;
   979	    atomic_store_explicit((_Atomic(long long)*)obj, value,
   980	                          memory_order_relaxed);
   981	}
   982	
   983	
   984	// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
   985	
   986	static inline void *
   987	_Py_atomic_load_ptr_acquire(const void *obj)
   988	{
   989	    _Py_USING_STD;
   990	    return atomic_load_explicit((const _Atomic(void*)*)obj,
   991	                                memory_order_acquire);
   992	}
   993	
   994	static inline uintptr_t
   995	_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
   996	{
   997	    _Py_USING_STD;
   998	    return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
   999	                                memory_order_acquire);
  1000	}
  1001	
  1002	static inline void
  1003	_Py_atomic_store_ptr_release(void *obj, void *value)
  1004	{
  1005	    _Py_USING_STD;
  1006	    atomic_store_explicit((_Atomic(void*)*)obj, value,
  1007	                          memory_order_release);
  1008	}
  1009	
  1010	static inline void
  1011	_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
  1012	{
  1013	    _Py_USING_STD;
  1014	    atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
  1015	                          memory_order_release);
  1016	}
  1017	
  1018	static inline void
  1019	_Py_atomic_store_int_release(int *obj, int value)
  1020	{
  1021	    _Py_USING_STD;
  1022	    atomic_store_explicit((_Atomic(int)*)obj, value,
  1023	                          memory_order_release);
  1024	}
  1025	
  1026	static inline void
  1027	_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value)
  1028	{
  1029	    _Py_USING_STD;
  1030	    atomic_store_explicit((_Atomic(unsigned int)*)obj, value,
  1031	                          memory_order_release);
  1032	}
  1033	
  1034	static inline void
  1035	_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
  1036	{
  1037	    _Py_USING_STD;
  1038	    atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value,
  1039	                          memory_order_release);
  1040	}
  1041	
  1042	static inline int
  1043	_Py_atomic_load_int_acquire(const int *obj)
  1044	{
  1045	    _Py_USING_STD;
  1046	    return atomic_load_explicit((const _Atomic(int)*)obj,
  1047	                                memory_order_acquire);
  1048	}
  1049	
  1050	static inline void
  1051	_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)
  1052	{
  1053	    _Py_USING_STD;
  1054	    atomic_store_explicit((_Atomic(uint32_t)*)obj, value,
  1055	                          memory_order_release);
  1056	}
  1057	
  1058	static inline void
  1059	_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
  1060	{
  1061	    _Py_USING_STD;
  1062	    atomic_store_explicit((_Atomic(uint64_t)*)obj, value,
  1063	                          memory_order_release);
  1064	}
  1065	
  1066	static inline uint64_t
  1067	_Py_atomic_load_uint64_acquire(const uint64_t *obj)
  1068	{
  1069	    _Py_USING_STD;
  1070	    return atomic_load_explicit((const _Atomic(uint64_t)*)obj,
  1071	                                memory_order_acquire);
  1072	}
  1073	
  1074	static inline uint32_t
  1075	_Py_atomic_load_uint32_acquire(const uint32_t *obj)
  1076	{
  1077	    _Py_USING_STD;
  1078	    return atomic_load_explicit((const _Atomic(uint32_t)*)obj,
  1079	                                memory_order_acquire);
  1080	}
  1081	
  1082	static inline Py_ssize_t
  1083	_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)
  1084	{
  1085	    _Py_USING_STD;
  1086	    return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj,
  1087	                                memory_order_acquire);
  1088	}
  1089	
  1090	
  1091	// --- _Py_atomic_fence ------------------------------------------------------
  1092	
  1093	 static inline void
  1094	_Py_atomic_fence_seq_cst(void)
  1095	{
  1096	    _Py_USING_STD;
  1097	    atomic_thread_fence(memory_order_seq_cst);
  1098	}
  1099	
  1100	 static inline void
  1101	_Py_atomic_fence_acquire(void)
  1102	{
  1103	    _Py_USING_STD;
  1104	    atomic_thread_fence(memory_order_acquire);
  1105	}
  1106	
  1107	 static inline void
  1108	_Py_atomic_fence_release(void)
  1109	{
  1110	    _Py_USING_STD;
  1111	    atomic_thread_fence(memory_order_release);
  1112	}