Where Online Learning is simpler!
The C and C++ Include Header Files
/usr/include/c++/11/experimental/bits/simd.h
$ cat -n /usr/include/c++/11/experimental/bits/simd.h 1 // Definition of the public simd interfaces -*- C++ -*- 2 3 // Copyright (C) 2020-2021 Free Software Foundation, Inc. 4 // 5 // This file is part of the GNU ISO C++ Library. This library is free 6 // software; you can redistribute it and/or modify it under the 7 // terms of the GNU General Public License as published by the 8 // Free Software Foundation; either version 3, or (at your option) 9 // any later version. 10 11 // This library is distributed in the hope that it will be useful, 12 // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 // GNU General Public License for more details. 15 16 // Under Section 7 of GPL version 3, you are granted additional 17 // permissions described in the GCC Runtime Library Exception, version 18 // 3.1, as published by the Free Software Foundation. 19 20 // You should have received a copy of the GNU General Public License and 21 // a copy of the GCC Runtime Library Exception along with this program; 22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 // <http://www.gnu.org/licenses/>. 24 25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_H 26 #define _GLIBCXX_EXPERIMENTAL_SIMD_H 27 28 #if __cplusplus >= 201703L 29 30 #include "simd_detail.h" 31 #include "numeric_traits.h" 32 #include <bit> 33 #include <bitset> 34 #ifdef _GLIBCXX_DEBUG_UB 35 #include <cstdio> // for stderr 36 #endif 37 #include <cstring> 38 #include <functional> 39 #include <iosfwd> 40 #include <utility> 41 42 #if _GLIBCXX_SIMD_X86INTRIN 43 #include <x86intrin.h> 44 #elif _GLIBCXX_SIMD_HAVE_NEON 45 #pragma GCC diagnostic push 46 // narrowing conversion of '__a' from 'uint64_t' {aka 'long long unsigned int'} to 47 // 'int64x1_t' {aka 'long long int'} [-Wnarrowing] 48 #pragma GCC diagnostic ignored "-Wnarrowing" 49 #include <arm_neon.h> 50 #pragma GCC diagnostic pop 51 #endif 52 53 /** @ingroup ts_simd 54 * @{ 55 */ 56 /* There are several closely related types, with the following naming 57 * convention: 58 * _Tp: vectorizable (arithmetic) type (or any type) 59 * _TV: __vector_type_t<_Tp, _Np> 60 * _TW: _SimdWrapper<_Tp, _Np> 61 * _TI: __intrinsic_type_t<_Tp, _Np> 62 * _TVT: _VectorTraits<_TV> or _VectorTraits<_TW> 63 * If one additional type is needed use _U instead of _T. 64 * Otherwise use _T\d, _TV\d, _TW\d, TI\d, _TVT\d. 65 * 66 * More naming conventions: 67 * _Ap or _Abi: An ABI tag from the simd_abi namespace 68 * _Ip: often used for integer types with sizeof(_Ip) == sizeof(_Tp), 69 * _IV, _IW as for _TV, _TW 70 * _Np: number of elements (not bytes) 71 * _Bytes: number of bytes 72 * 73 * Variable names: 74 * __k: mask object (vector- or bitmask) 75 */ 76 _GLIBCXX_SIMD_BEGIN_NAMESPACE 77 78 #if !_GLIBCXX_SIMD_X86INTRIN 79 using __m128 [[__gnu__::__vector_size__(16)]] = float; 80 using __m128d [[__gnu__::__vector_size__(16)]] = double; 81 using __m128i [[__gnu__::__vector_size__(16)]] = long long; 82 using __m256 [[__gnu__::__vector_size__(32)]] = float; 83 using __m256d [[__gnu__::__vector_size__(32)]] = double; 84 using __m256i [[__gnu__::__vector_size__(32)]] = long long; 85 using __m512 [[__gnu__::__vector_size__(64)]] = float; 86 using __m512d [[__gnu__::__vector_size__(64)]] = double; 87 using __m512i [[__gnu__::__vector_size__(64)]] = long long; 88 #endif 89 90 namespace simd_abi { 91 // simd_abi forward declarations {{{ 92 // implementation details: 93 struct _Scalar; 94 95 template <int _Np> 96 struct _Fixed; 97 98 // There are two major ABIs that appear on different architectures. 99 // Both have non-boolean values packed into an N Byte register 100 // -> #elements = N / sizeof(T) 101 // Masks differ: 102 // 1. Use value vector registers for masks (all 0 or all 1) 103 // 2. Use bitmasks (mask registers) with one bit per value in the corresponding 104 // value vector 105 // 106 // Both can be partially used, masking off the rest when doing horizontal 107 // operations or operations that can trap (e.g. FP_INVALID or integer division 108 // by 0). This is encoded as the number of used bytes. 109 template <int _UsedBytes> 110 struct _VecBuiltin; 111 112 template <int _UsedBytes> 113 struct _VecBltnBtmsk; 114 115 template <typename _Tp, int _Np> 116 using _VecN = _VecBuiltin<sizeof(_Tp) * _Np>; 117 118 template <int _UsedBytes = 16> 119 using _Sse = _VecBuiltin<_UsedBytes>; 120 121 template <int _UsedBytes = 32> 122 using _Avx = _VecBuiltin<_UsedBytes>; 123 124 template <int _UsedBytes = 64> 125 using _Avx512 = _VecBltnBtmsk<_UsedBytes>; 126 127 template <int _UsedBytes = 16> 128 using _Neon = _VecBuiltin<_UsedBytes>; 129 130 // implementation-defined: 131 using __sse = _Sse<>; 132 using __avx = _Avx<>; 133 using __avx512 = _Avx512<>; 134 using __neon = _Neon<>; 135 using __neon128 = _Neon<16>; 136 using __neon64 = _Neon<8>; 137 138 // standard: 139 template <typename _Tp, size_t _Np, typename...> 140 struct deduce; 141 142 template <int _Np> 143 using fixed_size = _Fixed<_Np>; 144 145 using scalar = _Scalar; 146 147 // }}} 148 } // namespace simd_abi 149 // forward declarations is_simd(_mask), simd(_mask), simd_size {{{ 150 template <typename _Tp> 151 struct is_simd; 152 153 template <typename _Tp> 154 struct is_simd_mask; 155 156 template <typename _Tp, typename _Abi> 157 class simd; 158 159 template <typename _Tp, typename _Abi> 160 class simd_mask; 161 162 template <typename _Tp, typename _Abi> 163 struct simd_size; 164 165 // }}} 166 // load/store flags {{{ 167 struct element_aligned_tag 168 { 169 template <typename _Tp, typename _Up = typename _Tp::value_type> 170 static constexpr size_t _S_alignment = alignof(_Up); 171 172 template <typename _Tp, typename _Up> 173 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up* 174 _S_apply(_Up* __ptr) 175 { return __ptr; } 176 }; 177 178 struct vector_aligned_tag 179 { 180 template <typename _Tp, typename _Up = typename _Tp::value_type> 181 static constexpr size_t _S_alignment 182 = std::__bit_ceil(sizeof(_Up) * _Tp::size()); 183 184 template <typename _Tp, typename _Up> 185 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up* 186 _S_apply(_Up* __ptr) 187 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _S_alignment<_Tp, _Up>)); } 188 }; 189 190 template <size_t _Np> struct overaligned_tag 191 { 192 template <typename _Tp, typename _Up = typename _Tp::value_type> 193 static constexpr size_t _S_alignment = _Np; 194 195 template <typename _Tp, typename _Up> 196 _GLIBCXX_SIMD_INTRINSIC static constexpr _Up* 197 _S_apply(_Up* __ptr) 198 { return static_cast<_Up*>(__builtin_assume_aligned(__ptr, _Np)); } 199 }; 200 201 inline constexpr element_aligned_tag element_aligned = {}; 202 203 inline constexpr vector_aligned_tag vector_aligned = {}; 204 205 template <size_t _Np> 206 inline constexpr overaligned_tag<_Np> overaligned = {}; 207 208 // }}} 209 template <size_t _Xp> 210 using _SizeConstant = integral_constant<size_t, _Xp>; 211 212 namespace __detail 213 { 214 struct _Minimum 215 { 216 template <typename _Tp> 217 _GLIBCXX_SIMD_INTRINSIC constexpr 218 _Tp 219 operator()(_Tp __a, _Tp __b) const 220 { 221 using std::min; 222 return min(__a, __b); 223 } 224 }; 225 226 struct _Maximum 227 { 228 template <typename _Tp> 229 _GLIBCXX_SIMD_INTRINSIC constexpr 230 _Tp 231 operator()(_Tp __a, _Tp __b) const 232 { 233 using std::max; 234 return max(__a, __b); 235 } 236 }; 237 } // namespace __detail 238 239 // unrolled/pack execution helpers 240 // __execute_n_times{{{ 241 template <typename _Fp, size_t... _I> 242 _GLIBCXX_SIMD_INTRINSIC constexpr void 243 __execute_on_index_sequence(_Fp&& __f, index_sequence<_I...>) 244 { ((void)__f(_SizeConstant<_I>()), ...); } 245 246 template <typename _Fp> 247 _GLIBCXX_SIMD_INTRINSIC constexpr void 248 __execute_on_index_sequence(_Fp&&, index_sequence<>) 249 { } 250 251 template <size_t _Np, typename _Fp> 252 _GLIBCXX_SIMD_INTRINSIC constexpr void 253 __execute_n_times(_Fp&& __f) 254 { 255 __execute_on_index_sequence(static_cast<_Fp&&>(__f), 256 make_index_sequence<_Np>{}); 257 } 258 259 // }}} 260 // __generate_from_n_evaluations{{{ 261 template <typename _R, typename _Fp, size_t... _I> 262 _GLIBCXX_SIMD_INTRINSIC constexpr _R 263 __execute_on_index_sequence_with_return(_Fp&& __f, index_sequence<_I...>) 264 { return _R{__f(_SizeConstant<_I>())...}; } 265 266 template <size_t _Np, typename _R, typename _Fp> 267 _GLIBCXX_SIMD_INTRINSIC constexpr _R 268 __generate_from_n_evaluations(_Fp&& __f) 269 { 270 return __execute_on_index_sequence_with_return<_R>( 271 static_cast<_Fp&&>(__f), make_index_sequence<_Np>{}); 272 } 273 274 // }}} 275 // __call_with_n_evaluations{{{ 276 template <size_t... _I, typename _F0, typename _FArgs> 277 _GLIBCXX_SIMD_INTRINSIC constexpr auto 278 __call_with_n_evaluations(index_sequence<_I...>, _F0&& __f0, _FArgs&& __fargs) 279 { return __f0(__fargs(_SizeConstant<_I>())...); } 280 281 template <size_t _Np, typename _F0, typename _FArgs> 282 _GLIBCXX_SIMD_INTRINSIC constexpr auto 283 __call_with_n_evaluations(_F0&& __f0, _FArgs&& __fargs) 284 { 285 return __call_with_n_evaluations(make_index_sequence<_Np>{}, 286 static_cast<_F0&&>(__f0), 287 static_cast<_FArgs&&>(__fargs)); 288 } 289 290 // }}} 291 // __call_with_subscripts{{{ 292 template <size_t _First = 0, size_t... _It, typename _Tp, typename _Fp> 293 _GLIBCXX_SIMD_INTRINSIC constexpr auto 294 __call_with_subscripts(_Tp&& __x, index_sequence<_It...>, _Fp&& __fun) 295 { return __fun(__x[_First + _It]...); } 296 297 template <size_t _Np, size_t _First = 0, typename _Tp, typename _Fp> 298 _GLIBCXX_SIMD_INTRINSIC constexpr auto 299 __call_with_subscripts(_Tp&& __x, _Fp&& __fun) 300 { 301 return __call_with_subscripts<_First>(static_cast<_Tp&&>(__x), 302 make_index_sequence<_Np>(), 303 static_cast<_Fp&&>(__fun)); 304 } 305 306 // }}} 307 308 // vvv ---- type traits ---- vvv 309 // integer type aliases{{{ 310 using _UChar = unsigned char; 311 using _SChar = signed char; 312 using _UShort = unsigned short; 313 using _UInt = unsigned int; 314 using _ULong = unsigned long; 315 using _ULLong = unsigned long long; 316 using _LLong = long long; 317 318 //}}} 319 // __first_of_pack{{{ 320 template <typename _T0, typename...> 321 struct __first_of_pack 322 { using type = _T0; }; 323 324 template <typename... _Ts> 325 using __first_of_pack_t = typename __first_of_pack<_Ts...>::type; 326 327 //}}} 328 // __value_type_or_identity_t {{{ 329 template <typename _Tp> 330 typename _Tp::value_type 331 __value_type_or_identity_impl(int); 332 333 template <typename _Tp> 334 _Tp 335 __value_type_or_identity_impl(float); 336 337 template <typename _Tp> 338 using __value_type_or_identity_t 339 = decltype(__value_type_or_identity_impl<_Tp>(int())); 340 341 // }}} 342 // __is_vectorizable {{{ 343 template <typename _Tp> 344 struct __is_vectorizable : public is_arithmetic<_Tp> {}; 345 346 template <> 347 struct __is_vectorizable<bool> : public false_type {}; 348 349 template <typename _Tp> 350 inline constexpr bool __is_vectorizable_v = __is_vectorizable<_Tp>::value; 351 352 // Deduces to a vectorizable type 353 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>> 354 using _Vectorizable = _Tp; 355 356 // }}} 357 // _LoadStorePtr / __is_possible_loadstore_conversion {{{ 358 template <typename _Ptr, typename _ValueType> 359 struct __is_possible_loadstore_conversion 360 : conjunction<__is_vectorizable<_Ptr>, __is_vectorizable<_ValueType>> {}; 361 362 template <> 363 struct __is_possible_loadstore_conversion<bool, bool> : true_type {}; 364 365 // Deduces to a type allowed for load/store with the given value type. 366 template <typename _Ptr, typename _ValueType, 367 typename = enable_if_t< 368 __is_possible_loadstore_conversion<_Ptr, _ValueType>::value>> 369 using _LoadStorePtr = _Ptr; 370 371 // }}} 372 // __is_bitmask{{{ 373 template <typename _Tp, typename = void_t<>> 374 struct __is_bitmask : false_type {}; 375 376 template <typename _Tp> 377 inline constexpr bool __is_bitmask_v = __is_bitmask<_Tp>::value; 378 379 // the __mmaskXX case: 380 template <typename _Tp> 381 struct __is_bitmask<_Tp, 382 void_t<decltype(declval<unsigned&>() = declval<_Tp>() & 1u)>> 383 : true_type {}; 384 385 // }}} 386 // __int_for_sizeof{{{ 387 #pragma GCC diagnostic push 388 #pragma GCC diagnostic ignored "-Wpedantic" 389 template <size_t _Bytes> 390 constexpr auto 391 __int_for_sizeof() 392 { 393 static_assert(_Bytes > 0); 394 if constexpr (_Bytes == sizeof(int)) 395 return int(); 396 #ifdef __clang__ 397 else if constexpr (_Bytes == sizeof(char)) 398 return char(); 399 #else 400 else if constexpr (_Bytes == sizeof(_SChar)) 401 return _SChar(); 402 #endif 403 else if constexpr (_Bytes == sizeof(short)) 404 return short(); 405 #ifndef __clang__ 406 else if constexpr (_Bytes == sizeof(long)) 407 return long(); 408 #endif 409 else if constexpr (_Bytes == sizeof(_LLong)) 410 return _LLong(); 411 #ifdef __SIZEOF_INT128__ 412 else if constexpr (_Bytes == sizeof(__int128)) 413 return __int128(); 414 #endif // __SIZEOF_INT128__ 415 else if constexpr (_Bytes % sizeof(int) == 0) 416 { 417 constexpr size_t _Np = _Bytes / sizeof(int); 418 struct _Ip 419 { 420 int _M_data[_Np]; 421 422 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip 423 operator&(_Ip __rhs) const 424 { 425 return __generate_from_n_evaluations<_Np, _Ip>( 426 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 427 return __rhs._M_data[__i] & _M_data[__i]; 428 }); 429 } 430 431 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip 432 operator|(_Ip __rhs) const 433 { 434 return __generate_from_n_evaluations<_Np, _Ip>( 435 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 436 return __rhs._M_data[__i] | _M_data[__i]; 437 }); 438 } 439 440 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip 441 operator^(_Ip __rhs) const 442 { 443 return __generate_from_n_evaluations<_Np, _Ip>( 444 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 445 return __rhs._M_data[__i] ^ _M_data[__i]; 446 }); 447 } 448 449 _GLIBCXX_SIMD_INTRINSIC constexpr _Ip 450 operator~() const 451 { 452 return __generate_from_n_evaluations<_Np, _Ip>( 453 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return ~_M_data[__i]; }); 454 } 455 }; 456 return _Ip{}; 457 } 458 else 459 static_assert(_Bytes == 0, "this should be unreachable"); 460 } 461 #pragma GCC diagnostic pop 462 463 template <typename _Tp> 464 using __int_for_sizeof_t = decltype(__int_for_sizeof<sizeof(_Tp)>()); 465 466 template <size_t _Np> 467 using __int_with_sizeof_t = decltype(__int_for_sizeof<_Np>()); 468 469 // }}} 470 // __is_fixed_size_abi{{{ 471 template <typename _Tp> 472 struct __is_fixed_size_abi : false_type {}; 473 474 template <int _Np> 475 struct __is_fixed_size_abi<simd_abi::fixed_size<_Np>> : true_type {}; 476 477 template <typename _Tp> 478 inline constexpr bool __is_fixed_size_abi_v = __is_fixed_size_abi<_Tp>::value; 479 480 // }}} 481 // constexpr feature detection{{{ 482 constexpr inline bool __have_mmx = _GLIBCXX_SIMD_HAVE_MMX; 483 constexpr inline bool __have_sse = _GLIBCXX_SIMD_HAVE_SSE; 484 constexpr inline bool __have_sse2 = _GLIBCXX_SIMD_HAVE_SSE2; 485 constexpr inline bool __have_sse3 = _GLIBCXX_SIMD_HAVE_SSE3; 486 constexpr inline bool __have_ssse3 = _GLIBCXX_SIMD_HAVE_SSSE3; 487 constexpr inline bool __have_sse4_1 = _GLIBCXX_SIMD_HAVE_SSE4_1; 488 constexpr inline bool __have_sse4_2 = _GLIBCXX_SIMD_HAVE_SSE4_2; 489 constexpr inline bool __have_xop = _GLIBCXX_SIMD_HAVE_XOP; 490 constexpr inline bool __have_avx = _GLIBCXX_SIMD_HAVE_AVX; 491 constexpr inline bool __have_avx2 = _GLIBCXX_SIMD_HAVE_AVX2; 492 constexpr inline bool __have_bmi = _GLIBCXX_SIMD_HAVE_BMI1; 493 constexpr inline bool __have_bmi2 = _GLIBCXX_SIMD_HAVE_BMI2; 494 constexpr inline bool __have_lzcnt = _GLIBCXX_SIMD_HAVE_LZCNT; 495 constexpr inline bool __have_sse4a = _GLIBCXX_SIMD_HAVE_SSE4A; 496 constexpr inline bool __have_fma = _GLIBCXX_SIMD_HAVE_FMA; 497 constexpr inline bool __have_fma4 = _GLIBCXX_SIMD_HAVE_FMA4; 498 constexpr inline bool __have_f16c = _GLIBCXX_SIMD_HAVE_F16C; 499 constexpr inline bool __have_popcnt = _GLIBCXX_SIMD_HAVE_POPCNT; 500 constexpr inline bool __have_avx512f = _GLIBCXX_SIMD_HAVE_AVX512F; 501 constexpr inline bool __have_avx512dq = _GLIBCXX_SIMD_HAVE_AVX512DQ; 502 constexpr inline bool __have_avx512vl = _GLIBCXX_SIMD_HAVE_AVX512VL; 503 constexpr inline bool __have_avx512bw = _GLIBCXX_SIMD_HAVE_AVX512BW; 504 constexpr inline bool __have_avx512dq_vl = __have_avx512dq && __have_avx512vl; 505 constexpr inline bool __have_avx512bw_vl = __have_avx512bw && __have_avx512vl; 506 507 constexpr inline bool __have_neon = _GLIBCXX_SIMD_HAVE_NEON; 508 constexpr inline bool __have_neon_a32 = _GLIBCXX_SIMD_HAVE_NEON_A32; 509 constexpr inline bool __have_neon_a64 = _GLIBCXX_SIMD_HAVE_NEON_A64; 510 constexpr inline bool __support_neon_float = 511 #if defined __GCC_IEC_559 512 __GCC_IEC_559 == 0; 513 #elif defined __FAST_MATH__ 514 true; 515 #else 516 false; 517 #endif 518 519 #ifdef _ARCH_PWR10 520 constexpr inline bool __have_power10vec = true; 521 #else 522 constexpr inline bool __have_power10vec = false; 523 #endif 524 #ifdef __POWER9_VECTOR__ 525 constexpr inline bool __have_power9vec = true; 526 #else 527 constexpr inline bool __have_power9vec = false; 528 #endif 529 #if defined __POWER8_VECTOR__ 530 constexpr inline bool __have_power8vec = true; 531 #else 532 constexpr inline bool __have_power8vec = __have_power9vec; 533 #endif 534 #if defined __VSX__ 535 constexpr inline bool __have_power_vsx = true; 536 #else 537 constexpr inline bool __have_power_vsx = __have_power8vec; 538 #endif 539 #if defined __ALTIVEC__ 540 constexpr inline bool __have_power_vmx = true; 541 #else 542 constexpr inline bool __have_power_vmx = __have_power_vsx; 543 #endif 544 545 // }}} 546 // __is_scalar_abi {{{ 547 template <typename _Abi> 548 constexpr bool 549 __is_scalar_abi() 550 { return is_same_v<simd_abi::scalar, _Abi>; } 551 552 // }}} 553 // __abi_bytes_v {{{ 554 template <template <int> class _Abi, int _Bytes> 555 constexpr int 556 __abi_bytes_impl(_Abi<_Bytes>*) 557 { return _Bytes; } 558 559 template <typename _Tp> 560 constexpr int 561 __abi_bytes_impl(_Tp*) 562 { return -1; } 563 564 template <typename _Abi> 565 inline constexpr int __abi_bytes_v 566 = __abi_bytes_impl(static_cast<_Abi*>(nullptr)); 567 568 // }}} 569 // __is_builtin_bitmask_abi {{{ 570 template <typename _Abi> 571 constexpr bool 572 __is_builtin_bitmask_abi() 573 { return is_same_v<simd_abi::_VecBltnBtmsk<__abi_bytes_v<_Abi>>, _Abi>; } 574 575 // }}} 576 // __is_sse_abi {{{ 577 template <typename _Abi> 578 constexpr bool 579 __is_sse_abi() 580 { 581 constexpr auto _Bytes = __abi_bytes_v<_Abi>; 582 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>; 583 } 584 585 // }}} 586 // __is_avx_abi {{{ 587 template <typename _Abi> 588 constexpr bool 589 __is_avx_abi() 590 { 591 constexpr auto _Bytes = __abi_bytes_v<_Abi>; 592 return _Bytes > 16 && _Bytes <= 32 593 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>; 594 } 595 596 // }}} 597 // __is_avx512_abi {{{ 598 template <typename _Abi> 599 constexpr bool 600 __is_avx512_abi() 601 { 602 constexpr auto _Bytes = __abi_bytes_v<_Abi>; 603 return _Bytes <= 64 && is_same_v<simd_abi::_Avx512<_Bytes>, _Abi>; 604 } 605 606 // }}} 607 // __is_neon_abi {{{ 608 template <typename _Abi> 609 constexpr bool 610 __is_neon_abi() 611 { 612 constexpr auto _Bytes = __abi_bytes_v<_Abi>; 613 return _Bytes <= 16 && is_same_v<simd_abi::_VecBuiltin<_Bytes>, _Abi>; 614 } 615 616 // }}} 617 // __make_dependent_t {{{ 618 template <typename, typename _Up> 619 struct __make_dependent 620 { using type = _Up; }; 621 622 template <typename _Tp, typename _Up> 623 using __make_dependent_t = typename __make_dependent<_Tp, _Up>::type; 624 625 // }}} 626 // ^^^ ---- type traits ---- ^^^ 627 628 // __invoke_ub{{{ 629 template <typename... _Args> 630 [[noreturn]] _GLIBCXX_SIMD_ALWAYS_INLINE void 631 __invoke_ub([[maybe_unused]] const char* __msg, [[maybe_unused]] const _Args&... __args) 632 { 633 #ifdef _GLIBCXX_DEBUG_UB 634 __builtin_fprintf(stderr, __msg, __args...); 635 __builtin_trap(); 636 #else 637 __builtin_unreachable(); 638 #endif 639 } 640 641 // }}} 642 // __assert_unreachable{{{ 643 template <typename _Tp> 644 struct __assert_unreachable 645 { static_assert(!is_same_v<_Tp, _Tp>, "this should be unreachable"); }; 646 647 // }}} 648 // __size_or_zero_v {{{ 649 template <typename _Tp, typename _Ap, size_t _Np = simd_size<_Tp, _Ap>::value> 650 constexpr size_t 651 __size_or_zero_dispatch(int) 652 { return _Np; } 653 654 template <typename _Tp, typename _Ap> 655 constexpr size_t 656 __size_or_zero_dispatch(float) 657 { return 0; } 658 659 template <typename _Tp, typename _Ap> 660 inline constexpr size_t __size_or_zero_v 661 = __size_or_zero_dispatch<_Tp, _Ap>(0); 662 663 // }}} 664 // __div_roundup {{{ 665 inline constexpr size_t 666 __div_roundup(size_t __a, size_t __b) 667 { return (__a + __b - 1) / __b; } 668 669 // }}} 670 // _ExactBool{{{ 671 class _ExactBool 672 { 673 const bool _M_data; 674 675 public: 676 _GLIBCXX_SIMD_INTRINSIC constexpr 677 _ExactBool(bool __b) : _M_data(__b) {} 678 679 _ExactBool(int) = delete; 680 681 _GLIBCXX_SIMD_INTRINSIC constexpr 682 operator bool() const 683 { return _M_data; } 684 }; 685 686 // }}} 687 // __may_alias{{{ 688 /**@internal 689 * Helper __may_alias<_Tp> that turns _Tp into the type to be used for an 690 * aliasing pointer. This adds the __may_alias attribute to _Tp (with compilers 691 * that support it). 692 */ 693 template <typename _Tp> 694 using __may_alias [[__gnu__::__may_alias__]] = _Tp; 695 696 // }}} 697 // _UnsupportedBase {{{ 698 // simd and simd_mask base for unsupported <_Tp, _Abi> 699 struct _UnsupportedBase 700 { 701 _UnsupportedBase() = delete; 702 _UnsupportedBase(const _UnsupportedBase&) = delete; 703 _UnsupportedBase& operator=(const _UnsupportedBase&) = delete; 704 ~_UnsupportedBase() = delete; 705 }; 706 707 // }}} 708 // _InvalidTraits {{{ 709 /** 710 * @internal 711 * Defines the implementation of __a given <_Tp, _Abi>. 712 * 713 * Implementations must ensure that only valid <_Tp, _Abi> instantiations are 714 * possible. Static assertions in the type definition do not suffice. It is 715 * important that SFINAE works. 716 */ 717 struct _InvalidTraits 718 { 719 using _IsValid = false_type; 720 using _SimdBase = _UnsupportedBase; 721 using _MaskBase = _UnsupportedBase; 722 723 static constexpr size_t _S_full_size = 0; 724 static constexpr bool _S_is_partial = false; 725 726 static constexpr size_t _S_simd_align = 1; 727 struct _SimdImpl; 728 struct _SimdMember {}; 729 struct _SimdCastType; 730 731 static constexpr size_t _S_mask_align = 1; 732 struct _MaskImpl; 733 struct _MaskMember {}; 734 struct _MaskCastType; 735 }; 736 737 // }}} 738 // _SimdTraits {{{ 739 template <typename _Tp, typename _Abi, typename = void_t<>> 740 struct _SimdTraits : _InvalidTraits {}; 741 742 // }}} 743 // __private_init, __bitset_init{{{ 744 /** 745 * @internal 746 * Tag used for private init constructor of simd and simd_mask 747 */ 748 inline constexpr struct _PrivateInit {} __private_init = {}; 749 750 inline constexpr struct _BitsetInit {} __bitset_init = {}; 751 752 // }}} 753 // __is_narrowing_conversion<_From, _To>{{{ 754 template <typename _From, typename _To, bool = is_arithmetic_v<_From>, 755 bool = is_arithmetic_v<_To>> 756 struct __is_narrowing_conversion; 757 758 // ignore "signed/unsigned mismatch" in the following trait. 759 // The implicit conversions will do the right thing here. 760 template <typename _From, typename _To> 761 struct __is_narrowing_conversion<_From, _To, true, true> 762 : public __bool_constant<( 763 __digits_v<_From> > __digits_v<_To> 764 || __finite_max_v<_From> > __finite_max_v<_To> 765 || __finite_min_v<_From> < __finite_min_v<_To> 766 || (is_signed_v<_From> && is_unsigned_v<_To>))> {}; 767 768 template <typename _Tp> 769 struct __is_narrowing_conversion<_Tp, bool, true, true> 770 : public true_type {}; 771 772 template <> 773 struct __is_narrowing_conversion<bool, bool, true, true> 774 : public false_type {}; 775 776 template <typename _Tp> 777 struct __is_narrowing_conversion<_Tp, _Tp, true, true> 778 : public false_type {}; 779 780 template <typename _From, typename _To> 781 struct __is_narrowing_conversion<_From, _To, false, true> 782 : public negation<is_convertible<_From, _To>> {}; 783 784 // }}} 785 // __converts_to_higher_integer_rank{{{ 786 template <typename _From, typename _To, bool = (sizeof(_From) < sizeof(_To))> 787 struct __converts_to_higher_integer_rank : public true_type {}; 788 789 // this may fail for char -> short if sizeof(char) == sizeof(short) 790 template <typename _From, typename _To> 791 struct __converts_to_higher_integer_rank<_From, _To, false> 792 : public is_same<decltype(declval<_From>() + declval<_To>()), _To> {}; 793 794 // }}} 795 // __data(simd/simd_mask) {{{ 796 template <typename _Tp, typename _Ap> 797 _GLIBCXX_SIMD_INTRINSIC constexpr const auto& 798 __data(const simd<_Tp, _Ap>& __x); 799 800 template <typename _Tp, typename _Ap> 801 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 802 __data(simd<_Tp, _Ap>& __x); 803 804 template <typename _Tp, typename _Ap> 805 _GLIBCXX_SIMD_INTRINSIC constexpr const auto& 806 __data(const simd_mask<_Tp, _Ap>& __x); 807 808 template <typename _Tp, typename _Ap> 809 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 810 __data(simd_mask<_Tp, _Ap>& __x); 811 812 // }}} 813 // _SimdConverter {{{ 814 template <typename _FromT, typename _FromA, typename _ToT, typename _ToA, 815 typename = void> 816 struct _SimdConverter; 817 818 template <typename _Tp, typename _Ap> 819 struct _SimdConverter<_Tp, _Ap, _Tp, _Ap, void> 820 { 821 template <typename _Up> 822 _GLIBCXX_SIMD_INTRINSIC const _Up& 823 operator()(const _Up& __x) 824 { return __x; } 825 }; 826 827 // }}} 828 // __to_value_type_or_member_type {{{ 829 template <typename _V> 830 _GLIBCXX_SIMD_INTRINSIC constexpr auto 831 __to_value_type_or_member_type(const _V& __x) -> decltype(__data(__x)) 832 { return __data(__x); } 833 834 template <typename _V> 835 _GLIBCXX_SIMD_INTRINSIC constexpr const typename _V::value_type& 836 __to_value_type_or_member_type(const typename _V::value_type& __x) 837 { return __x; } 838 839 // }}} 840 // __bool_storage_member_type{{{ 841 template <size_t _Size> 842 struct __bool_storage_member_type; 843 844 template <size_t _Size> 845 using __bool_storage_member_type_t = 846 typename __bool_storage_member_type<_Size>::type; 847 848 // }}} 849 // _SimdTuple {{{ 850 // why not tuple? 851 // 1. tuple gives no guarantee about the storage order, but I require 852 // storage 853 // equivalent to array<_Tp, _Np> 854 // 2. direct access to the element type (first template argument) 855 // 3. enforces equal element type, only different _Abi types are allowed 856 template <typename _Tp, typename... _Abis> 857 struct _SimdTuple; 858 859 //}}} 860 // __fixed_size_storage_t {{{ 861 template <typename _Tp, int _Np> 862 struct __fixed_size_storage; 863 864 template <typename _Tp, int _Np> 865 using __fixed_size_storage_t = typename __fixed_size_storage<_Tp, _Np>::type; 866 867 // }}} 868 // _SimdWrapper fwd decl{{{ 869 template <typename _Tp, size_t _Size, typename = void_t<>> 870 struct _SimdWrapper; 871 872 template <typename _Tp> 873 using _SimdWrapper8 = _SimdWrapper<_Tp, 8 / sizeof(_Tp)>; 874 template <typename _Tp> 875 using _SimdWrapper16 = _SimdWrapper<_Tp, 16 / sizeof(_Tp)>; 876 template <typename _Tp> 877 using _SimdWrapper32 = _SimdWrapper<_Tp, 32 / sizeof(_Tp)>; 878 template <typename _Tp> 879 using _SimdWrapper64 = _SimdWrapper<_Tp, 64 / sizeof(_Tp)>; 880 881 // }}} 882 // __is_simd_wrapper {{{ 883 template <typename _Tp> 884 struct __is_simd_wrapper : false_type {}; 885 886 template <typename _Tp, size_t _Np> 887 struct __is_simd_wrapper<_SimdWrapper<_Tp, _Np>> : true_type {}; 888 889 template <typename _Tp> 890 inline constexpr bool __is_simd_wrapper_v = __is_simd_wrapper<_Tp>::value; 891 892 // }}} 893 // _BitOps {{{ 894 struct _BitOps 895 { 896 // _S_bit_iteration {{{ 897 template <typename _Tp, typename _Fp> 898 static void 899 _S_bit_iteration(_Tp __mask, _Fp&& __f) 900 { 901 static_assert(sizeof(_ULLong) >= sizeof(_Tp)); 902 conditional_t<sizeof(_Tp) <= sizeof(_UInt), _UInt, _ULLong> __k; 903 if constexpr (is_convertible_v<_Tp, decltype(__k)>) 904 __k = __mask; 905 else 906 __k = __mask.to_ullong(); 907 while(__k) 908 { 909 __f(std::__countr_zero(__k)); 910 __k &= (__k - 1); 911 } 912 } 913 914 //}}} 915 }; 916 917 //}}} 918 // __increment, __decrement {{{ 919 template <typename _Tp = void> 920 struct __increment 921 { constexpr _Tp operator()(_Tp __a) const { return ++__a; } }; 922 923 template <> 924 struct __increment<void> 925 { 926 template <typename _Tp> 927 constexpr _Tp 928 operator()(_Tp __a) const 929 { return ++__a; } 930 }; 931 932 template <typename _Tp = void> 933 struct __decrement 934 { constexpr _Tp operator()(_Tp __a) const { return --__a; } }; 935 936 template <> 937 struct __decrement<void> 938 { 939 template <typename _Tp> 940 constexpr _Tp 941 operator()(_Tp __a) const 942 { return --__a; } 943 }; 944 945 // }}} 946 // _ValuePreserving(OrInt) {{{ 947 template <typename _From, typename _To, 948 typename = enable_if_t<negation< 949 __is_narrowing_conversion<__remove_cvref_t<_From>, _To>>::value>> 950 using _ValuePreserving = _From; 951 952 template <typename _From, typename _To, 953 typename _DecayedFrom = __remove_cvref_t<_From>, 954 typename = enable_if_t<conjunction< 955 is_convertible<_From, _To>, 956 disjunction< 957 is_same<_DecayedFrom, _To>, is_same<_DecayedFrom, int>, 958 conjunction<is_same<_DecayedFrom, _UInt>, is_unsigned<_To>>, 959 negation<__is_narrowing_conversion<_DecayedFrom, _To>>>>::value>> 960 using _ValuePreservingOrInt = _From; 961 962 // }}} 963 // __intrinsic_type {{{ 964 template <typename _Tp, size_t _Bytes, typename = void_t<>> 965 struct __intrinsic_type; 966 967 template <typename _Tp, size_t _Size> 968 using __intrinsic_type_t = 969 typename __intrinsic_type<_Tp, _Size * sizeof(_Tp)>::type; 970 971 template <typename _Tp> 972 using __intrinsic_type2_t = typename __intrinsic_type<_Tp, 2>::type; 973 template <typename _Tp> 974 using __intrinsic_type4_t = typename __intrinsic_type<_Tp, 4>::type; 975 template <typename _Tp> 976 using __intrinsic_type8_t = typename __intrinsic_type<_Tp, 8>::type; 977 template <typename _Tp> 978 using __intrinsic_type16_t = typename __intrinsic_type<_Tp, 16>::type; 979 template <typename _Tp> 980 using __intrinsic_type32_t = typename __intrinsic_type<_Tp, 32>::type; 981 template <typename _Tp> 982 using __intrinsic_type64_t = typename __intrinsic_type<_Tp, 64>::type; 983 984 // }}} 985 // _BitMask {{{ 986 template <size_t _Np, bool _Sanitized = false> 987 struct _BitMask; 988 989 template <size_t _Np, bool _Sanitized> 990 struct __is_bitmask<_BitMask<_Np, _Sanitized>, void> : true_type {}; 991 992 template <size_t _Np> 993 using _SanitizedBitMask = _BitMask<_Np, true>; 994 995 template <size_t _Np, bool _Sanitized> 996 struct _BitMask 997 { 998 static_assert(_Np > 0); 999 1000 static constexpr size_t _NBytes = __div_roundup(_Np, __CHAR_BIT__); 1001 1002 using _Tp = conditional_t<_Np == 1, bool, 1003 make_unsigned_t<__int_with_sizeof_t<std::min( 1004 sizeof(_ULLong), std::__bit_ceil(_NBytes))>>>; 1005 1006 static constexpr int _S_array_size = __div_roundup(_NBytes, sizeof(_Tp)); 1007 1008 _Tp _M_bits[_S_array_size]; 1009 1010 static constexpr int _S_unused_bits 1011 = _Np == 1 ? 0 : _S_array_size * sizeof(_Tp) * __CHAR_BIT__ - _Np; 1012 1013 static constexpr _Tp _S_bitmask = +_Tp(~_Tp()) >> _S_unused_bits; 1014 1015 constexpr _BitMask() noexcept = default; 1016 1017 constexpr _BitMask(unsigned long long __x) noexcept 1018 : _M_bits{static_cast<_Tp>(__x)} {} 1019 1020 _BitMask(bitset<_Np> __x) noexcept : _BitMask(__x.to_ullong()) {} 1021 1022 constexpr _BitMask(const _BitMask&) noexcept = default; 1023 1024 template <bool _RhsSanitized, typename = enable_if_t<_RhsSanitized == false 1025 && _Sanitized == true>> 1026 constexpr _BitMask(const _BitMask<_Np, _RhsSanitized>& __rhs) noexcept 1027 : _BitMask(__rhs._M_sanitized()) {} 1028 1029 constexpr operator _SimdWrapper<bool, _Np>() const noexcept 1030 { 1031 static_assert(_S_array_size == 1); 1032 return _M_bits[0]; 1033 } 1034 1035 // precondition: is sanitized 1036 constexpr _Tp 1037 _M_to_bits() const noexcept 1038 { 1039 static_assert(_S_array_size == 1); 1040 return _M_bits[0]; 1041 } 1042 1043 // precondition: is sanitized 1044 constexpr unsigned long long 1045 to_ullong() const noexcept 1046 { 1047 static_assert(_S_array_size == 1); 1048 return _M_bits[0]; 1049 } 1050 1051 // precondition: is sanitized 1052 constexpr unsigned long 1053 to_ulong() const noexcept 1054 { 1055 static_assert(_S_array_size == 1); 1056 return _M_bits[0]; 1057 } 1058 1059 constexpr bitset<_Np> 1060 _M_to_bitset() const noexcept 1061 { 1062 static_assert(_S_array_size == 1); 1063 return _M_bits[0]; 1064 } 1065 1066 constexpr decltype(auto) 1067 _M_sanitized() const noexcept 1068 { 1069 if constexpr (_Sanitized) 1070 return *this; 1071 else if constexpr (_Np == 1) 1072 return _SanitizedBitMask<_Np>(_M_bits[0]); 1073 else 1074 { 1075 _SanitizedBitMask<_Np> __r = {}; 1076 for (int __i = 0; __i < _S_array_size; ++__i) 1077 __r._M_bits[__i] = _M_bits[__i]; 1078 if constexpr (_S_unused_bits > 0) 1079 __r._M_bits[_S_array_size - 1] &= _S_bitmask; 1080 return __r; 1081 } 1082 } 1083 1084 template <size_t _Mp, bool _LSanitized> 1085 constexpr _BitMask<_Np + _Mp, _Sanitized> 1086 _M_prepend(_BitMask<_Mp, _LSanitized> __lsb) const noexcept 1087 { 1088 constexpr size_t _RN = _Np + _Mp; 1089 using _Rp = _BitMask<_RN, _Sanitized>; 1090 if constexpr (_Rp::_S_array_size == 1) 1091 { 1092 _Rp __r{{_M_bits[0]}}; 1093 __r._M_bits[0] <<= _Mp; 1094 __r._M_bits[0] |= __lsb._M_sanitized()._M_bits[0]; 1095 return __r; 1096 } 1097 else 1098 __assert_unreachable<_Rp>(); 1099 } 1100 1101 // Return a new _BitMask with size _NewSize while dropping _DropLsb least 1102 // significant bits. If the operation implicitly produces a sanitized bitmask, 1103 // the result type will have _Sanitized set. 1104 template <size_t _DropLsb, size_t _NewSize = _Np - _DropLsb> 1105 constexpr auto 1106 _M_extract() const noexcept 1107 { 1108 static_assert(_Np > _DropLsb); 1109 static_assert(_DropLsb + _NewSize <= sizeof(_ULLong) * __CHAR_BIT__, 1110 "not implemented for bitmasks larger than one ullong"); 1111 if constexpr (_NewSize == 1) 1112 // must sanitize because the return _Tp is bool 1113 return _SanitizedBitMask<1>(_M_bits[0] & (_Tp(1) << _DropLsb)); 1114 else 1115 return _BitMask<_NewSize, 1116 ((_NewSize + _DropLsb == sizeof(_Tp) * __CHAR_BIT__ 1117 && _NewSize + _DropLsb <= _Np) 1118 || ((_Sanitized || _Np == sizeof(_Tp) * __CHAR_BIT__) 1119 && _NewSize + _DropLsb >= _Np))>(_M_bits[0] 1120 >> _DropLsb); 1121 } 1122 1123 // True if all bits are set. Implicitly sanitizes if _Sanitized == false. 1124 constexpr bool 1125 all() const noexcept 1126 { 1127 if constexpr (_Np == 1) 1128 return _M_bits[0]; 1129 else if constexpr (!_Sanitized) 1130 return _M_sanitized().all(); 1131 else 1132 { 1133 constexpr _Tp __allbits = ~_Tp(); 1134 for (int __i = 0; __i < _S_array_size - 1; ++__i) 1135 if (_M_bits[__i] != __allbits) 1136 return false; 1137 return _M_bits[_S_array_size - 1] == _S_bitmask; 1138 } 1139 } 1140 1141 // True if at least one bit is set. Implicitly sanitizes if _Sanitized == 1142 // false. 1143 constexpr bool 1144 any() const noexcept 1145 { 1146 if constexpr (_Np == 1) 1147 return _M_bits[0]; 1148 else if constexpr (!_Sanitized) 1149 return _M_sanitized().any(); 1150 else 1151 { 1152 for (int __i = 0; __i < _S_array_size - 1; ++__i) 1153 if (_M_bits[__i] != 0) 1154 return true; 1155 return _M_bits[_S_array_size - 1] != 0; 1156 } 1157 } 1158 1159 // True if no bit is set. Implicitly sanitizes if _Sanitized == false. 1160 constexpr bool 1161 none() const noexcept 1162 { 1163 if constexpr (_Np == 1) 1164 return !_M_bits[0]; 1165 else if constexpr (!_Sanitized) 1166 return _M_sanitized().none(); 1167 else 1168 { 1169 for (int __i = 0; __i < _S_array_size - 1; ++__i) 1170 if (_M_bits[__i] != 0) 1171 return false; 1172 return _M_bits[_S_array_size - 1] == 0; 1173 } 1174 } 1175 1176 // Returns the number of set bits. Implicitly sanitizes if _Sanitized == 1177 // false. 1178 constexpr int 1179 count() const noexcept 1180 { 1181 if constexpr (_Np == 1) 1182 return _M_bits[0]; 1183 else if constexpr (!_Sanitized) 1184 return _M_sanitized().none(); 1185 else 1186 { 1187 int __result = __builtin_popcountll(_M_bits[0]); 1188 for (int __i = 1; __i < _S_array_size; ++__i) 1189 __result += __builtin_popcountll(_M_bits[__i]); 1190 return __result; 1191 } 1192 } 1193 1194 // Returns the bit at offset __i as bool. 1195 constexpr bool 1196 operator[](size_t __i) const noexcept 1197 { 1198 if constexpr (_Np == 1) 1199 return _M_bits[0]; 1200 else if constexpr (_S_array_size == 1) 1201 return (_M_bits[0] >> __i) & 1; 1202 else 1203 { 1204 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__); 1205 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__); 1206 return (_M_bits[__j] >> __shift) & 1; 1207 } 1208 } 1209 1210 template <size_t __i> 1211 constexpr bool 1212 operator[](_SizeConstant<__i>) const noexcept 1213 { 1214 static_assert(__i < _Np); 1215 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__); 1216 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__); 1217 return static_cast<bool>(_M_bits[__j] & (_Tp(1) << __shift)); 1218 } 1219 1220 // Set the bit at offset __i to __x. 1221 constexpr void 1222 set(size_t __i, bool __x) noexcept 1223 { 1224 if constexpr (_Np == 1) 1225 _M_bits[0] = __x; 1226 else if constexpr (_S_array_size == 1) 1227 { 1228 _M_bits[0] &= ~_Tp(_Tp(1) << __i); 1229 _M_bits[0] |= _Tp(_Tp(__x) << __i); 1230 } 1231 else 1232 { 1233 const size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__); 1234 const size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__); 1235 _M_bits[__j] &= ~_Tp(_Tp(1) << __shift); 1236 _M_bits[__j] |= _Tp(_Tp(__x) << __shift); 1237 } 1238 } 1239 1240 template <size_t __i> 1241 constexpr void 1242 set(_SizeConstant<__i>, bool __x) noexcept 1243 { 1244 static_assert(__i < _Np); 1245 if constexpr (_Np == 1) 1246 _M_bits[0] = __x; 1247 else 1248 { 1249 constexpr size_t __j = __i / (sizeof(_Tp) * __CHAR_BIT__); 1250 constexpr size_t __shift = __i % (sizeof(_Tp) * __CHAR_BIT__); 1251 constexpr _Tp __mask = ~_Tp(_Tp(1) << __shift); 1252 _M_bits[__j] &= __mask; 1253 _M_bits[__j] |= _Tp(_Tp(__x) << __shift); 1254 } 1255 } 1256 1257 // Inverts all bits. Sanitized input leads to sanitized output. 1258 constexpr _BitMask 1259 operator~() const noexcept 1260 { 1261 if constexpr (_Np == 1) 1262 return !_M_bits[0]; 1263 else 1264 { 1265 _BitMask __result{}; 1266 for (int __i = 0; __i < _S_array_size - 1; ++__i) 1267 __result._M_bits[__i] = ~_M_bits[__i]; 1268 if constexpr (_Sanitized) 1269 __result._M_bits[_S_array_size - 1] 1270 = _M_bits[_S_array_size - 1] ^ _S_bitmask; 1271 else 1272 __result._M_bits[_S_array_size - 1] = ~_M_bits[_S_array_size - 1]; 1273 return __result; 1274 } 1275 } 1276 1277 constexpr _BitMask& 1278 operator^=(const _BitMask& __b) & noexcept 1279 { 1280 __execute_n_times<_S_array_size>( 1281 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] ^= __b._M_bits[__i]; }); 1282 return *this; 1283 } 1284 1285 constexpr _BitMask& 1286 operator|=(const _BitMask& __b) & noexcept 1287 { 1288 __execute_n_times<_S_array_size>( 1289 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] |= __b._M_bits[__i]; }); 1290 return *this; 1291 } 1292 1293 constexpr _BitMask& 1294 operator&=(const _BitMask& __b) & noexcept 1295 { 1296 __execute_n_times<_S_array_size>( 1297 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { _M_bits[__i] &= __b._M_bits[__i]; }); 1298 return *this; 1299 } 1300 1301 friend constexpr _BitMask 1302 operator^(const _BitMask& __a, const _BitMask& __b) noexcept 1303 { 1304 _BitMask __r = __a; 1305 __r ^= __b; 1306 return __r; 1307 } 1308 1309 friend constexpr _BitMask 1310 operator|(const _BitMask& __a, const _BitMask& __b) noexcept 1311 { 1312 _BitMask __r = __a; 1313 __r |= __b; 1314 return __r; 1315 } 1316 1317 friend constexpr _BitMask 1318 operator&(const _BitMask& __a, const _BitMask& __b) noexcept 1319 { 1320 _BitMask __r = __a; 1321 __r &= __b; 1322 return __r; 1323 } 1324 1325 _GLIBCXX_SIMD_INTRINSIC 1326 constexpr bool 1327 _M_is_constprop() const 1328 { 1329 if constexpr (_S_array_size == 0) 1330 return __builtin_constant_p(_M_bits[0]); 1331 else 1332 { 1333 for (int __i = 0; __i < _S_array_size; ++__i) 1334 if (!__builtin_constant_p(_M_bits[__i])) 1335 return false; 1336 return true; 1337 } 1338 } 1339 }; 1340 1341 // }}} 1342 1343 // vvv ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- vvv 1344 // __min_vector_size {{{ 1345 template <typename _Tp = void> 1346 static inline constexpr int __min_vector_size = 2 * sizeof(_Tp); 1347 1348 #if _GLIBCXX_SIMD_HAVE_NEON 1349 template <> 1350 inline constexpr int __min_vector_size<void> = 8; 1351 #else 1352 template <> 1353 inline constexpr int __min_vector_size<void> = 16; 1354 #endif 1355 1356 // }}} 1357 // __vector_type {{{ 1358 template <typename _Tp, size_t _Np, typename = void> 1359 struct __vector_type_n {}; 1360 1361 // substition failure for 0-element case 1362 template <typename _Tp> 1363 struct __vector_type_n<_Tp, 0, void> {}; 1364 1365 // special case 1-element to be _Tp itself 1366 template <typename _Tp> 1367 struct __vector_type_n<_Tp, 1, enable_if_t<__is_vectorizable_v<_Tp>>> 1368 { using type = _Tp; }; 1369 1370 // else, use GNU-style builtin vector types 1371 template <typename _Tp, size_t _Np> 1372 struct __vector_type_n<_Tp, _Np, enable_if_t<__is_vectorizable_v<_Tp> && _Np >= 2>> 1373 { 1374 static constexpr size_t _S_Np2 = std::__bit_ceil(_Np * sizeof(_Tp)); 1375 1376 static constexpr size_t _S_Bytes = 1377 #ifdef __i386__ 1378 // Using [[gnu::vector_size(8)]] would wreak havoc on the FPU because 1379 // those objects are passed via MMX registers and nothing ever calls EMMS. 1380 _S_Np2 == 8 ? 16 : 1381 #endif 1382 _S_Np2 < __min_vector_size<_Tp> ? __min_vector_size<_Tp> 1383 : _S_Np2; 1384 1385 using type [[__gnu__::__vector_size__(_S_Bytes)]] = _Tp; 1386 }; 1387 1388 template <typename _Tp, size_t _Bytes, size_t = _Bytes % sizeof(_Tp)> 1389 struct __vector_type; 1390 1391 template <typename _Tp, size_t _Bytes> 1392 struct __vector_type<_Tp, _Bytes, 0> 1393 : __vector_type_n<_Tp, _Bytes / sizeof(_Tp)> {}; 1394 1395 template <typename _Tp, size_t _Size> 1396 using __vector_type_t = typename __vector_type_n<_Tp, _Size>::type; 1397 1398 template <typename _Tp> 1399 using __vector_type2_t = typename __vector_type<_Tp, 2>::type; 1400 template <typename _Tp> 1401 using __vector_type4_t = typename __vector_type<_Tp, 4>::type; 1402 template <typename _Tp> 1403 using __vector_type8_t = typename __vector_type<_Tp, 8>::type; 1404 template <typename _Tp> 1405 using __vector_type16_t = typename __vector_type<_Tp, 16>::type; 1406 template <typename _Tp> 1407 using __vector_type32_t = typename __vector_type<_Tp, 32>::type; 1408 template <typename _Tp> 1409 using __vector_type64_t = typename __vector_type<_Tp, 64>::type; 1410 1411 // }}} 1412 // __is_vector_type {{{ 1413 template <typename _Tp, typename = void_t<>> 1414 struct __is_vector_type : false_type {}; 1415 1416 template <typename _Tp> 1417 struct __is_vector_type< 1418 _Tp, void_t<typename __vector_type< 1419 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>> 1420 : is_same<_Tp, typename __vector_type< 1421 remove_reference_t<decltype(declval<_Tp>()[0])>, 1422 sizeof(_Tp)>::type> {}; 1423 1424 template <typename _Tp> 1425 inline constexpr bool __is_vector_type_v = __is_vector_type<_Tp>::value; 1426 1427 // }}} 1428 // __is_intrinsic_type {{{ 1429 #if _GLIBCXX_SIMD_HAVE_SSE_ABI 1430 template <typename _Tp> 1431 using __is_intrinsic_type = __is_vector_type<_Tp>; 1432 #else // not SSE (x86) 1433 template <typename _Tp, typename = void_t<>> 1434 struct __is_intrinsic_type : false_type {}; 1435 1436 template <typename _Tp> 1437 struct __is_intrinsic_type< 1438 _Tp, void_t<typename __intrinsic_type< 1439 remove_reference_t<decltype(declval<_Tp>()[0])>, sizeof(_Tp)>::type>> 1440 : is_same<_Tp, typename __intrinsic_type< 1441 remove_reference_t<decltype(declval<_Tp>()[0])>, 1442 sizeof(_Tp)>::type> {}; 1443 #endif 1444 1445 template <typename _Tp> 1446 inline constexpr bool __is_intrinsic_type_v = __is_intrinsic_type<_Tp>::value; 1447 1448 // }}} 1449 // _VectorTraits{{{ 1450 template <typename _Tp, typename = void_t<>> 1451 struct _VectorTraitsImpl; 1452 1453 template <typename _Tp> 1454 struct _VectorTraitsImpl<_Tp, enable_if_t<__is_vector_type_v<_Tp> 1455 || __is_intrinsic_type_v<_Tp>>> 1456 { 1457 using type = _Tp; 1458 using value_type = remove_reference_t<decltype(declval<_Tp>()[0])>; 1459 static constexpr int _S_full_size = sizeof(_Tp) / sizeof(value_type); 1460 using _Wrapper = _SimdWrapper<value_type, _S_full_size>; 1461 template <typename _Up, int _W = _S_full_size> 1462 static constexpr bool _S_is 1463 = is_same_v<value_type, _Up> && _W == _S_full_size; 1464 }; 1465 1466 template <typename _Tp, size_t _Np> 1467 struct _VectorTraitsImpl<_SimdWrapper<_Tp, _Np>, 1468 void_t<__vector_type_t<_Tp, _Np>>> 1469 { 1470 using type = __vector_type_t<_Tp, _Np>; 1471 using value_type = _Tp; 1472 static constexpr int _S_full_size = sizeof(type) / sizeof(value_type); 1473 using _Wrapper = _SimdWrapper<_Tp, _Np>; 1474 static constexpr bool _S_is_partial = (_Np == _S_full_size); 1475 static constexpr int _S_partial_width = _Np; 1476 template <typename _Up, int _W = _S_full_size> 1477 static constexpr bool _S_is 1478 = is_same_v<value_type, _Up>&& _W == _S_full_size; 1479 }; 1480 1481 template <typename _Tp, typename = typename _VectorTraitsImpl<_Tp>::type> 1482 using _VectorTraits = _VectorTraitsImpl<_Tp>; 1483 1484 // }}} 1485 // __as_vector{{{ 1486 template <typename _V> 1487 _GLIBCXX_SIMD_INTRINSIC constexpr auto 1488 __as_vector(_V __x) 1489 { 1490 if constexpr (__is_vector_type_v<_V>) 1491 return __x; 1492 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value) 1493 return __data(__x)._M_data; 1494 else if constexpr (__is_vectorizable_v<_V>) 1495 return __vector_type_t<_V, 2>{__x}; 1496 else 1497 return __x._M_data; 1498 } 1499 1500 // }}} 1501 // __as_wrapper{{{ 1502 template <size_t _Np = 0, typename _V> 1503 _GLIBCXX_SIMD_INTRINSIC constexpr auto 1504 __as_wrapper(_V __x) 1505 { 1506 if constexpr (__is_vector_type_v<_V>) 1507 return _SimdWrapper<typename _VectorTraits<_V>::value_type, 1508 (_Np > 0 ? _Np : _VectorTraits<_V>::_S_full_size)>(__x); 1509 else if constexpr (is_simd<_V>::value || is_simd_mask<_V>::value) 1510 { 1511 static_assert(_V::size() == _Np); 1512 return __data(__x); 1513 } 1514 else 1515 { 1516 static_assert(_V::_S_size == _Np); 1517 return __x; 1518 } 1519 } 1520 1521 // }}} 1522 // __intrin_bitcast{{{ 1523 template <typename _To, typename _From> 1524 _GLIBCXX_SIMD_INTRINSIC constexpr _To 1525 __intrin_bitcast(_From __v) 1526 { 1527 static_assert((__is_vector_type_v<_From> || __is_intrinsic_type_v<_From>) 1528 && (__is_vector_type_v<_To> || __is_intrinsic_type_v<_To>)); 1529 if constexpr (sizeof(_To) == sizeof(_From)) 1530 return reinterpret_cast<_To>(__v); 1531 else if constexpr (sizeof(_From) > sizeof(_To)) 1532 if constexpr (sizeof(_To) >= 16) 1533 return reinterpret_cast<const __may_alias<_To>&>(__v); 1534 else 1535 { 1536 _To __r; 1537 __builtin_memcpy(&__r, &__v, sizeof(_To)); 1538 return __r; 1539 } 1540 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__ 1541 else if constexpr (__have_avx && sizeof(_From) == 16 && sizeof(_To) == 32) 1542 return reinterpret_cast<_To>(__builtin_ia32_ps256_ps( 1543 reinterpret_cast<__vector_type_t<float, 4>>(__v))); 1544 else if constexpr (__have_avx512f && sizeof(_From) == 16 1545 && sizeof(_To) == 64) 1546 return reinterpret_cast<_To>(__builtin_ia32_ps512_ps( 1547 reinterpret_cast<__vector_type_t<float, 4>>(__v))); 1548 else if constexpr (__have_avx512f && sizeof(_From) == 32 1549 && sizeof(_To) == 64) 1550 return reinterpret_cast<_To>(__builtin_ia32_ps512_256ps( 1551 reinterpret_cast<__vector_type_t<float, 8>>(__v))); 1552 #endif // _GLIBCXX_SIMD_X86INTRIN 1553 else if constexpr (sizeof(__v) <= 8) 1554 return reinterpret_cast<_To>( 1555 __vector_type_t<__int_for_sizeof_t<_From>, sizeof(_To) / sizeof(_From)>{ 1556 reinterpret_cast<__int_for_sizeof_t<_From>>(__v)}); 1557 else 1558 { 1559 static_assert(sizeof(_To) > sizeof(_From)); 1560 _To __r = {}; 1561 __builtin_memcpy(&__r, &__v, sizeof(_From)); 1562 return __r; 1563 } 1564 } 1565 1566 // }}} 1567 // __vector_bitcast{{{ 1568 template <typename _To, size_t _NN = 0, typename _From, 1569 typename _FromVT = _VectorTraits<_From>, 1570 size_t _Np = _NN == 0 ? sizeof(_From) / sizeof(_To) : _NN> 1571 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np> 1572 __vector_bitcast(_From __x) 1573 { 1574 using _R = __vector_type_t<_To, _Np>; 1575 return __intrin_bitcast<_R>(__x); 1576 } 1577 1578 template <typename _To, size_t _NN = 0, typename _Tp, size_t _Nx, 1579 size_t _Np 1580 = _NN == 0 ? sizeof(_SimdWrapper<_Tp, _Nx>) / sizeof(_To) : _NN> 1581 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_To, _Np> 1582 __vector_bitcast(const _SimdWrapper<_Tp, _Nx>& __x) 1583 { 1584 static_assert(_Np > 1); 1585 return __intrin_bitcast<__vector_type_t<_To, _Np>>(__x._M_data); 1586 } 1587 1588 // }}} 1589 // __convert_x86 declarations {{{ 1590 #ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048 1591 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>> 1592 _To __convert_x86(_Tp); 1593 1594 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>> 1595 _To __convert_x86(_Tp, _Tp); 1596 1597 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>> 1598 _To __convert_x86(_Tp, _Tp, _Tp, _Tp); 1599 1600 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>> 1601 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp); 1602 1603 template <typename _To, typename _Tp, typename _TVT = _VectorTraits<_Tp>> 1604 _To __convert_x86(_Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, _Tp, 1605 _Tp, _Tp, _Tp, _Tp); 1606 #endif // _GLIBCXX_SIMD_WORKAROUND_PR85048 1607 1608 //}}} 1609 // __bit_cast {{{ 1610 template <typename _To, typename _From> 1611 _GLIBCXX_SIMD_INTRINSIC constexpr _To 1612 __bit_cast(const _From __x) 1613 { 1614 // TODO: implement with / replace by __builtin_bit_cast ASAP 1615 static_assert(sizeof(_To) == sizeof(_From)); 1616 constexpr bool __to_is_vectorizable 1617 = is_arithmetic_v<_To> || is_enum_v<_To>; 1618 constexpr bool __from_is_vectorizable 1619 = is_arithmetic_v<_From> || is_enum_v<_From>; 1620 if constexpr (__is_vector_type_v<_To> && __is_vector_type_v<_From>) 1621 return reinterpret_cast<_To>(__x); 1622 else if constexpr (__is_vector_type_v<_To> && __from_is_vectorizable) 1623 { 1624 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From; 1625 return reinterpret_cast<_To>(_FV{__x}); 1626 } 1627 else if constexpr (__to_is_vectorizable && __from_is_vectorizable) 1628 { 1629 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To; 1630 using _FV [[__gnu__::__vector_size__(sizeof(_From))]] = _From; 1631 return reinterpret_cast<_TV>(_FV{__x})[0]; 1632 } 1633 else if constexpr (__to_is_vectorizable && __is_vector_type_v<_From>) 1634 { 1635 using _TV [[__gnu__::__vector_size__(sizeof(_To))]] = _To; 1636 return reinterpret_cast<_TV>(__x)[0]; 1637 } 1638 else 1639 { 1640 _To __r; 1641 __builtin_memcpy(reinterpret_cast<char*>(&__r), 1642 reinterpret_cast<const char*>(&__x), sizeof(_To)); 1643 return __r; 1644 } 1645 } 1646 1647 // }}} 1648 // __to_intrin {{{ 1649 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>, 1650 typename _R = __intrinsic_type_t<typename _TVT::value_type, _TVT::_S_full_size>> 1651 _GLIBCXX_SIMD_INTRINSIC constexpr _R 1652 __to_intrin(_Tp __x) 1653 { 1654 static_assert(sizeof(__x) <= sizeof(_R), 1655 "__to_intrin may never drop values off the end"); 1656 if constexpr (sizeof(__x) == sizeof(_R)) 1657 return reinterpret_cast<_R>(__as_vector(__x)); 1658 else 1659 { 1660 using _Up = __int_for_sizeof_t<_Tp>; 1661 return reinterpret_cast<_R>( 1662 __vector_type_t<_Up, sizeof(_R) / sizeof(_Up)>{__bit_cast<_Up>(__x)}); 1663 } 1664 } 1665 1666 // }}} 1667 // __make_vector{{{ 1668 template <typename _Tp, typename... _Args> 1669 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, sizeof...(_Args)> 1670 __make_vector(const _Args&... __args) 1671 { return __vector_type_t<_Tp, sizeof...(_Args)>{static_cast<_Tp>(__args)...}; } 1672 1673 // }}} 1674 // __vector_broadcast{{{ 1675 template <size_t _Np, typename _Tp, size_t... _I> 1676 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np> 1677 __vector_broadcast_impl(_Tp __x, index_sequence<_I...>) 1678 { return __vector_type_t<_Tp, _Np>{((void)_I, __x)...}; } 1679 1680 template <size_t _Np, typename _Tp> 1681 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np> 1682 __vector_broadcast(_Tp __x) 1683 { return __vector_broadcast_impl<_Np, _Tp>(__x, make_index_sequence<_Np>()); } 1684 1685 // }}} 1686 // __generate_vector{{{ 1687 template <typename _Tp, size_t _Np, typename _Gp, size_t... _I> 1688 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np> 1689 __generate_vector_impl(_Gp&& __gen, index_sequence<_I...>) 1690 { return __vector_type_t<_Tp, _Np>{ static_cast<_Tp>(__gen(_SizeConstant<_I>()))...}; } 1691 1692 template <typename _V, typename _VVT = _VectorTraits<_V>, typename _Gp> 1693 _GLIBCXX_SIMD_INTRINSIC constexpr _V 1694 __generate_vector(_Gp&& __gen) 1695 { 1696 if constexpr (__is_vector_type_v<_V>) 1697 return __generate_vector_impl<typename _VVT::value_type, 1698 _VVT::_S_full_size>( 1699 static_cast<_Gp&&>(__gen), make_index_sequence<_VVT::_S_full_size>()); 1700 else 1701 return __generate_vector_impl<typename _VVT::value_type, 1702 _VVT::_S_partial_width>( 1703 static_cast<_Gp&&>(__gen), 1704 make_index_sequence<_VVT::_S_partial_width>()); 1705 } 1706 1707 template <typename _Tp, size_t _Np, typename _Gp> 1708 _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Tp, _Np> 1709 __generate_vector(_Gp&& __gen) 1710 { 1711 return __generate_vector_impl<_Tp, _Np>(static_cast<_Gp&&>(__gen), 1712 make_index_sequence<_Np>()); 1713 } 1714 1715 // }}} 1716 // __xor{{{ 1717 template <typename _TW> 1718 _GLIBCXX_SIMD_INTRINSIC constexpr _TW 1719 __xor(_TW __a, _TW __b) noexcept 1720 { 1721 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>) 1722 { 1723 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW, 1724 _VectorTraitsImpl<_TW>>::value_type; 1725 if constexpr (is_floating_point_v<_Tp>) 1726 { 1727 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>; 1728 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a) 1729 ^ __vector_bitcast<_Ip>(__b)); 1730 } 1731 else if constexpr (__is_vector_type_v<_TW>) 1732 return __a ^ __b; 1733 else 1734 return __a._M_data ^ __b._M_data; 1735 } 1736 else 1737 return __a ^ __b; 1738 } 1739 1740 // }}} 1741 // __or{{{ 1742 template <typename _TW> 1743 _GLIBCXX_SIMD_INTRINSIC constexpr _TW 1744 __or(_TW __a, _TW __b) noexcept 1745 { 1746 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>) 1747 { 1748 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW, 1749 _VectorTraitsImpl<_TW>>::value_type; 1750 if constexpr (is_floating_point_v<_Tp>) 1751 { 1752 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>; 1753 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a) 1754 | __vector_bitcast<_Ip>(__b)); 1755 } 1756 else if constexpr (__is_vector_type_v<_TW>) 1757 return __a | __b; 1758 else 1759 return __a._M_data | __b._M_data; 1760 } 1761 else 1762 return __a | __b; 1763 } 1764 1765 // }}} 1766 // __and{{{ 1767 template <typename _TW> 1768 _GLIBCXX_SIMD_INTRINSIC constexpr _TW 1769 __and(_TW __a, _TW __b) noexcept 1770 { 1771 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>) 1772 { 1773 using _Tp = typename conditional_t<__is_simd_wrapper_v<_TW>, _TW, 1774 _VectorTraitsImpl<_TW>>::value_type; 1775 if constexpr (is_floating_point_v<_Tp>) 1776 { 1777 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>; 1778 return __vector_bitcast<_Tp>(__vector_bitcast<_Ip>(__a) 1779 & __vector_bitcast<_Ip>(__b)); 1780 } 1781 else if constexpr (__is_vector_type_v<_TW>) 1782 return __a & __b; 1783 else 1784 return __a._M_data & __b._M_data; 1785 } 1786 else 1787 return __a & __b; 1788 } 1789 1790 // }}} 1791 // __andnot{{{ 1792 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__ 1793 static constexpr struct 1794 { 1795 _GLIBCXX_SIMD_INTRINSIC __v4sf 1796 operator()(__v4sf __a, __v4sf __b) const noexcept 1797 { return __builtin_ia32_andnps(__a, __b); } 1798 1799 _GLIBCXX_SIMD_INTRINSIC __v2df 1800 operator()(__v2df __a, __v2df __b) const noexcept 1801 { return __builtin_ia32_andnpd(__a, __b); } 1802 1803 _GLIBCXX_SIMD_INTRINSIC __v2di 1804 operator()(__v2di __a, __v2di __b) const noexcept 1805 { return __builtin_ia32_pandn128(__a, __b); } 1806 1807 _GLIBCXX_SIMD_INTRINSIC __v8sf 1808 operator()(__v8sf __a, __v8sf __b) const noexcept 1809 { return __builtin_ia32_andnps256(__a, __b); } 1810 1811 _GLIBCXX_SIMD_INTRINSIC __v4df 1812 operator()(__v4df __a, __v4df __b) const noexcept 1813 { return __builtin_ia32_andnpd256(__a, __b); } 1814 1815 _GLIBCXX_SIMD_INTRINSIC __v4di 1816 operator()(__v4di __a, __v4di __b) const noexcept 1817 { 1818 if constexpr (__have_avx2) 1819 return __builtin_ia32_andnotsi256(__a, __b); 1820 else 1821 return reinterpret_cast<__v4di>( 1822 __builtin_ia32_andnpd256(reinterpret_cast<__v4df>(__a), 1823 reinterpret_cast<__v4df>(__b))); 1824 } 1825 1826 _GLIBCXX_SIMD_INTRINSIC __v16sf 1827 operator()(__v16sf __a, __v16sf __b) const noexcept 1828 { 1829 if constexpr (__have_avx512dq) 1830 return _mm512_andnot_ps(__a, __b); 1831 else 1832 return reinterpret_cast<__v16sf>( 1833 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a), 1834 reinterpret_cast<__v8di>(__b))); 1835 } 1836 1837 _GLIBCXX_SIMD_INTRINSIC __v8df 1838 operator()(__v8df __a, __v8df __b) const noexcept 1839 { 1840 if constexpr (__have_avx512dq) 1841 return _mm512_andnot_pd(__a, __b); 1842 else 1843 return reinterpret_cast<__v8df>( 1844 _mm512_andnot_si512(reinterpret_cast<__v8di>(__a), 1845 reinterpret_cast<__v8di>(__b))); 1846 } 1847 1848 _GLIBCXX_SIMD_INTRINSIC __v8di 1849 operator()(__v8di __a, __v8di __b) const noexcept 1850 { return _mm512_andnot_si512(__a, __b); } 1851 } _S_x86_andnot; 1852 #endif // _GLIBCXX_SIMD_X86INTRIN && !__clang__ 1853 1854 template <typename _TW> 1855 _GLIBCXX_SIMD_INTRINSIC constexpr _TW 1856 __andnot(_TW __a, _TW __b) noexcept 1857 { 1858 if constexpr (__is_vector_type_v<_TW> || __is_simd_wrapper_v<_TW>) 1859 { 1860 using _TVT = conditional_t<__is_simd_wrapper_v<_TW>, _TW, 1861 _VectorTraitsImpl<_TW>>; 1862 using _Tp = typename _TVT::value_type; 1863 #if _GLIBCXX_SIMD_X86INTRIN && !defined __clang__ 1864 if constexpr (sizeof(_TW) >= 16) 1865 { 1866 const auto __ai = __to_intrin(__a); 1867 const auto __bi = __to_intrin(__b); 1868 if (!__builtin_is_constant_evaluated() 1869 && !(__builtin_constant_p(__ai) && __builtin_constant_p(__bi))) 1870 { 1871 const auto __r = _S_x86_andnot(__ai, __bi); 1872 if constexpr (is_convertible_v<decltype(__r), _TW>) 1873 return __r; 1874 else 1875 return reinterpret_cast<typename _TVT::type>(__r); 1876 } 1877 } 1878 #endif // _GLIBCXX_SIMD_X86INTRIN 1879 using _Ip = make_unsigned_t<__int_for_sizeof_t<_Tp>>; 1880 return __vector_bitcast<_Tp>(~__vector_bitcast<_Ip>(__a) 1881 & __vector_bitcast<_Ip>(__b)); 1882 } 1883 else 1884 return ~__a & __b; 1885 } 1886 1887 // }}} 1888 // __not{{{ 1889 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>> 1890 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp 1891 __not(_Tp __a) noexcept 1892 { 1893 if constexpr (is_floating_point_v<typename _TVT::value_type>) 1894 return reinterpret_cast<typename _TVT::type>( 1895 ~__vector_bitcast<unsigned>(__a)); 1896 else 1897 return ~__a; 1898 } 1899 1900 // }}} 1901 // __concat{{{ 1902 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>, 1903 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size * 2>> 1904 constexpr _R 1905 __concat(_Tp a_, _Tp b_) 1906 { 1907 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1 1908 using _W 1909 = conditional_t<is_floating_point_v<typename _TVT::value_type>, double, 1910 conditional_t<(sizeof(_Tp) >= 2 * sizeof(long long)), 1911 long long, typename _TVT::value_type>>; 1912 constexpr int input_width = sizeof(_Tp) / sizeof(_W); 1913 const auto __a = __vector_bitcast<_W>(a_); 1914 const auto __b = __vector_bitcast<_W>(b_); 1915 using _Up = __vector_type_t<_W, sizeof(_R) / sizeof(_W)>; 1916 #else 1917 constexpr int input_width = _TVT::_S_full_size; 1918 const _Tp& __a = a_; 1919 const _Tp& __b = b_; 1920 using _Up = _R; 1921 #endif 1922 if constexpr (input_width == 2) 1923 return reinterpret_cast<_R>(_Up{__a[0], __a[1], __b[0], __b[1]}); 1924 else if constexpr (input_width == 4) 1925 return reinterpret_cast<_R>( 1926 _Up{__a[0], __a[1], __a[2], __a[3], __b[0], __b[1], __b[2], __b[3]}); 1927 else if constexpr (input_width == 8) 1928 return reinterpret_cast<_R>( 1929 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], __a[7], 1930 __b[0], __b[1], __b[2], __b[3], __b[4], __b[5], __b[6], __b[7]}); 1931 else if constexpr (input_width == 16) 1932 return reinterpret_cast<_R>( 1933 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], 1934 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13], 1935 __a[14], __a[15], __b[0], __b[1], __b[2], __b[3], __b[4], 1936 __b[5], __b[6], __b[7], __b[8], __b[9], __b[10], __b[11], 1937 __b[12], __b[13], __b[14], __b[15]}); 1938 else if constexpr (input_width == 32) 1939 return reinterpret_cast<_R>( 1940 _Up{__a[0], __a[1], __a[2], __a[3], __a[4], __a[5], __a[6], 1941 __a[7], __a[8], __a[9], __a[10], __a[11], __a[12], __a[13], 1942 __a[14], __a[15], __a[16], __a[17], __a[18], __a[19], __a[20], 1943 __a[21], __a[22], __a[23], __a[24], __a[25], __a[26], __a[27], 1944 __a[28], __a[29], __a[30], __a[31], __b[0], __b[1], __b[2], 1945 __b[3], __b[4], __b[5], __b[6], __b[7], __b[8], __b[9], 1946 __b[10], __b[11], __b[12], __b[13], __b[14], __b[15], __b[16], 1947 __b[17], __b[18], __b[19], __b[20], __b[21], __b[22], __b[23], 1948 __b[24], __b[25], __b[26], __b[27], __b[28], __b[29], __b[30], 1949 __b[31]}); 1950 } 1951 1952 // }}} 1953 // __zero_extend {{{ 1954 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>> 1955 struct _ZeroExtendProxy 1956 { 1957 using value_type = typename _TVT::value_type; 1958 static constexpr size_t _Np = _TVT::_S_full_size; 1959 const _Tp __x; 1960 1961 template <typename _To, typename _ToVT = _VectorTraits<_To>, 1962 typename 1963 = enable_if_t<is_same_v<typename _ToVT::value_type, value_type>>> 1964 _GLIBCXX_SIMD_INTRINSIC operator _To() const 1965 { 1966 constexpr size_t _ToN = _ToVT::_S_full_size; 1967 if constexpr (_ToN == _Np) 1968 return __x; 1969 else if constexpr (_ToN == 2 * _Np) 1970 { 1971 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3 1972 if constexpr (__have_avx && _TVT::template _S_is<float, 4>) 1973 return __vector_bitcast<value_type>( 1974 _mm256_insertf128_ps(__m256(), __x, 0)); 1975 else if constexpr (__have_avx && _TVT::template _S_is<double, 2>) 1976 return __vector_bitcast<value_type>( 1977 _mm256_insertf128_pd(__m256d(), __x, 0)); 1978 else if constexpr (__have_avx2 && _Np * sizeof(value_type) == 16) 1979 return __vector_bitcast<value_type>( 1980 _mm256_insertf128_si256(__m256i(), __to_intrin(__x), 0)); 1981 else if constexpr (__have_avx512f && _TVT::template _S_is<float, 8>) 1982 { 1983 if constexpr (__have_avx512dq) 1984 return __vector_bitcast<value_type>( 1985 _mm512_insertf32x8(__m512(), __x, 0)); 1986 else 1987 return reinterpret_cast<__m512>( 1988 _mm512_insertf64x4(__m512d(), 1989 reinterpret_cast<__m256d>(__x), 0)); 1990 } 1991 else if constexpr (__have_avx512f 1992 && _TVT::template _S_is<double, 4>) 1993 return __vector_bitcast<value_type>( 1994 _mm512_insertf64x4(__m512d(), __x, 0)); 1995 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 32) 1996 return __vector_bitcast<value_type>( 1997 _mm512_inserti64x4(__m512i(), __to_intrin(__x), 0)); 1998 #endif 1999 return __concat(__x, _Tp()); 2000 } 2001 else if constexpr (_ToN == 4 * _Np) 2002 { 2003 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_3 2004 if constexpr (__have_avx512dq && _TVT::template _S_is<double, 2>) 2005 { 2006 return __vector_bitcast<value_type>( 2007 _mm512_insertf64x2(__m512d(), __x, 0)); 2008 } 2009 else if constexpr (__have_avx512f 2010 && is_floating_point_v<value_type>) 2011 { 2012 return __vector_bitcast<value_type>( 2013 _mm512_insertf32x4(__m512(), reinterpret_cast<__m128>(__x), 2014 0)); 2015 } 2016 else if constexpr (__have_avx512f && _Np * sizeof(value_type) == 16) 2017 { 2018 return __vector_bitcast<value_type>( 2019 _mm512_inserti32x4(__m512i(), __to_intrin(__x), 0)); 2020 } 2021 #endif 2022 return __concat(__concat(__x, _Tp()), 2023 __vector_type_t<value_type, _Np * 2>()); 2024 } 2025 else if constexpr (_ToN == 8 * _Np) 2026 return __concat(operator __vector_type_t<value_type, _Np * 4>(), 2027 __vector_type_t<value_type, _Np * 4>()); 2028 else if constexpr (_ToN == 16 * _Np) 2029 return __concat(operator __vector_type_t<value_type, _Np * 8>(), 2030 __vector_type_t<value_type, _Np * 8>()); 2031 else 2032 __assert_unreachable<_Tp>(); 2033 } 2034 }; 2035 2036 template <typename _Tp, typename _TVT = _VectorTraits<_Tp>> 2037 _GLIBCXX_SIMD_INTRINSIC _ZeroExtendProxy<_Tp, _TVT> 2038 __zero_extend(_Tp __x) 2039 { return {__x}; } 2040 2041 // }}} 2042 // __extract<_Np, By>{{{ 2043 template <int _Offset, 2044 int _SplitBy, 2045 typename _Tp, 2046 typename _TVT = _VectorTraits<_Tp>, 2047 typename _R = __vector_type_t<typename _TVT::value_type, _TVT::_S_full_size / _SplitBy>> 2048 _GLIBCXX_SIMD_INTRINSIC constexpr _R 2049 __extract(_Tp __in) 2050 { 2051 using value_type = typename _TVT::value_type; 2052 #if _GLIBCXX_SIMD_X86INTRIN // {{{ 2053 if constexpr (sizeof(_Tp) == 64 && _SplitBy == 4 && _Offset > 0) 2054 { 2055 if constexpr (__have_avx512dq && is_same_v<double, value_type>) 2056 return _mm512_extractf64x2_pd(__to_intrin(__in), _Offset); 2057 else if constexpr (is_floating_point_v<value_type>) 2058 return __vector_bitcast<value_type>( 2059 _mm512_extractf32x4_ps(__intrin_bitcast<__m512>(__in), _Offset)); 2060 else 2061 return reinterpret_cast<_R>( 2062 _mm512_extracti32x4_epi32(__intrin_bitcast<__m512i>(__in), 2063 _Offset)); 2064 } 2065 else 2066 #endif // _GLIBCXX_SIMD_X86INTRIN }}} 2067 { 2068 #ifdef _GLIBCXX_SIMD_WORKAROUND_XXX_1 2069 using _W = conditional_t< 2070 is_floating_point_v<value_type>, double, 2071 conditional_t<(sizeof(_R) >= 16), long long, value_type>>; 2072 static_assert(sizeof(_R) % sizeof(_W) == 0); 2073 constexpr int __return_width = sizeof(_R) / sizeof(_W); 2074 using _Up = __vector_type_t<_W, __return_width>; 2075 const auto __x = __vector_bitcast<_W>(__in); 2076 #else 2077 constexpr int __return_width = _TVT::_S_full_size / _SplitBy; 2078 using _Up = _R; 2079 const __vector_type_t<value_type, _TVT::_S_full_size>& __x 2080 = __in; // only needed for _Tp = _SimdWrapper<value_type, _Np> 2081 #endif 2082 constexpr int _O = _Offset * __return_width; 2083 return __call_with_subscripts<__return_width, _O>( 2084 __x, [](auto... __entries) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 2085 return reinterpret_cast<_R>(_Up{__entries...}); 2086 }); 2087 } 2088 } 2089 2090 // }}} 2091 // __lo/__hi64[z]{{{ 2092 template <typename _Tp, 2093 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>> 2094 _GLIBCXX_SIMD_INTRINSIC constexpr _R 2095 __lo64(_Tp __x) 2096 { 2097 _R __r{}; 2098 __builtin_memcpy(&__r, &__x, 8); 2099 return __r; 2100 } 2101 2102 template <typename _Tp, 2103 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>> 2104 _GLIBCXX_SIMD_INTRINSIC constexpr _R 2105 __hi64(_Tp __x) 2106 { 2107 static_assert(sizeof(_Tp) == 16, "use __hi64z if you meant it"); 2108 _R __r{}; 2109 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8); 2110 return __r; 2111 } 2112 2113 template <typename _Tp, 2114 typename _R = __vector_type8_t<typename _VectorTraits<_Tp>::value_type>> 2115 _GLIBCXX_SIMD_INTRINSIC constexpr _R 2116 __hi64z([[maybe_unused]] _Tp __x) 2117 { 2118 _R __r{}; 2119 if constexpr (sizeof(_Tp) == 16) 2120 __builtin_memcpy(&__r, reinterpret_cast<const char*>(&__x) + 8, 8); 2121 return __r; 2122 } 2123 2124 // }}} 2125 // __lo/__hi128{{{ 2126 template <typename _Tp> 2127 _GLIBCXX_SIMD_INTRINSIC constexpr auto 2128 __lo128(_Tp __x) 2129 { return __extract<0, sizeof(_Tp) / 16>(__x); } 2130 2131 template <typename _Tp> 2132 _GLIBCXX_SIMD_INTRINSIC constexpr auto 2133 __hi128(_Tp __x) 2134 { 2135 static_assert(sizeof(__x) == 32); 2136 return __extract<1, 2>(__x); 2137 } 2138 2139 // }}} 2140 // __lo/__hi256{{{ 2141 template <typename _Tp> 2142 _GLIBCXX_SIMD_INTRINSIC constexpr auto 2143 __lo256(_Tp __x) 2144 { 2145 static_assert(sizeof(__x) == 64); 2146 return __extract<0, 2>(__x); 2147 } 2148 2149 template <typename _Tp> 2150 _GLIBCXX_SIMD_INTRINSIC constexpr auto 2151 __hi256(_Tp __x) 2152 { 2153 static_assert(sizeof(__x) == 64); 2154 return __extract<1, 2>(__x); 2155 } 2156 2157 // }}} 2158 // __auto_bitcast{{{ 2159 template <typename _Tp> 2160 struct _AutoCast 2161 { 2162 static_assert(__is_vector_type_v<_Tp>); 2163 2164 const _Tp __x; 2165 2166 template <typename _Up, typename _UVT = _VectorTraits<_Up>> 2167 _GLIBCXX_SIMD_INTRINSIC constexpr operator _Up() const 2168 { return __intrin_bitcast<typename _UVT::type>(__x); } 2169 }; 2170 2171 template <typename _Tp> 2172 _GLIBCXX_SIMD_INTRINSIC constexpr _AutoCast<_Tp> 2173 __auto_bitcast(const _Tp& __x) 2174 { return {__x}; } 2175 2176 template <typename _Tp, size_t _Np> 2177 _GLIBCXX_SIMD_INTRINSIC constexpr 2178 _AutoCast<typename _SimdWrapper<_Tp, _Np>::_BuiltinType> 2179 __auto_bitcast(const _SimdWrapper<_Tp, _Np>& __x) 2180 { return {__x._M_data}; } 2181 2182 // }}} 2183 // ^^^ ---- builtin vector types [[gnu::vector_size(N)]] and operations ---- ^^^ 2184 2185 #if _GLIBCXX_SIMD_HAVE_SSE_ABI 2186 // __bool_storage_member_type{{{ 2187 #if _GLIBCXX_SIMD_HAVE_AVX512F && _GLIBCXX_SIMD_X86INTRIN 2188 template <size_t _Size> 2189 struct __bool_storage_member_type 2190 { 2191 static_assert((_Size & (_Size - 1)) != 0, 2192 "This trait may only be used for non-power-of-2 sizes. " 2193 "Power-of-2 sizes must be specialized."); 2194 using type = 2195 typename __bool_storage_member_type<std::__bit_ceil(_Size)>::type; 2196 }; 2197 2198 template <> 2199 struct __bool_storage_member_type<1> { using type = bool; }; 2200 2201 template <> 2202 struct __bool_storage_member_type<2> { using type = __mmask8; }; 2203 2204 template <> 2205 struct __bool_storage_member_type<4> { using type = __mmask8; }; 2206 2207 template <> 2208 struct __bool_storage_member_type<8> { using type = __mmask8; }; 2209 2210 template <> 2211 struct __bool_storage_member_type<16> { using type = __mmask16; }; 2212 2213 template <> 2214 struct __bool_storage_member_type<32> { using type = __mmask32; }; 2215 2216 template <> 2217 struct __bool_storage_member_type<64> { using type = __mmask64; }; 2218 #endif // _GLIBCXX_SIMD_HAVE_AVX512F 2219 2220 // }}} 2221 // __intrinsic_type (x86){{{ 2222 // the following excludes bool via __is_vectorizable 2223 #if _GLIBCXX_SIMD_HAVE_SSE 2224 template <typename _Tp, size_t _Bytes> 2225 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 64>> 2226 { 2227 static_assert(!is_same_v<_Tp, long double>, 2228 "no __intrinsic_type support for long double on x86"); 2229 2230 static constexpr size_t _S_VBytes = _Bytes <= 16 ? 16 : _Bytes <= 32 ? 32 : 64; 2231 2232 using type [[__gnu__::__vector_size__(_S_VBytes)]] 2233 = conditional_t<is_integral_v<_Tp>, long long int, _Tp>; 2234 }; 2235 #endif // _GLIBCXX_SIMD_HAVE_SSE 2236 2237 // }}} 2238 #endif // _GLIBCXX_SIMD_HAVE_SSE_ABI 2239 // __intrinsic_type (ARM){{{ 2240 #if _GLIBCXX_SIMD_HAVE_NEON 2241 template <> 2242 struct __intrinsic_type<float, 8, void> 2243 { using type = float32x2_t; }; 2244 2245 template <> 2246 struct __intrinsic_type<float, 16, void> 2247 { using type = float32x4_t; }; 2248 2249 template <> 2250 struct __intrinsic_type<double, 8, void> 2251 { 2252 #if _GLIBCXX_SIMD_HAVE_NEON_A64 2253 using type = float64x1_t; 2254 #endif 2255 }; 2256 2257 template <> 2258 struct __intrinsic_type<double, 16, void> 2259 { 2260 #if _GLIBCXX_SIMD_HAVE_NEON_A64 2261 using type = float64x2_t; 2262 #endif 2263 }; 2264 2265 #define _GLIBCXX_SIMD_ARM_INTRIN(_Bits, _Np) \ 2266 template <> \ 2267 struct __intrinsic_type<__int_with_sizeof_t<_Bits / 8>, \ 2268 _Np * _Bits / 8, void> \ 2269 { using type = int##_Bits##x##_Np##_t; }; \ 2270 template <> \ 2271 struct __intrinsic_type<make_unsigned_t<__int_with_sizeof_t<_Bits / 8>>, \ 2272 _Np * _Bits / 8, void> \ 2273 { using type = uint##_Bits##x##_Np##_t; } 2274 _GLIBCXX_SIMD_ARM_INTRIN(8, 8); 2275 _GLIBCXX_SIMD_ARM_INTRIN(8, 16); 2276 _GLIBCXX_SIMD_ARM_INTRIN(16, 4); 2277 _GLIBCXX_SIMD_ARM_INTRIN(16, 8); 2278 _GLIBCXX_SIMD_ARM_INTRIN(32, 2); 2279 _GLIBCXX_SIMD_ARM_INTRIN(32, 4); 2280 _GLIBCXX_SIMD_ARM_INTRIN(64, 1); 2281 _GLIBCXX_SIMD_ARM_INTRIN(64, 2); 2282 #undef _GLIBCXX_SIMD_ARM_INTRIN 2283 2284 template <typename _Tp, size_t _Bytes> 2285 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>> 2286 { 2287 static constexpr int _SVecBytes = _Bytes <= 8 ? 8 : 16; 2288 2289 using _Ip = __int_for_sizeof_t<_Tp>; 2290 2291 using _Up = conditional_t< 2292 is_floating_point_v<_Tp>, _Tp, 2293 conditional_t<is_unsigned_v<_Tp>, make_unsigned_t<_Ip>, _Ip>>; 2294 2295 static_assert(!is_same_v<_Tp, _Up> || _SVecBytes != _Bytes, 2296 "should use explicit specialization above"); 2297 2298 using type = typename __intrinsic_type<_Up, _SVecBytes>::type; 2299 }; 2300 #endif // _GLIBCXX_SIMD_HAVE_NEON 2301 2302 // }}} 2303 // __intrinsic_type (PPC){{{ 2304 #ifdef __ALTIVEC__ 2305 template <typename _Tp> 2306 struct __intrinsic_type_impl; 2307 2308 #define _GLIBCXX_SIMD_PPC_INTRIN(_Tp) \ 2309 template <> \ 2310 struct __intrinsic_type_impl<_Tp> { using type = __vector _Tp; } 2311 _GLIBCXX_SIMD_PPC_INTRIN(float); 2312 #ifdef __VSX__ 2313 _GLIBCXX_SIMD_PPC_INTRIN(double); 2314 #endif 2315 _GLIBCXX_SIMD_PPC_INTRIN(signed char); 2316 _GLIBCXX_SIMD_PPC_INTRIN(unsigned char); 2317 _GLIBCXX_SIMD_PPC_INTRIN(signed short); 2318 _GLIBCXX_SIMD_PPC_INTRIN(unsigned short); 2319 _GLIBCXX_SIMD_PPC_INTRIN(signed int); 2320 _GLIBCXX_SIMD_PPC_INTRIN(unsigned int); 2321 #if defined __VSX__ || __SIZEOF_LONG__ == 4 2322 _GLIBCXX_SIMD_PPC_INTRIN(signed long); 2323 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long); 2324 #endif 2325 #ifdef __VSX__ 2326 _GLIBCXX_SIMD_PPC_INTRIN(signed long long); 2327 _GLIBCXX_SIMD_PPC_INTRIN(unsigned long long); 2328 #endif 2329 #undef _GLIBCXX_SIMD_PPC_INTRIN 2330 2331 template <typename _Tp, size_t _Bytes> 2332 struct __intrinsic_type<_Tp, _Bytes, enable_if_t<__is_vectorizable_v<_Tp> && _Bytes <= 16>> 2333 { 2334 static constexpr bool _S_is_ldouble = is_same_v<_Tp, long double>; 2335 2336 // allow _Tp == long double with -mlong-double-64 2337 static_assert(!(_S_is_ldouble && sizeof(long double) > sizeof(double)), 2338 "no __intrinsic_type support for 128-bit floating point on PowerPC"); 2339 2340 #ifndef __VSX__ 2341 static_assert(!(is_same_v<_Tp, double> 2342 || (_S_is_ldouble && sizeof(long double) == sizeof(double))), 2343 "no __intrinsic_type support for 64-bit floating point on PowerPC w/o VSX"); 2344 #endif 2345 2346 static constexpr auto __element_type() 2347 { 2348 if constexpr (is_floating_point_v<_Tp>) 2349 { 2350 if constexpr (_S_is_ldouble) 2351 return double {}; 2352 else 2353 return _Tp {}; 2354 } 2355 else if constexpr (is_signed_v<_Tp>) 2356 { 2357 if constexpr (sizeof(_Tp) == sizeof(_SChar)) 2358 return _SChar {}; 2359 else if constexpr (sizeof(_Tp) == sizeof(short)) 2360 return short {}; 2361 else if constexpr (sizeof(_Tp) == sizeof(int)) 2362 return int {}; 2363 else if constexpr (sizeof(_Tp) == sizeof(_LLong)) 2364 return _LLong {}; 2365 } 2366 else 2367 { 2368 if constexpr (sizeof(_Tp) == sizeof(_UChar)) 2369 return _UChar {}; 2370 else if constexpr (sizeof(_Tp) == sizeof(_UShort)) 2371 return _UShort {}; 2372 else if constexpr (sizeof(_Tp) == sizeof(_UInt)) 2373 return _UInt {}; 2374 else if constexpr (sizeof(_Tp) == sizeof(_ULLong)) 2375 return _ULLong {}; 2376 } 2377 } 2378 2379 using type = typename __intrinsic_type_impl<decltype(__element_type())>::type; 2380 }; 2381 #endif // __ALTIVEC__ 2382 2383 // }}} 2384 // _SimdWrapper<bool>{{{1 2385 template <size_t _Width> 2386 struct _SimdWrapper<bool, _Width, 2387 void_t<typename __bool_storage_member_type<_Width>::type>> 2388 { 2389 using _BuiltinType = typename __bool_storage_member_type<_Width>::type; 2390 using value_type = bool; 2391 2392 static constexpr size_t _S_full_size = sizeof(_BuiltinType) * __CHAR_BIT__; 2393 2394 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<bool, _S_full_size> 2395 __as_full_vector() const 2396 { return _M_data; } 2397 2398 _GLIBCXX_SIMD_INTRINSIC constexpr 2399 _SimdWrapper() = default; 2400 2401 _GLIBCXX_SIMD_INTRINSIC constexpr 2402 _SimdWrapper(_BuiltinType __k) : _M_data(__k) {}; 2403 2404 _GLIBCXX_SIMD_INTRINSIC 2405 operator const _BuiltinType&() const 2406 { return _M_data; } 2407 2408 _GLIBCXX_SIMD_INTRINSIC 2409 operator _BuiltinType&() 2410 { return _M_data; } 2411 2412 _GLIBCXX_SIMD_INTRINSIC _BuiltinType 2413 __intrin() const 2414 { return _M_data; } 2415 2416 _GLIBCXX_SIMD_INTRINSIC constexpr value_type 2417 operator[](size_t __i) const 2418 { return _M_data & (_BuiltinType(1) << __i); } 2419 2420 template <size_t __i> 2421 _GLIBCXX_SIMD_INTRINSIC constexpr value_type 2422 operator[](_SizeConstant<__i>) const 2423 { return _M_data & (_BuiltinType(1) << __i); } 2424 2425 _GLIBCXX_SIMD_INTRINSIC constexpr void 2426 _M_set(size_t __i, value_type __x) 2427 { 2428 if (__x) 2429 _M_data |= (_BuiltinType(1) << __i); 2430 else 2431 _M_data &= ~(_BuiltinType(1) << __i); 2432 } 2433 2434 _GLIBCXX_SIMD_INTRINSIC constexpr bool 2435 _M_is_constprop() const 2436 { return __builtin_constant_p(_M_data); } 2437 2438 _GLIBCXX_SIMD_INTRINSIC constexpr bool 2439 _M_is_constprop_none_of() const 2440 { 2441 if (__builtin_constant_p(_M_data)) 2442 { 2443 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__; 2444 constexpr _BuiltinType __active_mask 2445 = ~_BuiltinType() >> (__nbits - _Width); 2446 return (_M_data & __active_mask) == 0; 2447 } 2448 return false; 2449 } 2450 2451 _GLIBCXX_SIMD_INTRINSIC constexpr bool 2452 _M_is_constprop_all_of() const 2453 { 2454 if (__builtin_constant_p(_M_data)) 2455 { 2456 constexpr int __nbits = sizeof(_BuiltinType) * __CHAR_BIT__; 2457 constexpr _BuiltinType __active_mask 2458 = ~_BuiltinType() >> (__nbits - _Width); 2459 return (_M_data & __active_mask) == __active_mask; 2460 } 2461 return false; 2462 } 2463 2464 _BuiltinType _M_data; 2465 }; 2466 2467 // _SimdWrapperBase{{{1 2468 template <bool _MustZeroInitPadding, typename _BuiltinType> 2469 struct _SimdWrapperBase; 2470 2471 template <typename _BuiltinType> 2472 struct _SimdWrapperBase<false, _BuiltinType> // no padding or no SNaNs 2473 { 2474 _GLIBCXX_SIMD_INTRINSIC constexpr 2475 _SimdWrapperBase() = default; 2476 2477 _GLIBCXX_SIMD_INTRINSIC constexpr 2478 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {} 2479 2480 _BuiltinType _M_data; 2481 }; 2482 2483 template <typename _BuiltinType> 2484 struct _SimdWrapperBase<true, _BuiltinType> // with padding that needs to 2485 // never become SNaN 2486 { 2487 _GLIBCXX_SIMD_INTRINSIC constexpr 2488 _SimdWrapperBase() : _M_data() {} 2489 2490 _GLIBCXX_SIMD_INTRINSIC constexpr 2491 _SimdWrapperBase(_BuiltinType __init) : _M_data(__init) {} 2492 2493 _BuiltinType _M_data; 2494 }; 2495 2496 // }}} 2497 // _SimdWrapper{{{ 2498 template <typename _Tp, size_t _Width> 2499 struct _SimdWrapper< 2500 _Tp, _Width, 2501 void_t<__vector_type_t<_Tp, _Width>, __intrinsic_type_t<_Tp, _Width>>> 2502 : _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value 2503 && sizeof(_Tp) * _Width 2504 == sizeof(__vector_type_t<_Tp, _Width>), 2505 __vector_type_t<_Tp, _Width>> 2506 { 2507 using _Base 2508 = _SimdWrapperBase<__has_iec559_behavior<__signaling_NaN, _Tp>::value 2509 && sizeof(_Tp) * _Width 2510 == sizeof(__vector_type_t<_Tp, _Width>), 2511 __vector_type_t<_Tp, _Width>>; 2512 2513 static_assert(__is_vectorizable_v<_Tp>); 2514 static_assert(_Width >= 2); // 1 doesn't make sense, use _Tp directly then 2515 2516 using _BuiltinType = __vector_type_t<_Tp, _Width>; 2517 using value_type = _Tp; 2518 2519 static inline constexpr size_t _S_full_size 2520 = sizeof(_BuiltinType) / sizeof(value_type); 2521 static inline constexpr int _S_size = _Width; 2522 static inline constexpr bool _S_is_partial = _S_full_size != _S_size; 2523 2524 using _Base::_M_data; 2525 2526 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<_Tp, _S_full_size> 2527 __as_full_vector() const 2528 { return _M_data; } 2529 2530 _GLIBCXX_SIMD_INTRINSIC constexpr 2531 _SimdWrapper(initializer_list<_Tp> __init) 2532 : _Base(__generate_from_n_evaluations<_Width, _BuiltinType>( 2533 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 2534 return __init.begin()[__i.value]; 2535 })) {} 2536 2537 _GLIBCXX_SIMD_INTRINSIC constexpr 2538 _SimdWrapper() = default; 2539 2540 _GLIBCXX_SIMD_INTRINSIC constexpr 2541 _SimdWrapper(const _SimdWrapper&) = default; 2542 2543 _GLIBCXX_SIMD_INTRINSIC constexpr 2544 _SimdWrapper(_SimdWrapper&&) = default; 2545 2546 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper& 2547 operator=(const _SimdWrapper&) = default; 2548 2549 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper& 2550 operator=(_SimdWrapper&&) = default; 2551 2552 template <typename _V, typename = enable_if_t<disjunction_v< 2553 is_same<_V, __vector_type_t<_Tp, _Width>>, 2554 is_same<_V, __intrinsic_type_t<_Tp, _Width>>>>> 2555 _GLIBCXX_SIMD_INTRINSIC constexpr 2556 _SimdWrapper(_V __x) 2557 // __vector_bitcast can convert e.g. __m128 to __vector(2) float 2558 : _Base(__vector_bitcast<_Tp, _Width>(__x)) {} 2559 2560 template <typename... _As, 2561 typename = enable_if_t<((is_same_v<simd_abi::scalar, _As> && ...) 2562 && sizeof...(_As) <= _Width)>> 2563 _GLIBCXX_SIMD_INTRINSIC constexpr 2564 operator _SimdTuple<_Tp, _As...>() const 2565 { 2566 return __generate_from_n_evaluations<sizeof...(_As), _SimdTuple<_Tp, _As...>>( 2567 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 2568 { return _M_data[int(__i)]; }); 2569 } 2570 2571 _GLIBCXX_SIMD_INTRINSIC constexpr 2572 operator const _BuiltinType&() const 2573 { return _M_data; } 2574 2575 _GLIBCXX_SIMD_INTRINSIC constexpr 2576 operator _BuiltinType&() 2577 { return _M_data; } 2578 2579 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp 2580 operator[](size_t __i) const 2581 { return _M_data[__i]; } 2582 2583 template <size_t __i> 2584 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp 2585 operator[](_SizeConstant<__i>) const 2586 { return _M_data[__i]; } 2587 2588 _GLIBCXX_SIMD_INTRINSIC constexpr void 2589 _M_set(size_t __i, _Tp __x) 2590 { 2591 if (__builtin_is_constant_evaluated()) 2592 _M_data = __generate_from_n_evaluations<_Width, _BuiltinType>([&](auto __j) { 2593 return __j == __i ? __x : _M_data[__j()]; 2594 }); 2595 else 2596 _M_data[__i] = __x; 2597 } 2598 2599 _GLIBCXX_SIMD_INTRINSIC 2600 constexpr bool 2601 _M_is_constprop() const 2602 { return __builtin_constant_p(_M_data); } 2603 2604 _GLIBCXX_SIMD_INTRINSIC constexpr bool 2605 _M_is_constprop_none_of() const 2606 { 2607 if (__builtin_constant_p(_M_data)) 2608 { 2609 bool __r = true; 2610 if constexpr (is_floating_point_v<_Tp>) 2611 { 2612 using _Ip = __int_for_sizeof_t<_Tp>; 2613 const auto __intdata = __vector_bitcast<_Ip>(_M_data); 2614 __execute_n_times<_Width>( 2615 [&](auto __i) { __r &= __intdata[__i.value] == _Ip(); }); 2616 } 2617 else 2618 __execute_n_times<_Width>( 2619 [&](auto __i) { __r &= _M_data[__i.value] == _Tp(); }); 2620 if (__builtin_constant_p(__r)) 2621 return __r; 2622 } 2623 return false; 2624 } 2625 2626 _GLIBCXX_SIMD_INTRINSIC constexpr bool 2627 _M_is_constprop_all_of() const 2628 { 2629 if (__builtin_constant_p(_M_data)) 2630 { 2631 bool __r = true; 2632 if constexpr (is_floating_point_v<_Tp>) 2633 { 2634 using _Ip = __int_for_sizeof_t<_Tp>; 2635 const auto __intdata = __vector_bitcast<_Ip>(_M_data); 2636 __execute_n_times<_Width>( 2637 [&](auto __i) { __r &= __intdata[__i.value] == ~_Ip(); }); 2638 } 2639 else 2640 __execute_n_times<_Width>( 2641 [&](auto __i) { __r &= _M_data[__i.value] == ~_Tp(); }); 2642 if (__builtin_constant_p(__r)) 2643 return __r; 2644 } 2645 return false; 2646 } 2647 }; 2648 2649 // }}} 2650 2651 // __vectorized_sizeof {{{ 2652 template <typename _Tp> 2653 constexpr size_t 2654 __vectorized_sizeof() 2655 { 2656 if constexpr (!__is_vectorizable_v<_Tp>) 2657 return 0; 2658 2659 if constexpr (sizeof(_Tp) <= 8) 2660 { 2661 // X86: 2662 if constexpr (__have_avx512bw) 2663 return 64; 2664 if constexpr (__have_avx512f && sizeof(_Tp) >= 4) 2665 return 64; 2666 if constexpr (__have_avx2) 2667 return 32; 2668 if constexpr (__have_avx && is_floating_point_v<_Tp>) 2669 return 32; 2670 if constexpr (__have_sse2) 2671 return 16; 2672 if constexpr (__have_sse && is_same_v<_Tp, float>) 2673 return 16; 2674 /* The following is too much trouble because of mixed MMX and x87 code. 2675 * While nothing here explicitly calls MMX instructions of registers, 2676 * they are still emitted but no EMMS cleanup is done. 2677 if constexpr (__have_mmx && sizeof(_Tp) <= 4 && is_integral_v<_Tp>) 2678 return 8; 2679 */ 2680 2681 // PowerPC: 2682 if constexpr (__have_power8vec 2683 || (__have_power_vmx && (sizeof(_Tp) < 8)) 2684 || (__have_power_vsx && is_floating_point_v<_Tp>) ) 2685 return 16; 2686 2687 // ARM: 2688 if constexpr (__have_neon_a64 2689 || (__have_neon_a32 && !is_same_v<_Tp, double>) ) 2690 return 16; 2691 if constexpr (__have_neon 2692 && sizeof(_Tp) < 8 2693 // Only allow fp if the user allows non-ICE559 fp (e.g. 2694 // via -ffast-math). ARMv7 NEON fp is not conforming to 2695 // IEC559. 2696 && (__support_neon_float || !is_floating_point_v<_Tp>)) 2697 return 16; 2698 } 2699 2700 return sizeof(_Tp); 2701 } 2702 2703 // }}} 2704 namespace simd_abi { 2705 // most of simd_abi is defined in simd_detail.h 2706 template <typename _Tp> 2707 inline constexpr int max_fixed_size 2708 = (__have_avx512bw && sizeof(_Tp) == 1) ? 64 : 32; 2709 2710 // compatible {{{ 2711 #if defined __x86_64__ || defined __aarch64__ 2712 template <typename _Tp> 2713 using compatible = conditional_t<(sizeof(_Tp) <= 8), _VecBuiltin<16>, scalar>; 2714 #elif defined __ARM_NEON 2715 // FIXME: not sure, probably needs to be scalar (or dependent on the hard-float 2716 // ABI?) 2717 template <typename _Tp> 2718 using compatible 2719 = conditional_t<(sizeof(_Tp) < 8 2720 && (__support_neon_float || !is_floating_point_v<_Tp>)), 2721 _VecBuiltin<16>, scalar>; 2722 #else 2723 template <typename> 2724 using compatible = scalar; 2725 #endif 2726 2727 // }}} 2728 // native {{{ 2729 template <typename _Tp> 2730 constexpr auto 2731 __determine_native_abi() 2732 { 2733 constexpr size_t __bytes = __vectorized_sizeof<_Tp>(); 2734 if constexpr (__bytes == sizeof(_Tp)) 2735 return static_cast<scalar*>(nullptr); 2736 else if constexpr (__have_avx512vl || (__have_avx512f && __bytes == 64)) 2737 return static_cast<_VecBltnBtmsk<__bytes>*>(nullptr); 2738 else 2739 return static_cast<_VecBuiltin<__bytes>*>(nullptr); 2740 } 2741 2742 template <typename _Tp, typename = enable_if_t<__is_vectorizable_v<_Tp>>> 2743 using native = remove_pointer_t<decltype(__determine_native_abi<_Tp>())>; 2744 2745 // }}} 2746 // __default_abi {{{ 2747 #if defined _GLIBCXX_SIMD_DEFAULT_ABI 2748 template <typename _Tp> 2749 using __default_abi = _GLIBCXX_SIMD_DEFAULT_ABI<_Tp>; 2750 #else 2751 template <typename _Tp> 2752 using __default_abi = compatible<_Tp>; 2753 #endif 2754 2755 // }}} 2756 } // namespace simd_abi 2757 2758 // traits {{{1 2759 template <typename _Tp> 2760 struct is_simd_flag_type 2761 : false_type 2762 {}; 2763 2764 template <> 2765 struct is_simd_flag_type<element_aligned_tag> 2766 : true_type 2767 {}; 2768 2769 template <> 2770 struct is_simd_flag_type<vector_aligned_tag> 2771 : true_type 2772 {}; 2773 2774 template <size_t _Np> 2775 struct is_simd_flag_type<overaligned_tag<_Np>> 2776 : __bool_constant<(_Np > 0) and __has_single_bit(_Np)> 2777 {}; 2778 2779 template <typename _Tp> 2780 inline constexpr bool is_simd_flag_type_v = is_simd_flag_type<_Tp>::value; 2781 2782 template <typename _Tp, typename = enable_if_t<is_simd_flag_type_v<_Tp>>> 2783 using _IsSimdFlagType = _Tp; 2784 2785 // is_abi_tag {{{2 2786 template <typename _Tp, typename = void_t<>> 2787 struct is_abi_tag : false_type {}; 2788 2789 template <typename _Tp> 2790 struct is_abi_tag<_Tp, void_t<typename _Tp::_IsValidAbiTag>> 2791 : public _Tp::_IsValidAbiTag {}; 2792 2793 template <typename _Tp> 2794 inline constexpr bool is_abi_tag_v = is_abi_tag<_Tp>::value; 2795 2796 // is_simd(_mask) {{{2 2797 template <typename _Tp> 2798 struct is_simd : public false_type {}; 2799 2800 template <typename _Tp> 2801 inline constexpr bool is_simd_v = is_simd<_Tp>::value; 2802 2803 template <typename _Tp> 2804 struct is_simd_mask : public false_type {}; 2805 2806 template <typename _Tp> 2807 inline constexpr bool is_simd_mask_v = is_simd_mask<_Tp>::value; 2808 2809 // simd_size {{{2 2810 template <typename _Tp, typename _Abi, typename = void> 2811 struct __simd_size_impl {}; 2812 2813 template <typename _Tp, typename _Abi> 2814 struct __simd_size_impl< 2815 _Tp, _Abi, 2816 enable_if_t<conjunction_v<__is_vectorizable<_Tp>, is_abi_tag<_Abi>>>> 2817 : _SizeConstant<_Abi::template _S_size<_Tp>> {}; 2818 2819 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>> 2820 struct simd_size : __simd_size_impl<_Tp, _Abi> {}; 2821 2822 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>> 2823 inline constexpr size_t simd_size_v = simd_size<_Tp, _Abi>::value; 2824 2825 // simd_abi::deduce {{{2 2826 template <typename _Tp, size_t _Np, typename = void> 2827 struct __deduce_impl; 2828 2829 namespace simd_abi { 2830 /** 2831 * @tparam _Tp The requested `value_type` for the elements. 2832 * @tparam _Np The requested number of elements. 2833 * @tparam _Abis This parameter is ignored, since this implementation cannot 2834 * make any use of it. Either __a good native ABI is matched and used as `type` 2835 * alias, or the `fixed_size<_Np>` ABI is used, which internally is built from 2836 * the best matching native ABIs. 2837 */ 2838 template <typename _Tp, size_t _Np, typename...> 2839 struct deduce : __deduce_impl<_Tp, _Np> {}; 2840 2841 template <typename _Tp, size_t _Np, typename... _Abis> 2842 using deduce_t = typename deduce<_Tp, _Np, _Abis...>::type; 2843 } // namespace simd_abi 2844 2845 // }}}2 2846 // rebind_simd {{{2 2847 template <typename _Tp, typename _V, typename = void> 2848 struct rebind_simd; 2849 2850 template <typename _Tp, typename _Up, typename _Abi> 2851 struct rebind_simd<_Tp, simd<_Up, _Abi>, 2852 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>> 2853 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; }; 2854 2855 template <typename _Tp, typename _Up, typename _Abi> 2856 struct rebind_simd<_Tp, simd_mask<_Up, _Abi>, 2857 void_t<simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>> 2858 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, simd_size_v<_Up, _Abi>, _Abi>>; }; 2859 2860 template <typename _Tp, typename _V> 2861 using rebind_simd_t = typename rebind_simd<_Tp, _V>::type; 2862 2863 // resize_simd {{{2 2864 template <int _Np, typename _V, typename = void> 2865 struct resize_simd; 2866 2867 template <int _Np, typename _Tp, typename _Abi> 2868 struct resize_simd<_Np, simd<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>> 2869 { using type = simd<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; }; 2870 2871 template <int _Np, typename _Tp, typename _Abi> 2872 struct resize_simd<_Np, simd_mask<_Tp, _Abi>, void_t<simd_abi::deduce_t<_Tp, _Np, _Abi>>> 2873 { using type = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np, _Abi>>; }; 2874 2875 template <int _Np, typename _V> 2876 using resize_simd_t = typename resize_simd<_Np, _V>::type; 2877 2878 // }}}2 2879 // memory_alignment {{{2 2880 template <typename _Tp, typename _Up = typename _Tp::value_type> 2881 struct memory_alignment 2882 : public _SizeConstant<vector_aligned_tag::_S_alignment<_Tp, _Up>> {}; 2883 2884 template <typename _Tp, typename _Up = typename _Tp::value_type> 2885 inline constexpr size_t memory_alignment_v = memory_alignment<_Tp, _Up>::value; 2886 2887 // class template simd [simd] {{{1 2888 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>> 2889 class simd; 2890 2891 template <typename _Tp, typename _Abi> 2892 struct is_simd<simd<_Tp, _Abi>> : public true_type {}; 2893 2894 template <typename _Tp> 2895 using native_simd = simd<_Tp, simd_abi::native<_Tp>>; 2896 2897 template <typename _Tp, int _Np> 2898 using fixed_size_simd = simd<_Tp, simd_abi::fixed_size<_Np>>; 2899 2900 template <typename _Tp, size_t _Np> 2901 using __deduced_simd = simd<_Tp, simd_abi::deduce_t<_Tp, _Np>>; 2902 2903 // class template simd_mask [simd_mask] {{{1 2904 template <typename _Tp, typename _Abi = simd_abi::__default_abi<_Tp>> 2905 class simd_mask; 2906 2907 template <typename _Tp, typename _Abi> 2908 struct is_simd_mask<simd_mask<_Tp, _Abi>> : public true_type {}; 2909 2910 template <typename _Tp> 2911 using native_simd_mask = simd_mask<_Tp, simd_abi::native<_Tp>>; 2912 2913 template <typename _Tp, int _Np> 2914 using fixed_size_simd_mask = simd_mask<_Tp, simd_abi::fixed_size<_Np>>; 2915 2916 template <typename _Tp, size_t _Np> 2917 using __deduced_simd_mask = simd_mask<_Tp, simd_abi::deduce_t<_Tp, _Np>>; 2918 2919 // casts [simd.casts] {{{1 2920 // static_simd_cast {{{2 2921 template <typename _Tp, typename _Up, typename _Ap, bool = is_simd_v<_Tp>, typename = void> 2922 struct __static_simd_cast_return_type; 2923 2924 template <typename _Tp, typename _A0, typename _Up, typename _Ap> 2925 struct __static_simd_cast_return_type<simd_mask<_Tp, _A0>, _Up, _Ap, false, void> 2926 : __static_simd_cast_return_type<simd<_Tp, _A0>, _Up, _Ap> {}; 2927 2928 template <typename _Tp, typename _Up, typename _Ap> 2929 struct __static_simd_cast_return_type< 2930 _Tp, _Up, _Ap, true, enable_if_t<_Tp::size() == simd_size_v<_Up, _Ap>>> 2931 { using type = _Tp; }; 2932 2933 template <typename _Tp, typename _Ap> 2934 struct __static_simd_cast_return_type<_Tp, _Tp, _Ap, false, 2935 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66 2936 enable_if_t<__is_vectorizable_v<_Tp>> 2937 #else 2938 void 2939 #endif 2940 > 2941 { using type = simd<_Tp, _Ap>; }; 2942 2943 template <typename _Tp, typename = void> 2944 struct __safe_make_signed { using type = _Tp;}; 2945 2946 template <typename _Tp> 2947 struct __safe_make_signed<_Tp, enable_if_t<is_integral_v<_Tp>>> 2948 { 2949 // the extra make_unsigned_t is because of PR85951 2950 using type = make_signed_t<make_unsigned_t<_Tp>>; 2951 }; 2952 2953 template <typename _Tp> 2954 using safe_make_signed_t = typename __safe_make_signed<_Tp>::type; 2955 2956 template <typename _Tp, typename _Up, typename _Ap> 2957 struct __static_simd_cast_return_type<_Tp, _Up, _Ap, false, 2958 #ifdef _GLIBCXX_SIMD_FIX_P2TS_ISSUE66 2959 enable_if_t<__is_vectorizable_v<_Tp>> 2960 #else 2961 void 2962 #endif 2963 > 2964 { 2965 using type = conditional_t< 2966 (is_integral_v<_Up> && is_integral_v<_Tp> && 2967 #ifndef _GLIBCXX_SIMD_FIX_P2TS_ISSUE65 2968 is_signed_v<_Up> != is_signed_v<_Tp> && 2969 #endif 2970 is_same_v<safe_make_signed_t<_Up>, safe_make_signed_t<_Tp>>), 2971 simd<_Tp, _Ap>, fixed_size_simd<_Tp, simd_size_v<_Up, _Ap>>>; 2972 }; 2973 2974 template <typename _Tp, typename _Up, typename _Ap, 2975 typename _R 2976 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type> 2977 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _R 2978 static_simd_cast(const simd<_Up, _Ap>& __x) 2979 { 2980 if constexpr (is_same<_R, simd<_Up, _Ap>>::value) 2981 return __x; 2982 else 2983 { 2984 _SimdConverter<_Up, _Ap, typename _R::value_type, typename _R::abi_type> 2985 __c; 2986 return _R(__private_init, __c(__data(__x))); 2987 } 2988 } 2989 2990 namespace __proposed { 2991 template <typename _Tp, typename _Up, typename _Ap, 2992 typename _R 2993 = typename __static_simd_cast_return_type<_Tp, _Up, _Ap>::type> 2994 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR typename _R::mask_type 2995 static_simd_cast(const simd_mask<_Up, _Ap>& __x) 2996 { 2997 using _RM = typename _R::mask_type; 2998 return {__private_init, _RM::abi_type::_MaskImpl::template _S_convert< 2999 typename _RM::simd_type::value_type>(__x)}; 3000 } 3001 } // namespace __proposed 3002 3003 // simd_cast {{{2 3004 template <typename _Tp, typename _Up, typename _Ap, 3005 typename _To = __value_type_or_identity_t<_Tp>> 3006 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto 3007 simd_cast(const simd<_ValuePreserving<_Up, _To>, _Ap>& __x) 3008 -> decltype(static_simd_cast<_Tp>(__x)) 3009 { return static_simd_cast<_Tp>(__x); } 3010 3011 namespace __proposed { 3012 template <typename _Tp, typename _Up, typename _Ap, 3013 typename _To = __value_type_or_identity_t<_Tp>> 3014 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR auto 3015 simd_cast(const simd_mask<_ValuePreserving<_Up, _To>, _Ap>& __x) 3016 -> decltype(static_simd_cast<_Tp>(__x)) 3017 { return static_simd_cast<_Tp>(__x); } 3018 } // namespace __proposed 3019 3020 // }}}2 3021 // resizing_simd_cast {{{ 3022 namespace __proposed { 3023 /* Proposed spec: 3024 3025 template <class T, class U, class Abi> 3026 T resizing_simd_cast(const simd<U, Abi>& x) 3027 3028 p1 Constraints: 3029 - is_simd_v<T> is true and 3030 - T::value_type is the same type as U 3031 3032 p2 Returns: 3033 A simd object with the i^th element initialized to x[i] for all i in the 3034 range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger 3035 than simd_size_v<U, Abi>, the remaining elements are value-initialized. 3036 3037 template <class T, class U, class Abi> 3038 T resizing_simd_cast(const simd_mask<U, Abi>& x) 3039 3040 p1 Constraints: is_simd_mask_v<T> is true 3041 3042 p2 Returns: 3043 A simd_mask object with the i^th element initialized to x[i] for all i in 3044 the range of [0, min(T::size(), simd_size_v<U, Abi>)). If T::size() is larger 3045 than simd_size_v<U, Abi>, the remaining elements are initialized to false. 3046 3047 */ 3048 3049 template <typename _Tp, typename _Up, typename _Ap> 3050 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR enable_if_t< 3051 conjunction_v<is_simd<_Tp>, is_same<typename _Tp::value_type, _Up>>, _Tp> 3052 resizing_simd_cast(const simd<_Up, _Ap>& __x) 3053 { 3054 if constexpr (is_same_v<typename _Tp::abi_type, _Ap>) 3055 return __x; 3056 else if (__builtin_is_constant_evaluated()) 3057 return _Tp([&](auto __i) constexpr { 3058 return __i < simd_size_v<_Up, _Ap> ? __x[__i] : _Up(); 3059 }); 3060 else if constexpr (simd_size_v<_Up, _Ap> == 1) 3061 { 3062 _Tp __r{}; 3063 __r[0] = __x[0]; 3064 return __r; 3065 } 3066 else if constexpr (_Tp::size() == 1) 3067 return __x[0]; 3068 else if constexpr (sizeof(_Tp) == sizeof(__x) 3069 && !__is_fixed_size_abi_v<_Ap>) 3070 return {__private_init, 3071 __vector_bitcast<typename _Tp::value_type, _Tp::size()>( 3072 _Ap::_S_masked(__data(__x))._M_data)}; 3073 else 3074 { 3075 _Tp __r{}; 3076 __builtin_memcpy(&__data(__r), &__data(__x), 3077 sizeof(_Up) 3078 * std::min(_Tp::size(), simd_size_v<_Up, _Ap>)); 3079 return __r; 3080 } 3081 } 3082 3083 template <typename _Tp, typename _Up, typename _Ap> 3084 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3085 enable_if_t<is_simd_mask_v<_Tp>, _Tp> 3086 resizing_simd_cast(const simd_mask<_Up, _Ap>& __x) 3087 { 3088 return {__private_init, _Tp::abi_type::_MaskImpl::template _S_convert< 3089 typename _Tp::simd_type::value_type>(__x)}; 3090 } 3091 } // namespace __proposed 3092 3093 // }}} 3094 // to_fixed_size {{{2 3095 template <typename _Tp, int _Np> 3096 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, _Np> 3097 to_fixed_size(const fixed_size_simd<_Tp, _Np>& __x) 3098 { return __x; } 3099 3100 template <typename _Tp, int _Np> 3101 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, _Np> 3102 to_fixed_size(const fixed_size_simd_mask<_Tp, _Np>& __x) 3103 { return __x; } 3104 3105 template <typename _Tp, typename _Ap> 3106 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>> 3107 to_fixed_size(const simd<_Tp, _Ap>& __x) 3108 { 3109 using _Rp = fixed_size_simd<_Tp, simd_size_v<_Tp, _Ap>>; 3110 return _Rp([&__x](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }); 3111 } 3112 3113 template <typename _Tp, typename _Ap> 3114 _GLIBCXX_SIMD_INTRINSIC fixed_size_simd_mask<_Tp, simd_size_v<_Tp, _Ap>> 3115 to_fixed_size(const simd_mask<_Tp, _Ap>& __x) 3116 { 3117 return {__private_init, 3118 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }}; 3119 } 3120 3121 // to_native {{{2 3122 template <typename _Tp, int _Np> 3123 _GLIBCXX_SIMD_INTRINSIC 3124 enable_if_t<(_Np == native_simd<_Tp>::size()), native_simd<_Tp>> 3125 to_native(const fixed_size_simd<_Tp, _Np>& __x) 3126 { 3127 alignas(memory_alignment_v<native_simd<_Tp>>) _Tp __mem[_Np]; 3128 __x.copy_to(__mem, vector_aligned); 3129 return {__mem, vector_aligned}; 3130 } 3131 3132 template <typename _Tp, int _Np> 3133 _GLIBCXX_SIMD_INTRINSIC 3134 enable_if_t<(_Np == native_simd_mask<_Tp>::size()), native_simd_mask<_Tp>> 3135 to_native(const fixed_size_simd_mask<_Tp, _Np>& __x) 3136 { 3137 return native_simd_mask<_Tp>( 3138 __private_init, 3139 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }); 3140 } 3141 3142 // to_compatible {{{2 3143 template <typename _Tp, int _Np> 3144 _GLIBCXX_SIMD_INTRINSIC enable_if_t<(_Np == simd<_Tp>::size()), simd<_Tp>> 3145 to_compatible(const simd<_Tp, simd_abi::fixed_size<_Np>>& __x) 3146 { 3147 alignas(memory_alignment_v<simd<_Tp>>) _Tp __mem[_Np]; 3148 __x.copy_to(__mem, vector_aligned); 3149 return {__mem, vector_aligned}; 3150 } 3151 3152 template <typename _Tp, int _Np> 3153 _GLIBCXX_SIMD_INTRINSIC 3154 enable_if_t<(_Np == simd_mask<_Tp>::size()), simd_mask<_Tp>> 3155 to_compatible(const simd_mask<_Tp, simd_abi::fixed_size<_Np>>& __x) 3156 { 3157 return simd_mask<_Tp>( 3158 __private_init, 3159 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }); 3160 } 3161 3162 // masked assignment [simd_mask.where] {{{1 3163 3164 // where_expression {{{1 3165 // const_where_expression<M, T> {{{2 3166 template <typename _M, typename _Tp> 3167 class const_where_expression 3168 { 3169 using _V = _Tp; 3170 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>); 3171 3172 struct _Wrapper { using value_type = _V; }; 3173 3174 protected: 3175 using _Impl = typename _V::_Impl; 3176 3177 using value_type = 3178 typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type; 3179 3180 _GLIBCXX_SIMD_INTRINSIC friend const _M& 3181 __get_mask(const const_where_expression& __x) 3182 { return __x._M_k; } 3183 3184 _GLIBCXX_SIMD_INTRINSIC friend const _Tp& 3185 __get_lvalue(const const_where_expression& __x) 3186 { return __x._M_value; } 3187 3188 const _M& _M_k; 3189 _Tp& _M_value; 3190 3191 public: 3192 const_where_expression(const const_where_expression&) = delete; 3193 3194 const_where_expression& operator=(const const_where_expression&) = delete; 3195 3196 _GLIBCXX_SIMD_INTRINSIC constexpr 3197 const_where_expression(const _M& __kk, const _Tp& dd) 3198 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {} 3199 3200 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V 3201 operator-() const&& 3202 { 3203 return {__private_init, 3204 _Impl::template _S_masked_unary<negate>(__data(_M_k), 3205 __data(_M_value))}; 3206 } 3207 3208 template <typename _Up, typename _Flags> 3209 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V 3210 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&& 3211 { 3212 return {__private_init, 3213 _Impl::_S_masked_load(__data(_M_value), __data(_M_k), 3214 _Flags::template _S_apply<_V>(__mem))}; 3215 } 3216 3217 template <typename _Up, typename _Flags> 3218 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3219 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&& 3220 { 3221 _Impl::_S_masked_store(__data(_M_value), 3222 _Flags::template _S_apply<_V>(__mem), 3223 __data(_M_k)); 3224 } 3225 }; 3226 3227 // const_where_expression<bool, T> {{{2 3228 template <typename _Tp> 3229 class const_where_expression<bool, _Tp> 3230 { 3231 using _M = bool; 3232 using _V = _Tp; 3233 3234 static_assert(is_same_v<_V, __remove_cvref_t<_Tp>>); 3235 3236 struct _Wrapper { using value_type = _V; }; 3237 3238 protected: 3239 using value_type 3240 = typename conditional_t<is_arithmetic_v<_V>, _Wrapper, _V>::value_type; 3241 3242 _GLIBCXX_SIMD_INTRINSIC friend const _M& 3243 __get_mask(const const_where_expression& __x) 3244 { return __x._M_k; } 3245 3246 _GLIBCXX_SIMD_INTRINSIC friend const _Tp& 3247 __get_lvalue(const const_where_expression& __x) 3248 { return __x._M_value; } 3249 3250 const bool _M_k; 3251 _Tp& _M_value; 3252 3253 public: 3254 const_where_expression(const const_where_expression&) = delete; 3255 const_where_expression& operator=(const const_where_expression&) = delete; 3256 3257 _GLIBCXX_SIMD_INTRINSIC constexpr 3258 const_where_expression(const bool __kk, const _Tp& dd) 3259 : _M_k(__kk), _M_value(const_cast<_Tp&>(dd)) {} 3260 3261 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V 3262 operator-() const&& 3263 { return _M_k ? -_M_value : _M_value; } 3264 3265 template <typename _Up, typename _Flags> 3266 [[nodiscard]] _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _V 3267 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&& 3268 { return _M_k ? static_cast<_V>(__mem[0]) : _M_value; } 3269 3270 template <typename _Up, typename _Flags> 3271 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3272 copy_to(_LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) const&& 3273 { 3274 if (_M_k) 3275 __mem[0] = _M_value; 3276 } 3277 }; 3278 3279 // where_expression<M, T> {{{2 3280 template <typename _M, typename _Tp> 3281 class where_expression : public const_where_expression<_M, _Tp> 3282 { 3283 using _Impl = typename const_where_expression<_M, _Tp>::_Impl; 3284 3285 static_assert(!is_const<_Tp>::value, 3286 "where_expression may only be instantiated with __a non-const " 3287 "_Tp parameter"); 3288 3289 using typename const_where_expression<_M, _Tp>::value_type; 3290 using const_where_expression<_M, _Tp>::_M_k; 3291 using const_where_expression<_M, _Tp>::_M_value; 3292 3293 static_assert( 3294 is_same<typename _M::abi_type, typename _Tp::abi_type>::value, ""); 3295 static_assert(_M::size() == _Tp::size(), ""); 3296 3297 _GLIBCXX_SIMD_INTRINSIC friend constexpr _Tp& 3298 __get_lvalue(where_expression& __x) 3299 { return __x._M_value; } 3300 3301 public: 3302 where_expression(const where_expression&) = delete; 3303 where_expression& operator=(const where_expression&) = delete; 3304 3305 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3306 where_expression(const _M& __kk, _Tp& dd) 3307 : const_where_expression<_M, _Tp>(__kk, dd) {} 3308 3309 template <typename _Up> 3310 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3311 operator=(_Up&& __x) && 3312 { 3313 _Impl::_S_masked_assign(__data(_M_k), __data(_M_value), 3314 __to_value_type_or_member_type<_Tp>( 3315 static_cast<_Up&&>(__x))); 3316 } 3317 3318 #define _GLIBCXX_SIMD_OP_(__op, __name) \ 3319 template <typename _Up> \ 3320 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \ 3321 operator __op##=(_Up&& __x)&& \ 3322 { \ 3323 _Impl::template _S_masked_cassign( \ 3324 __data(_M_k), __data(_M_value), \ 3325 __to_value_type_or_member_type<_Tp>(static_cast<_Up&&>(__x)), \ 3326 [](auto __impl, auto __lhs, auto __rhs) \ 3327 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \ 3328 { return __impl.__name(__lhs, __rhs); }); \ 3329 } \ 3330 static_assert(true) 3331 _GLIBCXX_SIMD_OP_(+, _S_plus); 3332 _GLIBCXX_SIMD_OP_(-, _S_minus); 3333 _GLIBCXX_SIMD_OP_(*, _S_multiplies); 3334 _GLIBCXX_SIMD_OP_(/, _S_divides); 3335 _GLIBCXX_SIMD_OP_(%, _S_modulus); 3336 _GLIBCXX_SIMD_OP_(&, _S_bit_and); 3337 _GLIBCXX_SIMD_OP_(|, _S_bit_or); 3338 _GLIBCXX_SIMD_OP_(^, _S_bit_xor); 3339 _GLIBCXX_SIMD_OP_(<<, _S_shift_left); 3340 _GLIBCXX_SIMD_OP_(>>, _S_shift_right); 3341 #undef _GLIBCXX_SIMD_OP_ 3342 3343 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3344 operator++() && 3345 { 3346 __data(_M_value) 3347 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value)); 3348 } 3349 3350 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3351 operator++(int) && 3352 { 3353 __data(_M_value) 3354 = _Impl::template _S_masked_unary<__increment>(__data(_M_k), __data(_M_value)); 3355 } 3356 3357 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3358 operator--() && 3359 { 3360 __data(_M_value) 3361 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value)); 3362 } 3363 3364 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3365 operator--(int) && 3366 { 3367 __data(_M_value) 3368 = _Impl::template _S_masked_unary<__decrement>(__data(_M_k), __data(_M_value)); 3369 } 3370 3371 // intentionally hides const_where_expression::copy_from 3372 template <typename _Up, typename _Flags> 3373 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3374 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) && 3375 { 3376 __data(_M_value) = _Impl::_S_masked_load(__data(_M_value), __data(_M_k), 3377 _Flags::template _S_apply<_Tp>(__mem)); 3378 } 3379 }; 3380 3381 // where_expression<bool, T> {{{2 3382 template <typename _Tp> 3383 class where_expression<bool, _Tp> 3384 : public const_where_expression<bool, _Tp> 3385 { 3386 using _M = bool; 3387 using typename const_where_expression<_M, _Tp>::value_type; 3388 using const_where_expression<_M, _Tp>::_M_k; 3389 using const_where_expression<_M, _Tp>::_M_value; 3390 3391 public: 3392 where_expression(const where_expression&) = delete; 3393 where_expression& operator=(const where_expression&) = delete; 3394 3395 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3396 where_expression(const _M& __kk, _Tp& dd) 3397 : const_where_expression<_M, _Tp>(__kk, dd) {} 3398 3399 #define _GLIBCXX_SIMD_OP_(__op) \ 3400 template <typename _Up> \ 3401 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void \ 3402 operator __op(_Up&& __x)&& \ 3403 { if (_M_k) _M_value __op static_cast<_Up&&>(__x); } 3404 3405 _GLIBCXX_SIMD_OP_(=) 3406 _GLIBCXX_SIMD_OP_(+=) 3407 _GLIBCXX_SIMD_OP_(-=) 3408 _GLIBCXX_SIMD_OP_(*=) 3409 _GLIBCXX_SIMD_OP_(/=) 3410 _GLIBCXX_SIMD_OP_(%=) 3411 _GLIBCXX_SIMD_OP_(&=) 3412 _GLIBCXX_SIMD_OP_(|=) 3413 _GLIBCXX_SIMD_OP_(^=) 3414 _GLIBCXX_SIMD_OP_(<<=) 3415 _GLIBCXX_SIMD_OP_(>>=) 3416 #undef _GLIBCXX_SIMD_OP_ 3417 3418 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3419 operator++() && 3420 { if (_M_k) ++_M_value; } 3421 3422 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3423 operator++(int) && 3424 { if (_M_k) ++_M_value; } 3425 3426 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3427 operator--() && 3428 { if (_M_k) --_M_value; } 3429 3430 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3431 operator--(int) && 3432 { if (_M_k) --_M_value; } 3433 3434 // intentionally hides const_where_expression::copy_from 3435 template <typename _Up, typename _Flags> 3436 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR void 3437 copy_from(const _LoadStorePtr<_Up, value_type>* __mem, _IsSimdFlagType<_Flags>) && 3438 { if (_M_k) _M_value = __mem[0]; } 3439 }; 3440 3441 // where {{{1 3442 template <typename _Tp, typename _Ap> 3443 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3444 where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>> 3445 where(const typename simd<_Tp, _Ap>::mask_type& __k, simd<_Tp, _Ap>& __value) 3446 { return {__k, __value}; } 3447 3448 template <typename _Tp, typename _Ap> 3449 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3450 const_where_expression<simd_mask<_Tp, _Ap>, simd<_Tp, _Ap>> 3451 where(const typename simd<_Tp, _Ap>::mask_type& __k, const simd<_Tp, _Ap>& __value) 3452 { return {__k, __value}; } 3453 3454 template <typename _Tp, typename _Ap> 3455 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3456 where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>> 3457 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, simd_mask<_Tp, _Ap>& __value) 3458 { return {__k, __value}; } 3459 3460 template <typename _Tp, typename _Ap> 3461 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3462 const_where_expression<simd_mask<_Tp, _Ap>, simd_mask<_Tp, _Ap>> 3463 where(const remove_const_t<simd_mask<_Tp, _Ap>>& __k, const simd_mask<_Tp, _Ap>& __value) 3464 { return {__k, __value}; } 3465 3466 template <typename _Tp> 3467 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR where_expression<bool, _Tp> 3468 where(_ExactBool __k, _Tp& __value) 3469 { return {__k, __value}; } 3470 3471 template <typename _Tp> 3472 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR const_where_expression<bool, _Tp> 3473 where(_ExactBool __k, const _Tp& __value) 3474 { return {__k, __value}; } 3475 3476 template <typename _Tp, typename _Ap> 3477 _GLIBCXX_SIMD_CONSTEXPR void 3478 where(bool __k, simd<_Tp, _Ap>& __value) = delete; 3479 3480 template <typename _Tp, typename _Ap> 3481 _GLIBCXX_SIMD_CONSTEXPR void 3482 where(bool __k, const simd<_Tp, _Ap>& __value) = delete; 3483 3484 // proposed mask iterations {{{1 3485 namespace __proposed { 3486 template <size_t _Np> 3487 class where_range 3488 { 3489 const bitset<_Np> __bits; 3490 3491 public: 3492 where_range(bitset<_Np> __b) : __bits(__b) {} 3493 3494 class iterator 3495 { 3496 size_t __mask; 3497 size_t __bit; 3498 3499 _GLIBCXX_SIMD_INTRINSIC void 3500 __next_bit() 3501 { __bit = __builtin_ctzl(__mask); } 3502 3503 _GLIBCXX_SIMD_INTRINSIC void 3504 __reset_lsb() 3505 { 3506 // 01100100 - 1 = 01100011 3507 __mask &= (__mask - 1); 3508 // __asm__("btr %1,%0" : "+r"(__mask) : "r"(__bit)); 3509 } 3510 3511 public: 3512 iterator(decltype(__mask) __m) : __mask(__m) { __next_bit(); } 3513 iterator(const iterator&) = default; 3514 iterator(iterator&&) = default; 3515 3516 _GLIBCXX_SIMD_ALWAYS_INLINE size_t 3517 operator->() const 3518 { return __bit; } 3519 3520 _GLIBCXX_SIMD_ALWAYS_INLINE size_t 3521 operator*() const 3522 { return __bit; } 3523 3524 _GLIBCXX_SIMD_ALWAYS_INLINE iterator& 3525 operator++() 3526 { 3527 __reset_lsb(); 3528 __next_bit(); 3529 return *this; 3530 } 3531 3532 _GLIBCXX_SIMD_ALWAYS_INLINE iterator 3533 operator++(int) 3534 { 3535 iterator __tmp = *this; 3536 __reset_lsb(); 3537 __next_bit(); 3538 return __tmp; 3539 } 3540 3541 _GLIBCXX_SIMD_ALWAYS_INLINE bool 3542 operator==(const iterator& __rhs) const 3543 { return __mask == __rhs.__mask; } 3544 3545 _GLIBCXX_SIMD_ALWAYS_INLINE bool 3546 operator!=(const iterator& __rhs) const 3547 { return __mask != __rhs.__mask; } 3548 }; 3549 3550 iterator 3551 begin() const 3552 { return __bits.to_ullong(); } 3553 3554 iterator 3555 end() const 3556 { return 0; } 3557 }; 3558 3559 template <typename _Tp, typename _Ap> 3560 where_range<simd_size_v<_Tp, _Ap>> 3561 where(const simd_mask<_Tp, _Ap>& __k) 3562 { return __k.__to_bitset(); } 3563 3564 } // namespace __proposed 3565 3566 // }}}1 3567 // reductions [simd.reductions] {{{1 3568 template <typename _Tp, typename _Abi, typename _BinaryOperation = plus<>> 3569 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp 3570 reduce(const simd<_Tp, _Abi>& __v, _BinaryOperation __binary_op = _BinaryOperation()) 3571 { return _Abi::_SimdImpl::_S_reduce(__v, __binary_op); } 3572 3573 template <typename _M, typename _V, typename _BinaryOperation = plus<>> 3574 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type 3575 reduce(const const_where_expression<_M, _V>& __x, 3576 typename _V::value_type __identity_element, _BinaryOperation __binary_op) 3577 { 3578 if (__builtin_expect(none_of(__get_mask(__x)), false)) 3579 return __identity_element; 3580 3581 _V __tmp = __identity_element; 3582 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp), 3583 __data(__get_lvalue(__x))); 3584 return reduce(__tmp, __binary_op); 3585 } 3586 3587 template <typename _M, typename _V> 3588 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type 3589 reduce(const const_where_expression<_M, _V>& __x, plus<> __binary_op = {}) 3590 { return reduce(__x, 0, __binary_op); } 3591 3592 template <typename _M, typename _V> 3593 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type 3594 reduce(const const_where_expression<_M, _V>& __x, multiplies<> __binary_op) 3595 { return reduce(__x, 1, __binary_op); } 3596 3597 template <typename _M, typename _V> 3598 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type 3599 reduce(const const_where_expression<_M, _V>& __x, bit_and<> __binary_op) 3600 { return reduce(__x, ~typename _V::value_type(), __binary_op); } 3601 3602 template <typename _M, typename _V> 3603 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type 3604 reduce(const const_where_expression<_M, _V>& __x, bit_or<> __binary_op) 3605 { return reduce(__x, 0, __binary_op); } 3606 3607 template <typename _M, typename _V> 3608 _GLIBCXX_SIMD_INTRINSIC typename _V::value_type 3609 reduce(const const_where_expression<_M, _V>& __x, bit_xor<> __binary_op) 3610 { return reduce(__x, 0, __binary_op); } 3611 3612 template <typename _Tp, typename _Abi> 3613 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp 3614 hmin(const simd<_Tp, _Abi>& __v) noexcept 3615 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Minimum()); } 3616 3617 template <typename _Tp, typename _Abi> 3618 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR _Tp 3619 hmax(const simd<_Tp, _Abi>& __v) noexcept 3620 { return _Abi::_SimdImpl::_S_reduce(__v, __detail::_Maximum()); } 3621 3622 template <typename _M, typename _V> 3623 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3624 typename _V::value_type 3625 hmin(const const_where_expression<_M, _V>& __x) noexcept 3626 { 3627 using _Tp = typename _V::value_type; 3628 constexpr _Tp __id_elem = 3629 #ifdef __FINITE_MATH_ONLY__ 3630 __finite_max_v<_Tp>; 3631 #else 3632 __value_or<__infinity, _Tp>(__finite_max_v<_Tp>); 3633 #endif 3634 _V __tmp = __id_elem; 3635 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp), 3636 __data(__get_lvalue(__x))); 3637 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Minimum()); 3638 } 3639 3640 template <typename _M, typename _V> 3641 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3642 typename _V::value_type 3643 hmax(const const_where_expression<_M, _V>& __x) noexcept 3644 { 3645 using _Tp = typename _V::value_type; 3646 constexpr _Tp __id_elem = 3647 #ifdef __FINITE_MATH_ONLY__ 3648 __finite_min_v<_Tp>; 3649 #else 3650 [] { 3651 if constexpr (__value_exists_v<__infinity, _Tp>) 3652 return -__infinity_v<_Tp>; 3653 else 3654 return __finite_min_v<_Tp>; 3655 }(); 3656 #endif 3657 _V __tmp = __id_elem; 3658 _V::_Impl::_S_masked_assign(__data(__get_mask(__x)), __data(__tmp), 3659 __data(__get_lvalue(__x))); 3660 return _V::abi_type::_SimdImpl::_S_reduce(__tmp, __detail::_Maximum()); 3661 } 3662 3663 // }}}1 3664 // algorithms [simd.alg] {{{ 3665 template <typename _Tp, typename _Ap> 3666 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap> 3667 min(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b) 3668 { return {__private_init, _Ap::_SimdImpl::_S_min(__data(__a), __data(__b))}; } 3669 3670 template <typename _Tp, typename _Ap> 3671 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap> 3672 max(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b) 3673 { return {__private_init, _Ap::_SimdImpl::_S_max(__data(__a), __data(__b))}; } 3674 3675 template <typename _Tp, typename _Ap> 3676 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 3677 pair<simd<_Tp, _Ap>, simd<_Tp, _Ap>> 3678 minmax(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b) 3679 { 3680 const auto pair_of_members 3681 = _Ap::_SimdImpl::_S_minmax(__data(__a), __data(__b)); 3682 return {simd<_Tp, _Ap>(__private_init, pair_of_members.first), 3683 simd<_Tp, _Ap>(__private_init, pair_of_members.second)}; 3684 } 3685 3686 template <typename _Tp, typename _Ap> 3687 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap> 3688 clamp(const simd<_Tp, _Ap>& __v, const simd<_Tp, _Ap>& __lo, const simd<_Tp, _Ap>& __hi) 3689 { 3690 using _Impl = typename _Ap::_SimdImpl; 3691 return {__private_init, 3692 _Impl::_S_min(__data(__hi), 3693 _Impl::_S_max(__data(__lo), __data(__v)))}; 3694 } 3695 3696 // }}} 3697 3698 template <size_t... _Sizes, typename _Tp, typename _Ap, 3699 typename = enable_if_t<((_Sizes + ...) == simd<_Tp, _Ap>::size())>> 3700 inline tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...> 3701 split(const simd<_Tp, _Ap>&); 3702 3703 // __extract_part {{{ 3704 template <int _Index, int _Total, int _Combine = 1, typename _Tp, size_t _Np> 3705 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr 3706 _SimdWrapper<_Tp, _Np / _Total * _Combine> 3707 __extract_part(const _SimdWrapper<_Tp, _Np> __x); 3708 3709 template <int _Index, int _Parts, int _Combine = 1, typename _Tp, typename _A0, typename... _As> 3710 _GLIBCXX_SIMD_INTRINSIC constexpr auto 3711 __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x); 3712 3713 // }}} 3714 // _SizeList {{{ 3715 template <size_t _V0, size_t... _Values> 3716 struct _SizeList 3717 { 3718 template <size_t _I> 3719 static constexpr size_t 3720 _S_at(_SizeConstant<_I> = {}) 3721 { 3722 if constexpr (_I == 0) 3723 return _V0; 3724 else 3725 return _SizeList<_Values...>::template _S_at<_I - 1>(); 3726 } 3727 3728 template <size_t _I> 3729 static constexpr auto 3730 _S_before(_SizeConstant<_I> = {}) 3731 { 3732 if constexpr (_I == 0) 3733 return _SizeConstant<0>(); 3734 else 3735 return _SizeConstant< 3736 _V0 + _SizeList<_Values...>::template _S_before<_I - 1>()>(); 3737 } 3738 3739 template <size_t _Np> 3740 static constexpr auto 3741 _S_pop_front(_SizeConstant<_Np> = {}) 3742 { 3743 if constexpr (_Np == 0) 3744 return _SizeList(); 3745 else 3746 return _SizeList<_Values...>::template _S_pop_front<_Np - 1>(); 3747 } 3748 }; 3749 3750 // }}} 3751 // __extract_center {{{ 3752 template <typename _Tp, size_t _Np> 3753 _GLIBCXX_SIMD_INTRINSIC _SimdWrapper<_Tp, _Np / 2> 3754 __extract_center(_SimdWrapper<_Tp, _Np> __x) 3755 { 3756 static_assert(_Np >= 4); 3757 static_assert(_Np % 4 == 0); // x0 - x1 - x2 - x3 -> return {x1, x2} 3758 #if _GLIBCXX_SIMD_X86INTRIN // {{{ 3759 if constexpr (__have_avx512f && sizeof(_Tp) * _Np == 64) 3760 { 3761 const auto __intrin = __to_intrin(__x); 3762 if constexpr (is_integral_v<_Tp>) 3763 return __vector_bitcast<_Tp>(_mm512_castsi512_si256( 3764 _mm512_shuffle_i32x4(__intrin, __intrin, 3765 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40))); 3766 else if constexpr (sizeof(_Tp) == 4) 3767 return __vector_bitcast<_Tp>(_mm512_castps512_ps256( 3768 _mm512_shuffle_f32x4(__intrin, __intrin, 3769 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40))); 3770 else if constexpr (sizeof(_Tp) == 8) 3771 return __vector_bitcast<_Tp>(_mm512_castpd512_pd256( 3772 _mm512_shuffle_f64x2(__intrin, __intrin, 3773 1 + 2 * 0x4 + 2 * 0x10 + 3 * 0x40))); 3774 else 3775 __assert_unreachable<_Tp>(); 3776 } 3777 else if constexpr (sizeof(_Tp) * _Np == 32 && is_floating_point_v<_Tp>) 3778 return __vector_bitcast<_Tp>( 3779 _mm_shuffle_pd(__lo128(__vector_bitcast<double>(__x)), 3780 __hi128(__vector_bitcast<double>(__x)), 1)); 3781 else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) * _Np <= 32) 3782 return __vector_bitcast<_Tp>( 3783 _mm_alignr_epi8(__hi128(__vector_bitcast<_LLong>(__x)), 3784 __lo128(__vector_bitcast<_LLong>(__x)), 3785 sizeof(_Tp) * _Np / 4)); 3786 else 3787 #endif // _GLIBCXX_SIMD_X86INTRIN }}} 3788 { 3789 __vector_type_t<_Tp, _Np / 2> __r; 3790 __builtin_memcpy(&__r, 3791 reinterpret_cast<const char*>(&__x) 3792 + sizeof(_Tp) * _Np / 4, 3793 sizeof(_Tp) * _Np / 2); 3794 return __r; 3795 } 3796 } 3797 3798 template <typename _Tp, typename _A0, typename... _As> 3799 _GLIBCXX_SIMD_INTRINSIC 3800 _SimdWrapper<_Tp, _SimdTuple<_Tp, _A0, _As...>::_S_size() / 2> 3801 __extract_center(const _SimdTuple<_Tp, _A0, _As...>& __x) 3802 { 3803 if constexpr (sizeof...(_As) == 0) 3804 return __extract_center(__x.first); 3805 else 3806 return __extract_part<1, 4, 2>(__x); 3807 } 3808 3809 // }}} 3810 // __split_wrapper {{{ 3811 template <size_t... _Sizes, typename _Tp, typename... _As> 3812 auto 3813 __split_wrapper(_SizeList<_Sizes...>, const _SimdTuple<_Tp, _As...>& __x) 3814 { 3815 return split<_Sizes...>( 3816 fixed_size_simd<_Tp, _SimdTuple<_Tp, _As...>::_S_size()>(__private_init, 3817 __x)); 3818 } 3819 3820 // }}} 3821 3822 // split<simd>(simd) {{{ 3823 template <typename _V, typename _Ap, 3824 size_t _Parts = simd_size_v<typename _V::value_type, _Ap> / _V::size()> 3825 enable_if_t<simd_size_v<typename _V::value_type, _Ap> == _Parts * _V::size() 3826 && is_simd_v<_V>, array<_V, _Parts>> 3827 split(const simd<typename _V::value_type, _Ap>& __x) 3828 { 3829 using _Tp = typename _V::value_type; 3830 if constexpr (_Parts == 1) 3831 { 3832 return {simd_cast<_V>(__x)}; 3833 } 3834 else if (__x._M_is_constprop()) 3835 { 3836 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>( 3837 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3838 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 3839 { return __x[__i * _V::size() + __j]; }); 3840 }); 3841 } 3842 else if constexpr ( 3843 __is_fixed_size_abi_v<_Ap> 3844 && (is_same_v<typename _V::abi_type, simd_abi::scalar> 3845 || (__is_fixed_size_abi_v<typename _V::abi_type> 3846 && sizeof(_V) == sizeof(_Tp) * _V::size() // _V doesn't have padding 3847 ))) 3848 { 3849 // fixed_size -> fixed_size (w/o padding) or scalar 3850 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS 3851 const __may_alias<_Tp>* const __element_ptr 3852 = reinterpret_cast<const __may_alias<_Tp>*>(&__data(__x)); 3853 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>( 3854 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 3855 { return _V(__element_ptr + __i * _V::size(), vector_aligned); }); 3856 #else 3857 const auto& __xx = __data(__x); 3858 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>( 3859 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3860 [[maybe_unused]] constexpr size_t __offset 3861 = decltype(__i)::value * _V::size(); 3862 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3863 constexpr _SizeConstant<__j + __offset> __k; 3864 return __xx[__k]; 3865 }); 3866 }); 3867 #endif 3868 } 3869 else if constexpr (is_same_v<typename _V::abi_type, simd_abi::scalar>) 3870 { 3871 // normally memcpy should work here as well 3872 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>( 3873 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }); 3874 } 3875 else 3876 { 3877 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>( 3878 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3879 if constexpr (__is_fixed_size_abi_v<typename _V::abi_type>) 3880 return _V([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3881 return __x[__i * _V::size() + __j]; 3882 }); 3883 else 3884 return _V(__private_init, 3885 __extract_part<decltype(__i)::value, _Parts>(__data(__x))); 3886 }); 3887 } 3888 } 3889 3890 // }}} 3891 // split<simd_mask>(simd_mask) {{{ 3892 template <typename _V, typename _Ap, 3893 size_t _Parts = simd_size_v<typename _V::simd_type::value_type, _Ap> / _V::size()> 3894 enable_if_t<is_simd_mask_v<_V> && simd_size_v<typename 3895 _V::simd_type::value_type, _Ap> == _Parts * _V::size(), array<_V, _Parts>> 3896 split(const simd_mask<typename _V::simd_type::value_type, _Ap>& __x) 3897 { 3898 if constexpr (is_same_v<_Ap, typename _V::abi_type>) 3899 return {__x}; 3900 else if constexpr (_Parts == 1) 3901 return {__proposed::static_simd_cast<_V>(__x)}; 3902 else if constexpr (_Parts == 2 && __is_sse_abi<typename _V::abi_type>() 3903 && __is_avx_abi<_Ap>()) 3904 return {_V(__private_init, __lo128(__data(__x))), 3905 _V(__private_init, __hi128(__data(__x)))}; 3906 else if constexpr (_V::size() <= __CHAR_BIT__ * sizeof(_ULLong)) 3907 { 3908 const bitset __bits = __x.__to_bitset(); 3909 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>( 3910 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3911 constexpr size_t __offset = __i * _V::size(); 3912 return _V(__bitset_init, (__bits >> __offset).to_ullong()); 3913 }); 3914 } 3915 else 3916 { 3917 return __generate_from_n_evaluations<_Parts, array<_V, _Parts>>( 3918 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3919 constexpr size_t __offset = __i * _V::size(); 3920 return _V(__private_init, 3921 [&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3922 return __x[__j + __offset]; 3923 }); 3924 }); 3925 } 3926 } 3927 3928 // }}} 3929 // split<_Sizes...>(simd) {{{ 3930 template <size_t... _Sizes, typename _Tp, typename _Ap, typename> 3931 _GLIBCXX_SIMD_ALWAYS_INLINE 3932 tuple<simd<_Tp, simd_abi::deduce_t<_Tp, _Sizes>>...> 3933 split(const simd<_Tp, _Ap>& __x) 3934 { 3935 using _SL = _SizeList<_Sizes...>; 3936 using _Tuple = tuple<__deduced_simd<_Tp, _Sizes>...>; 3937 constexpr size_t _Np = simd_size_v<_Tp, _Ap>; 3938 constexpr size_t _N0 = _SL::template _S_at<0>(); 3939 using _V = __deduced_simd<_Tp, _N0>; 3940 3941 if (__x._M_is_constprop()) 3942 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>( 3943 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3944 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>; 3945 constexpr size_t __offset = _SL::_S_before(__i); 3946 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 3947 return __x[__offset + __j]; 3948 }); 3949 }); 3950 else if constexpr (_Np == _N0) 3951 { 3952 static_assert(sizeof...(_Sizes) == 1); 3953 return {simd_cast<_V>(__x)}; 3954 } 3955 else if constexpr // split from fixed_size, such that __x::first.size == _N0 3956 (__is_fixed_size_abi_v< 3957 _Ap> && __fixed_size_storage_t<_Tp, _Np>::_S_first_size == _N0) 3958 { 3959 static_assert( 3960 !__is_fixed_size_abi_v<typename _V::abi_type>, 3961 "How can <_Tp, _Np> be __a single _SimdTuple entry but __a " 3962 "fixed_size_simd " 3963 "when deduced?"); 3964 // extract first and recurse (__split_wrapper is needed to deduce a new 3965 // _Sizes pack) 3966 return tuple_cat(make_tuple(_V(__private_init, __data(__x).first)), 3967 __split_wrapper(_SL::template _S_pop_front<1>(), 3968 __data(__x).second)); 3969 } 3970 else if constexpr ((!is_same_v<simd_abi::scalar, 3971 simd_abi::deduce_t<_Tp, _Sizes>> && ...) 3972 && (!__is_fixed_size_abi_v< 3973 simd_abi::deduce_t<_Tp, _Sizes>> && ...)) 3974 { 3975 if constexpr (((_Sizes * 2 == _Np) && ...)) 3976 return {{__private_init, __extract_part<0, 2>(__data(__x))}, 3977 {__private_init, __extract_part<1, 2>(__data(__x))}}; 3978 else if constexpr (is_same_v<_SizeList<_Sizes...>, 3979 _SizeList<_Np / 3, _Np / 3, _Np / 3>>) 3980 return {{__private_init, __extract_part<0, 3>(__data(__x))}, 3981 {__private_init, __extract_part<1, 3>(__data(__x))}, 3982 {__private_init, __extract_part<2, 3>(__data(__x))}}; 3983 else if constexpr (is_same_v<_SizeList<_Sizes...>, 3984 _SizeList<2 * _Np / 3, _Np / 3>>) 3985 return {{__private_init, __extract_part<0, 3, 2>(__data(__x))}, 3986 {__private_init, __extract_part<2, 3>(__data(__x))}}; 3987 else if constexpr (is_same_v<_SizeList<_Sizes...>, 3988 _SizeList<_Np / 3, 2 * _Np / 3>>) 3989 return {{__private_init, __extract_part<0, 3>(__data(__x))}, 3990 {__private_init, __extract_part<1, 3, 2>(__data(__x))}}; 3991 else if constexpr (is_same_v<_SizeList<_Sizes...>, 3992 _SizeList<_Np / 2, _Np / 4, _Np / 4>>) 3993 return {{__private_init, __extract_part<0, 2>(__data(__x))}, 3994 {__private_init, __extract_part<2, 4>(__data(__x))}, 3995 {__private_init, __extract_part<3, 4>(__data(__x))}}; 3996 else if constexpr (is_same_v<_SizeList<_Sizes...>, 3997 _SizeList<_Np / 4, _Np / 4, _Np / 2>>) 3998 return {{__private_init, __extract_part<0, 4>(__data(__x))}, 3999 {__private_init, __extract_part<1, 4>(__data(__x))}, 4000 {__private_init, __extract_part<1, 2>(__data(__x))}}; 4001 else if constexpr (is_same_v<_SizeList<_Sizes...>, 4002 _SizeList<_Np / 4, _Np / 2, _Np / 4>>) 4003 return {{__private_init, __extract_part<0, 4>(__data(__x))}, 4004 {__private_init, __extract_center(__data(__x))}, 4005 {__private_init, __extract_part<3, 4>(__data(__x))}}; 4006 else if constexpr (((_Sizes * 4 == _Np) && ...)) 4007 return {{__private_init, __extract_part<0, 4>(__data(__x))}, 4008 {__private_init, __extract_part<1, 4>(__data(__x))}, 4009 {__private_init, __extract_part<2, 4>(__data(__x))}, 4010 {__private_init, __extract_part<3, 4>(__data(__x))}}; 4011 // else fall through 4012 } 4013 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS 4014 const __may_alias<_Tp>* const __element_ptr 4015 = reinterpret_cast<const __may_alias<_Tp>*>(&__x); 4016 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>( 4017 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4018 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>; 4019 constexpr size_t __offset = _SL::_S_before(__i); 4020 constexpr size_t __base_align = alignof(simd<_Tp, _Ap>); 4021 constexpr size_t __a 4022 = __base_align - ((__offset * sizeof(_Tp)) % __base_align); 4023 constexpr size_t __b = ((__a - 1) & __a) ^ __a; 4024 constexpr size_t __alignment = __b == 0 ? __a : __b; 4025 return _Vi(__element_ptr + __offset, overaligned<__alignment>); 4026 }); 4027 #else 4028 return __generate_from_n_evaluations<sizeof...(_Sizes), _Tuple>( 4029 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4030 using _Vi = __deduced_simd<_Tp, _SL::_S_at(__i)>; 4031 const auto& __xx = __data(__x); 4032 using _Offset = decltype(_SL::_S_before(__i)); 4033 return _Vi([&](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4034 constexpr _SizeConstant<_Offset::value + __j> __k; 4035 return __xx[__k]; 4036 }); 4037 }); 4038 #endif 4039 } 4040 4041 // }}} 4042 4043 // __subscript_in_pack {{{ 4044 template <size_t _I, typename _Tp, typename _Ap, typename... _As> 4045 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp 4046 __subscript_in_pack(const simd<_Tp, _Ap>& __x, const simd<_Tp, _As>&... __xs) 4047 { 4048 if constexpr (_I < simd_size_v<_Tp, _Ap>) 4049 return __x[_I]; 4050 else 4051 return __subscript_in_pack<_I - simd_size_v<_Tp, _Ap>>(__xs...); 4052 } 4053 4054 // }}} 4055 // __store_pack_of_simd {{{ 4056 template <typename _Tp, typename _A0, typename... _As> 4057 _GLIBCXX_SIMD_INTRINSIC void 4058 __store_pack_of_simd(char* __mem, const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs) 4059 { 4060 constexpr size_t __n_bytes = sizeof(_Tp) * simd_size_v<_Tp, _A0>; 4061 __builtin_memcpy(__mem, &__data(__x0), __n_bytes); 4062 if constexpr (sizeof...(__xs) > 0) 4063 __store_pack_of_simd(__mem + __n_bytes, __xs...); 4064 } 4065 4066 // }}} 4067 // concat(simd...) {{{ 4068 template <typename _Tp, typename... _As> 4069 inline _GLIBCXX_SIMD_CONSTEXPR 4070 simd<_Tp, simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>> 4071 concat(const simd<_Tp, _As>&... __xs) 4072 { 4073 using _Rp = __deduced_simd<_Tp, (simd_size_v<_Tp, _As> + ...)>; 4074 if constexpr (sizeof...(__xs) == 1) 4075 return simd_cast<_Rp>(__xs...); 4076 else if ((... && __xs._M_is_constprop())) 4077 return simd<_Tp, 4078 simd_abi::deduce_t<_Tp, (simd_size_v<_Tp, _As> + ...)>>( 4079 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 4080 { return __subscript_in_pack<__i>(__xs...); }); 4081 else 4082 { 4083 _Rp __r{}; 4084 __store_pack_of_simd(reinterpret_cast<char*>(&__data(__r)), __xs...); 4085 return __r; 4086 } 4087 } 4088 4089 // }}} 4090 // concat(array<simd>) {{{ 4091 template <typename _Tp, typename _Abi, size_t _Np> 4092 _GLIBCXX_SIMD_ALWAYS_INLINE 4093 _GLIBCXX_SIMD_CONSTEXPR __deduced_simd<_Tp, simd_size_v<_Tp, _Abi> * _Np> 4094 concat(const array<simd<_Tp, _Abi>, _Np>& __x) 4095 { 4096 return __call_with_subscripts<_Np>( 4097 __x, [](const auto&... __xs) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4098 return concat(__xs...); 4099 }); 4100 } 4101 4102 // }}} 4103 4104 /// @cond undocumented 4105 // _SmartReference {{{ 4106 template <typename _Up, typename _Accessor = _Up, 4107 typename _ValueType = typename _Up::value_type> 4108 class _SmartReference 4109 { 4110 friend _Accessor; 4111 int _M_index; 4112 _Up& _M_obj; 4113 4114 _GLIBCXX_SIMD_INTRINSIC constexpr _ValueType 4115 _M_read() const noexcept 4116 { 4117 if constexpr (is_arithmetic_v<_Up>) 4118 return _M_obj; 4119 else 4120 return _M_obj[_M_index]; 4121 } 4122 4123 template <typename _Tp> 4124 _GLIBCXX_SIMD_INTRINSIC constexpr void 4125 _M_write(_Tp&& __x) const 4126 { _Accessor::_S_set(_M_obj, _M_index, static_cast<_Tp&&>(__x)); } 4127 4128 public: 4129 _GLIBCXX_SIMD_INTRINSIC constexpr 4130 _SmartReference(_Up& __o, int __i) noexcept 4131 : _M_index(__i), _M_obj(__o) {} 4132 4133 using value_type = _ValueType; 4134 4135 _GLIBCXX_SIMD_INTRINSIC 4136 _SmartReference(const _SmartReference&) = delete; 4137 4138 _GLIBCXX_SIMD_INTRINSIC constexpr 4139 operator value_type() const noexcept 4140 { return _M_read(); } 4141 4142 template <typename _Tp, typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, value_type>> 4143 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference 4144 operator=(_Tp&& __x) && 4145 { 4146 _M_write(static_cast<_Tp&&>(__x)); 4147 return {_M_obj, _M_index}; 4148 } 4149 4150 #define _GLIBCXX_SIMD_OP_(__op) \ 4151 template <typename _Tp, \ 4152 typename _TT = decltype(declval<value_type>() __op declval<_Tp>()), \ 4153 typename = _ValuePreservingOrInt<__remove_cvref_t<_Tp>, _TT>, \ 4154 typename = _ValuePreservingOrInt<_TT, value_type>> \ 4155 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference \ 4156 operator __op##=(_Tp&& __x) && \ 4157 { \ 4158 const value_type& __lhs = _M_read(); \ 4159 _M_write(__lhs __op __x); \ 4160 return {_M_obj, _M_index}; \ 4161 } 4162 _GLIBCXX_SIMD_ALL_ARITHMETICS(_GLIBCXX_SIMD_OP_); 4163 _GLIBCXX_SIMD_ALL_SHIFTS(_GLIBCXX_SIMD_OP_); 4164 _GLIBCXX_SIMD_ALL_BINARY(_GLIBCXX_SIMD_OP_); 4165 #undef _GLIBCXX_SIMD_OP_ 4166 4167 template <typename _Tp = void, 4168 typename = decltype(++declval<conditional_t<true, value_type, _Tp>&>())> 4169 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference 4170 operator++() && 4171 { 4172 value_type __x = _M_read(); 4173 _M_write(++__x); 4174 return {_M_obj, _M_index}; 4175 } 4176 4177 template <typename _Tp = void, 4178 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()++)> 4179 _GLIBCXX_SIMD_INTRINSIC constexpr value_type 4180 operator++(int) && 4181 { 4182 const value_type __r = _M_read(); 4183 value_type __x = __r; 4184 _M_write(++__x); 4185 return __r; 4186 } 4187 4188 template <typename _Tp = void, 4189 typename = decltype(--declval<conditional_t<true, value_type, _Tp>&>())> 4190 _GLIBCXX_SIMD_INTRINSIC constexpr _SmartReference 4191 operator--() && 4192 { 4193 value_type __x = _M_read(); 4194 _M_write(--__x); 4195 return {_M_obj, _M_index}; 4196 } 4197 4198 template <typename _Tp = void, 4199 typename = decltype(declval<conditional_t<true, value_type, _Tp>&>()--)> 4200 _GLIBCXX_SIMD_INTRINSIC constexpr value_type 4201 operator--(int) && 4202 { 4203 const value_type __r = _M_read(); 4204 value_type __x = __r; 4205 _M_write(--__x); 4206 return __r; 4207 } 4208 4209 _GLIBCXX_SIMD_INTRINSIC friend void 4210 swap(_SmartReference&& __a, _SmartReference&& __b) noexcept( 4211 conjunction< 4212 is_nothrow_constructible<value_type, _SmartReference&&>, 4213 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value) 4214 { 4215 value_type __tmp = static_cast<_SmartReference&&>(__a); 4216 static_cast<_SmartReference&&>(__a) = static_cast<value_type>(__b); 4217 static_cast<_SmartReference&&>(__b) = std::move(__tmp); 4218 } 4219 4220 _GLIBCXX_SIMD_INTRINSIC friend void 4221 swap(value_type& __a, _SmartReference&& __b) noexcept( 4222 conjunction< 4223 is_nothrow_constructible<value_type, value_type&&>, 4224 is_nothrow_assignable<value_type&, value_type&&>, 4225 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value) 4226 { 4227 value_type __tmp(std::move(__a)); 4228 __a = static_cast<value_type>(__b); 4229 static_cast<_SmartReference&&>(__b) = std::move(__tmp); 4230 } 4231 4232 _GLIBCXX_SIMD_INTRINSIC friend void 4233 swap(_SmartReference&& __a, value_type& __b) noexcept( 4234 conjunction< 4235 is_nothrow_constructible<value_type, _SmartReference&&>, 4236 is_nothrow_assignable<value_type&, value_type&&>, 4237 is_nothrow_assignable<_SmartReference&&, value_type&&>>::value) 4238 { 4239 value_type __tmp(__a); 4240 static_cast<_SmartReference&&>(__a) = std::move(__b); 4241 __b = std::move(__tmp); 4242 } 4243 }; 4244 4245 // }}} 4246 // __scalar_abi_wrapper {{{ 4247 template <int _Bytes> 4248 struct __scalar_abi_wrapper 4249 { 4250 template <typename _Tp> static constexpr size_t _S_full_size = 1; 4251 template <typename _Tp> static constexpr size_t _S_size = 1; 4252 template <typename _Tp> static constexpr size_t _S_is_partial = false; 4253 4254 template <typename _Tp, typename _Abi = simd_abi::scalar> 4255 static constexpr bool _S_is_valid_v 4256 = _Abi::template _IsValid<_Tp>::value && sizeof(_Tp) == _Bytes; 4257 }; 4258 4259 // }}} 4260 // __decay_abi metafunction {{{ 4261 template <typename _Tp> 4262 struct __decay_abi { using type = _Tp; }; 4263 4264 template <int _Bytes> 4265 struct __decay_abi<__scalar_abi_wrapper<_Bytes>> 4266 { using type = simd_abi::scalar; }; 4267 4268 // }}} 4269 // __find_next_valid_abi metafunction {{{1 4270 // Given an ABI tag A<N>, find an N2 < N such that A<N2>::_S_is_valid_v<_Tp> == 4271 // true, N2 is a power-of-2, and A<N2>::_S_is_partial<_Tp> is false. Break 4272 // recursion at 2 elements in the resulting ABI tag. In this case 4273 // type::_S_is_valid_v<_Tp> may be false. 4274 template <template <int> class _Abi, int _Bytes, typename _Tp> 4275 struct __find_next_valid_abi 4276 { 4277 static constexpr auto 4278 _S_choose() 4279 { 4280 constexpr int _NextBytes = std::__bit_ceil(_Bytes) / 2; 4281 using _NextAbi = _Abi<_NextBytes>; 4282 if constexpr (_NextBytes < sizeof(_Tp) * 2) // break recursion 4283 return _Abi<_Bytes>(); 4284 else if constexpr (_NextAbi::template _S_is_partial<_Tp> == false 4285 && _NextAbi::template _S_is_valid_v<_Tp>) 4286 return _NextAbi(); 4287 else 4288 return __find_next_valid_abi<_Abi, _NextBytes, _Tp>::_S_choose(); 4289 } 4290 4291 using type = decltype(_S_choose()); 4292 }; 4293 4294 template <int _Bytes, typename _Tp> 4295 struct __find_next_valid_abi<__scalar_abi_wrapper, _Bytes, _Tp> 4296 { using type = simd_abi::scalar; }; 4297 4298 // _AbiList {{{1 4299 template <template <int> class...> 4300 struct _AbiList 4301 { 4302 template <typename, int> static constexpr bool _S_has_valid_abi = false; 4303 template <typename, int> using _FirstValidAbi = void; 4304 template <typename, int> using _BestAbi = void; 4305 }; 4306 4307 template <template <int> class _A0, template <int> class... _Rest> 4308 struct _AbiList<_A0, _Rest...> 4309 { 4310 template <typename _Tp, int _Np> 4311 static constexpr bool _S_has_valid_abi 4312 = _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v< 4313 _Tp> || _AbiList<_Rest...>::template _S_has_valid_abi<_Tp, _Np>; 4314 4315 template <typename _Tp, int _Np> 4316 using _FirstValidAbi = conditional_t< 4317 _A0<sizeof(_Tp) * _Np>::template _S_is_valid_v<_Tp>, 4318 typename __decay_abi<_A0<sizeof(_Tp) * _Np>>::type, 4319 typename _AbiList<_Rest...>::template _FirstValidAbi<_Tp, _Np>>; 4320 4321 template <typename _Tp, int _Np> 4322 static constexpr auto 4323 _S_determine_best_abi() 4324 { 4325 static_assert(_Np >= 1); 4326 constexpr int _Bytes = sizeof(_Tp) * _Np; 4327 if constexpr (_Np == 1) 4328 return __make_dependent_t<_Tp, simd_abi::scalar>{}; 4329 else 4330 { 4331 constexpr int __fullsize = _A0<_Bytes>::template _S_full_size<_Tp>; 4332 // _A0<_Bytes> is good if: 4333 // 1. The ABI tag is valid for _Tp 4334 // 2. The storage overhead is no more than padding to fill the next 4335 // power-of-2 number of bytes 4336 if constexpr (_A0<_Bytes>::template _S_is_valid_v< 4337 _Tp> && __fullsize / 2 < _Np) 4338 return typename __decay_abi<_A0<_Bytes>>::type{}; 4339 else 4340 { 4341 using _Bp = 4342 typename __find_next_valid_abi<_A0, _Bytes, _Tp>::type; 4343 if constexpr (_Bp::template _S_is_valid_v< 4344 _Tp> && _Bp::template _S_size<_Tp> <= _Np) 4345 return _Bp{}; 4346 else 4347 return 4348 typename _AbiList<_Rest...>::template _BestAbi<_Tp, _Np>{}; 4349 } 4350 } 4351 } 4352 4353 template <typename _Tp, int _Np> 4354 using _BestAbi = decltype(_S_determine_best_abi<_Tp, _Np>()); 4355 }; 4356 4357 // }}}1 4358 4359 // the following lists all native ABIs, which makes them accessible to 4360 // simd_abi::deduce and select_best_vector_type_t (for fixed_size). Order 4361 // matters: Whatever comes first has higher priority. 4362 using _AllNativeAbis = _AbiList<simd_abi::_VecBltnBtmsk, simd_abi::_VecBuiltin, 4363 __scalar_abi_wrapper>; 4364 4365 // valid _SimdTraits specialization {{{1 4366 template <typename _Tp, typename _Abi> 4367 struct _SimdTraits<_Tp, _Abi, void_t<typename _Abi::template _IsValid<_Tp>>> 4368 : _Abi::template __traits<_Tp> {}; 4369 4370 // __deduce_impl specializations {{{1 4371 // try all native ABIs (including scalar) first 4372 template <typename _Tp, size_t _Np> 4373 struct __deduce_impl< 4374 _Tp, _Np, enable_if_t<_AllNativeAbis::template _S_has_valid_abi<_Tp, _Np>>> 4375 { using type = _AllNativeAbis::_FirstValidAbi<_Tp, _Np>; }; 4376 4377 // fall back to fixed_size only if scalar and native ABIs don't match 4378 template <typename _Tp, size_t _Np, typename = void> 4379 struct __deduce_fixed_size_fallback {}; 4380 4381 template <typename _Tp, size_t _Np> 4382 struct __deduce_fixed_size_fallback<_Tp, _Np, 4383 enable_if_t<simd_abi::fixed_size<_Np>::template _S_is_valid_v<_Tp>>> 4384 { using type = simd_abi::fixed_size<_Np>; }; 4385 4386 template <typename _Tp, size_t _Np, typename> 4387 struct __deduce_impl : public __deduce_fixed_size_fallback<_Tp, _Np> {}; 4388 4389 //}}}1 4390 /// @endcond 4391 4392 // simd_mask {{{ 4393 template <typename _Tp, typename _Abi> 4394 class simd_mask : public _SimdTraits<_Tp, _Abi>::_MaskBase 4395 { 4396 // types, tags, and friends {{{ 4397 using _Traits = _SimdTraits<_Tp, _Abi>; 4398 using _MemberType = typename _Traits::_MaskMember; 4399 4400 // We map all masks with equal element sizeof to a single integer type, the 4401 // one given by __int_for_sizeof_t<_Tp>. This is the approach 4402 // [[gnu::vector_size(N)]] types take as well and it reduces the number of 4403 // template specializations in the implementation classes. 4404 using _Ip = __int_for_sizeof_t<_Tp>; 4405 static constexpr _Ip* _S_type_tag = nullptr; 4406 4407 friend typename _Traits::_MaskBase; 4408 friend class simd<_Tp, _Abi>; // to construct masks on return 4409 friend typename _Traits::_SimdImpl; // to construct masks on return and 4410 // inspect data on masked operations 4411 public: 4412 using _Impl = typename _Traits::_MaskImpl; 4413 friend _Impl; 4414 4415 // }}} 4416 // member types {{{ 4417 using value_type = bool; 4418 using reference = _SmartReference<_MemberType, _Impl, value_type>; 4419 using simd_type = simd<_Tp, _Abi>; 4420 using abi_type = _Abi; 4421 4422 // }}} 4423 static constexpr size_t size() // {{{ 4424 { return __size_or_zero_v<_Tp, _Abi>; } 4425 4426 // }}} 4427 // constructors & assignment {{{ 4428 simd_mask() = default; 4429 simd_mask(const simd_mask&) = default; 4430 simd_mask(simd_mask&&) = default; 4431 simd_mask& operator=(const simd_mask&) = default; 4432 simd_mask& operator=(simd_mask&&) = default; 4433 4434 // }}} 4435 // access to internal representation (optional feature) {{{ 4436 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR explicit 4437 simd_mask(typename _Traits::_MaskCastType __init) 4438 : _M_data{__init} {} 4439 // conversions to internal type is done in _MaskBase 4440 4441 // }}} 4442 // bitset interface (extension to be proposed) {{{ 4443 // TS_FEEDBACK: 4444 // Conversion of simd_mask to and from bitset makes it much easier to 4445 // interface with other facilities. I suggest adding `static 4446 // simd_mask::from_bitset` and `simd_mask::to_bitset`. 4447 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR static simd_mask 4448 __from_bitset(bitset<size()> bs) 4449 { return {__bitset_init, bs}; } 4450 4451 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bitset<size()> 4452 __to_bitset() const 4453 { return _Impl::_S_to_bits(_M_data)._M_to_bitset(); } 4454 4455 // }}} 4456 // explicit broadcast constructor {{{ 4457 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR 4458 simd_mask(value_type __x) 4459 : _M_data(_Impl::template _S_broadcast<_Ip>(__x)) {} 4460 4461 // }}} 4462 // implicit type conversion constructor {{{ 4463 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST 4464 // proposed improvement 4465 template <typename _Up, typename _A2, 4466 typename = enable_if_t<simd_size_v<_Up, _A2> == size()>> 4467 _GLIBCXX_SIMD_ALWAYS_INLINE explicit(sizeof(_MemberType) 4468 != sizeof(typename _SimdTraits<_Up, _A2>::_MaskMember)) 4469 simd_mask(const simd_mask<_Up, _A2>& __x) 4470 : simd_mask(__proposed::static_simd_cast<simd_mask>(__x)) {} 4471 #else 4472 // conforming to ISO/IEC 19570:2018 4473 template <typename _Up, typename = enable_if_t<conjunction< 4474 is_same<abi_type, simd_abi::fixed_size<size()>>, 4475 is_same<_Up, _Up>>::value>> 4476 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR 4477 simd_mask(const simd_mask<_Up, simd_abi::fixed_size<size()>>& __x) 4478 : _M_data(_Impl::_S_from_bitmask(__data(__x), _S_type_tag)) {} 4479 #endif 4480 4481 // }}} 4482 // load constructor {{{ 4483 template <typename _Flags> 4484 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR 4485 simd_mask(const value_type* __mem, _IsSimdFlagType<_Flags>) 4486 : _M_data(_Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem))) {} 4487 4488 template <typename _Flags> 4489 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR 4490 simd_mask(const value_type* __mem, simd_mask __k, _IsSimdFlagType<_Flags>) 4491 : _M_data{} 4492 { 4493 _M_data = _Impl::_S_masked_load(_M_data, __k._M_data, 4494 _Flags::template _S_apply<simd_mask>(__mem)); 4495 } 4496 4497 // }}} 4498 // loads [simd_mask.load] {{{ 4499 template <typename _Flags> 4500 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void 4501 copy_from(const value_type* __mem, _IsSimdFlagType<_Flags>) 4502 { _M_data = _Impl::template _S_load<_Ip>(_Flags::template _S_apply<simd_mask>(__mem)); } 4503 4504 // }}} 4505 // stores [simd_mask.store] {{{ 4506 template <typename _Flags> 4507 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void 4508 copy_to(value_type* __mem, _IsSimdFlagType<_Flags>) const 4509 { _Impl::_S_store(_M_data, _Flags::template _S_apply<simd_mask>(__mem)); } 4510 4511 // }}} 4512 // scalar access {{{ 4513 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference 4514 operator[](size_t __i) 4515 { 4516 if (__i >= size()) 4517 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1); 4518 return {_M_data, int(__i)}; 4519 } 4520 4521 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type 4522 operator[](size_t __i) const 4523 { 4524 if (__i >= size()) 4525 __invoke_ub("Subscript %d is out of range [0, %d]", __i, size() - 1); 4526 if constexpr (__is_scalar_abi<_Abi>()) 4527 return _M_data; 4528 else 4529 return static_cast<bool>(_M_data[__i]); 4530 } 4531 4532 // }}} 4533 // negation {{{ 4534 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd_mask 4535 operator!() const 4536 { return {__private_init, _Impl::_S_bit_not(_M_data)}; } 4537 4538 // }}} 4539 // simd_mask binary operators [simd_mask.binary] {{{ 4540 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST 4541 // simd_mask<int> && simd_mask<uint> needs disambiguation 4542 template <typename _Up, typename _A2, 4543 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>> 4544 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4545 operator&&(const simd_mask& __x, const simd_mask<_Up, _A2>& __y) 4546 { 4547 return {__private_init, 4548 _Impl::_S_logical_and(__x._M_data, simd_mask(__y)._M_data)}; 4549 } 4550 4551 template <typename _Up, typename _A2, 4552 typename = enable_if_t<is_convertible_v<simd_mask<_Up, _A2>, simd_mask>>> 4553 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4554 operator||(const simd_mask& __x, const simd_mask<_Up, _A2>& __y) 4555 { 4556 return {__private_init, 4557 _Impl::_S_logical_or(__x._M_data, simd_mask(__y)._M_data)}; 4558 } 4559 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST 4560 4561 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4562 operator&&(const simd_mask& __x, const simd_mask& __y) 4563 { return {__private_init, _Impl::_S_logical_and(__x._M_data, __y._M_data)}; } 4564 4565 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4566 operator||(const simd_mask& __x, const simd_mask& __y) 4567 { return {__private_init, _Impl::_S_logical_or(__x._M_data, __y._M_data)}; } 4568 4569 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4570 operator&(const simd_mask& __x, const simd_mask& __y) 4571 { return {__private_init, _Impl::_S_bit_and(__x._M_data, __y._M_data)}; } 4572 4573 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4574 operator|(const simd_mask& __x, const simd_mask& __y) 4575 { return {__private_init, _Impl::_S_bit_or(__x._M_data, __y._M_data)}; } 4576 4577 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4578 operator^(const simd_mask& __x, const simd_mask& __y) 4579 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; } 4580 4581 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask& 4582 operator&=(simd_mask& __x, const simd_mask& __y) 4583 { 4584 __x._M_data = _Impl::_S_bit_and(__x._M_data, __y._M_data); 4585 return __x; 4586 } 4587 4588 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask& 4589 operator|=(simd_mask& __x, const simd_mask& __y) 4590 { 4591 __x._M_data = _Impl::_S_bit_or(__x._M_data, __y._M_data); 4592 return __x; 4593 } 4594 4595 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask& 4596 operator^=(simd_mask& __x, const simd_mask& __y) 4597 { 4598 __x._M_data = _Impl::_S_bit_xor(__x._M_data, __y._M_data); 4599 return __x; 4600 } 4601 4602 // }}} 4603 // simd_mask compares [simd_mask.comparison] {{{ 4604 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4605 operator==(const simd_mask& __x, const simd_mask& __y) 4606 { return !operator!=(__x, __y); } 4607 4608 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4609 operator!=(const simd_mask& __x, const simd_mask& __y) 4610 { return {__private_init, _Impl::_S_bit_xor(__x._M_data, __y._M_data)}; } 4611 4612 // }}} 4613 // private_init ctor {{{ 4614 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 4615 simd_mask(_PrivateInit, typename _Traits::_MaskMember __init) 4616 : _M_data(__init) {} 4617 4618 // }}} 4619 // private_init generator ctor {{{ 4620 template <typename _Fp, typename = decltype(bool(declval<_Fp>()(size_t())))> 4621 _GLIBCXX_SIMD_INTRINSIC constexpr 4622 simd_mask(_PrivateInit, _Fp&& __gen) 4623 : _M_data() 4624 { 4625 __execute_n_times<size()>([&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4626 _Impl::_S_set(_M_data, __i, __gen(__i)); 4627 }); 4628 } 4629 4630 // }}} 4631 // bitset_init ctor {{{ 4632 _GLIBCXX_SIMD_INTRINSIC constexpr 4633 simd_mask(_BitsetInit, bitset<size()> __init) 4634 : _M_data(_Impl::_S_from_bitmask(_SanitizedBitMask<size()>(__init), _S_type_tag)) 4635 {} 4636 4637 // }}} 4638 // __cvt {{{ 4639 // TS_FEEDBACK: 4640 // The conversion operator this implements should be a ctor on simd_mask. 4641 // Once you call .__cvt() on a simd_mask it converts conveniently. 4642 // A useful variation: add `explicit(sizeof(_Tp) != sizeof(_Up))` 4643 struct _CvtProxy 4644 { 4645 template <typename _Up, typename _A2, 4646 typename = enable_if_t<simd_size_v<_Up, _A2> == simd_size_v<_Tp, _Abi>>> 4647 operator simd_mask<_Up, _A2>() && 4648 { 4649 using namespace std::experimental::__proposed; 4650 return static_simd_cast<simd_mask<_Up, _A2>>(_M_data); 4651 } 4652 4653 const simd_mask<_Tp, _Abi>& _M_data; 4654 }; 4655 4656 _GLIBCXX_SIMD_INTRINSIC _CvtProxy 4657 __cvt() const 4658 { return {*this}; } 4659 4660 // }}} 4661 // operator?: overloads (suggested extension) {{{ 4662 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__ 4663 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4664 operator?:(const simd_mask& __k, const simd_mask& __where_true, 4665 const simd_mask& __where_false) 4666 { 4667 auto __ret = __where_false; 4668 _Impl::_S_masked_assign(__k._M_data, __ret._M_data, __where_true._M_data); 4669 return __ret; 4670 } 4671 4672 template <typename _U1, typename _U2, 4673 typename _Rp = simd<common_type_t<_U1, _U2>, _Abi>, 4674 typename = enable_if_t<conjunction_v< 4675 is_convertible<_U1, _Rp>, is_convertible<_U2, _Rp>, 4676 is_convertible<simd_mask, typename _Rp::mask_type>>>> 4677 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend _Rp 4678 operator?:(const simd_mask& __k, const _U1& __where_true, 4679 const _U2& __where_false) 4680 { 4681 _Rp __ret = __where_false; 4682 _Rp::_Impl::_S_masked_assign( 4683 __data(static_cast<typename _Rp::mask_type>(__k)), __data(__ret), 4684 __data(static_cast<_Rp>(__where_true))); 4685 return __ret; 4686 } 4687 4688 #ifdef _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST 4689 template <typename _Kp, typename _Ak, typename _Up, typename _Au, 4690 typename = enable_if_t< 4691 conjunction_v<is_convertible<simd_mask<_Kp, _Ak>, simd_mask>, 4692 is_convertible<simd_mask<_Up, _Au>, simd_mask>>>> 4693 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd_mask 4694 operator?:(const simd_mask<_Kp, _Ak>& __k, const simd_mask& __where_true, 4695 const simd_mask<_Up, _Au>& __where_false) 4696 { 4697 simd_mask __ret = __where_false; 4698 _Impl::_S_masked_assign(simd_mask(__k)._M_data, __ret._M_data, 4699 __where_true._M_data); 4700 return __ret; 4701 } 4702 #endif // _GLIBCXX_SIMD_ENABLE_IMPLICIT_MASK_CAST 4703 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__ 4704 4705 // }}} 4706 // _M_is_constprop {{{ 4707 _GLIBCXX_SIMD_INTRINSIC constexpr bool 4708 _M_is_constprop() const 4709 { 4710 if constexpr (__is_scalar_abi<_Abi>()) 4711 return __builtin_constant_p(_M_data); 4712 else 4713 return _M_data._M_is_constprop(); 4714 } 4715 4716 // }}} 4717 4718 private: 4719 friend const auto& __data<_Tp, abi_type>(const simd_mask&); 4720 friend auto& __data<_Tp, abi_type>(simd_mask&); 4721 alignas(_Traits::_S_mask_align) _MemberType _M_data; 4722 }; 4723 4724 // }}} 4725 4726 /// @cond undocumented 4727 // __data(simd_mask) {{{ 4728 template <typename _Tp, typename _Ap> 4729 _GLIBCXX_SIMD_INTRINSIC constexpr const auto& 4730 __data(const simd_mask<_Tp, _Ap>& __x) 4731 { return __x._M_data; } 4732 4733 template <typename _Tp, typename _Ap> 4734 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 4735 __data(simd_mask<_Tp, _Ap>& __x) 4736 { return __x._M_data; } 4737 4738 // }}} 4739 /// @endcond 4740 4741 // simd_mask reductions [simd_mask.reductions] {{{ 4742 template <typename _Tp, typename _Abi> 4743 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4744 all_of(const simd_mask<_Tp, _Abi>& __k) noexcept 4745 { 4746 if (__builtin_is_constant_evaluated() || __k._M_is_constprop()) 4747 { 4748 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i) 4749 if (!__k[__i]) 4750 return false; 4751 return true; 4752 } 4753 else 4754 return _Abi::_MaskImpl::_S_all_of(__k); 4755 } 4756 4757 template <typename _Tp, typename _Abi> 4758 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4759 any_of(const simd_mask<_Tp, _Abi>& __k) noexcept 4760 { 4761 if (__builtin_is_constant_evaluated() || __k._M_is_constprop()) 4762 { 4763 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i) 4764 if (__k[__i]) 4765 return true; 4766 return false; 4767 } 4768 else 4769 return _Abi::_MaskImpl::_S_any_of(__k); 4770 } 4771 4772 template <typename _Tp, typename _Abi> 4773 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4774 none_of(const simd_mask<_Tp, _Abi>& __k) noexcept 4775 { 4776 if (__builtin_is_constant_evaluated() || __k._M_is_constprop()) 4777 { 4778 for (size_t __i = 0; __i < simd_size_v<_Tp, _Abi>; ++__i) 4779 if (__k[__i]) 4780 return false; 4781 return true; 4782 } 4783 else 4784 return _Abi::_MaskImpl::_S_none_of(__k); 4785 } 4786 4787 template <typename _Tp, typename _Abi> 4788 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4789 some_of(const simd_mask<_Tp, _Abi>& __k) noexcept 4790 { 4791 if (__builtin_is_constant_evaluated() || __k._M_is_constprop()) 4792 { 4793 for (size_t __i = 1; __i < simd_size_v<_Tp, _Abi>; ++__i) 4794 if (__k[__i] != __k[__i - 1]) 4795 return true; 4796 return false; 4797 } 4798 else 4799 return _Abi::_MaskImpl::_S_some_of(__k); 4800 } 4801 4802 template <typename _Tp, typename _Abi> 4803 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int 4804 popcount(const simd_mask<_Tp, _Abi>& __k) noexcept 4805 { 4806 if (__builtin_is_constant_evaluated() || __k._M_is_constprop()) 4807 { 4808 const int __r = __call_with_subscripts<simd_size_v<_Tp, _Abi>>( 4809 __k, [](auto... __elements) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4810 return ((__elements != 0) + ...); 4811 }); 4812 if (__builtin_is_constant_evaluated() || __builtin_constant_p(__r)) 4813 return __r; 4814 } 4815 return _Abi::_MaskImpl::_S_popcount(__k); 4816 } 4817 4818 template <typename _Tp, typename _Abi> 4819 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int 4820 find_first_set(const simd_mask<_Tp, _Abi>& __k) 4821 { 4822 if (__builtin_is_constant_evaluated() || __k._M_is_constprop()) 4823 { 4824 constexpr size_t _Np = simd_size_v<_Tp, _Abi>; 4825 const size_t _Idx = __call_with_n_evaluations<_Np>( 4826 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4827 return std::min({__indexes...}); 4828 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4829 return __k[__i] ? +__i : _Np; 4830 }); 4831 if (_Idx >= _Np) 4832 __invoke_ub("find_first_set(empty mask) is UB"); 4833 if (__builtin_constant_p(_Idx)) 4834 return _Idx; 4835 } 4836 return _Abi::_MaskImpl::_S_find_first_set(__k); 4837 } 4838 4839 template <typename _Tp, typename _Abi> 4840 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int 4841 find_last_set(const simd_mask<_Tp, _Abi>& __k) 4842 { 4843 if (__builtin_is_constant_evaluated() || __k._M_is_constprop()) 4844 { 4845 constexpr size_t _Np = simd_size_v<_Tp, _Abi>; 4846 const int _Idx = __call_with_n_evaluations<_Np>( 4847 [](auto... __indexes) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4848 return std::max({__indexes...}); 4849 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 4850 return __k[__i] ? int(__i) : -1; 4851 }); 4852 if (_Idx < 0) 4853 __invoke_ub("find_first_set(empty mask) is UB"); 4854 if (__builtin_constant_p(_Idx)) 4855 return _Idx; 4856 } 4857 return _Abi::_MaskImpl::_S_find_last_set(__k); 4858 } 4859 4860 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4861 all_of(_ExactBool __x) noexcept 4862 { return __x; } 4863 4864 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4865 any_of(_ExactBool __x) noexcept 4866 { return __x; } 4867 4868 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4869 none_of(_ExactBool __x) noexcept 4870 { return !__x; } 4871 4872 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR bool 4873 some_of(_ExactBool) noexcept 4874 { return false; } 4875 4876 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int 4877 popcount(_ExactBool __x) noexcept 4878 { return __x; } 4879 4880 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int 4881 find_first_set(_ExactBool) 4882 { return 0; } 4883 4884 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR int 4885 find_last_set(_ExactBool) 4886 { return 0; } 4887 4888 // }}} 4889 4890 /// @cond undocumented 4891 // _SimdIntOperators{{{1 4892 template <typename _V, typename _Impl, bool> 4893 class _SimdIntOperators {}; 4894 4895 template <typename _V, typename _Impl> 4896 class _SimdIntOperators<_V, _Impl, true> 4897 { 4898 _GLIBCXX_SIMD_INTRINSIC constexpr const _V& 4899 __derived() const 4900 { return *static_cast<const _V*>(this); } 4901 4902 template <typename _Tp> 4903 _GLIBCXX_SIMD_INTRINSIC static _GLIBCXX_SIMD_CONSTEXPR _V 4904 _S_make_derived(_Tp&& __d) 4905 { return {__private_init, static_cast<_Tp&&>(__d)}; } 4906 4907 public: 4908 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator%=(_V& __lhs, const _V& __x) 4909 { return __lhs = __lhs % __x; } 4910 4911 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator&=(_V& __lhs, const _V& __x) 4912 { return __lhs = __lhs & __x; } 4913 4914 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator|=(_V& __lhs, const _V& __x) 4915 { return __lhs = __lhs | __x; } 4916 4917 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator^=(_V& __lhs, const _V& __x) 4918 { return __lhs = __lhs ^ __x; } 4919 4920 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, const _V& __x) 4921 { return __lhs = __lhs << __x; } 4922 4923 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, const _V& __x) 4924 { return __lhs = __lhs >> __x; } 4925 4926 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator<<=(_V& __lhs, int __x) 4927 { return __lhs = __lhs << __x; } 4928 4929 _GLIBCXX_SIMD_CONSTEXPR friend _V& operator>>=(_V& __lhs, int __x) 4930 { return __lhs = __lhs >> __x; } 4931 4932 _GLIBCXX_SIMD_CONSTEXPR friend _V operator%(const _V& __x, const _V& __y) 4933 { 4934 return _SimdIntOperators::_S_make_derived( 4935 _Impl::_S_modulus(__data(__x), __data(__y))); 4936 } 4937 4938 _GLIBCXX_SIMD_CONSTEXPR friend _V operator&(const _V& __x, const _V& __y) 4939 { 4940 return _SimdIntOperators::_S_make_derived( 4941 _Impl::_S_bit_and(__data(__x), __data(__y))); 4942 } 4943 4944 _GLIBCXX_SIMD_CONSTEXPR friend _V operator|(const _V& __x, const _V& __y) 4945 { 4946 return _SimdIntOperators::_S_make_derived( 4947 _Impl::_S_bit_or(__data(__x), __data(__y))); 4948 } 4949 4950 _GLIBCXX_SIMD_CONSTEXPR friend _V operator^(const _V& __x, const _V& __y) 4951 { 4952 return _SimdIntOperators::_S_make_derived( 4953 _Impl::_S_bit_xor(__data(__x), __data(__y))); 4954 } 4955 4956 _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, const _V& __y) 4957 { 4958 return _SimdIntOperators::_S_make_derived( 4959 _Impl::_S_bit_shift_left(__data(__x), __data(__y))); 4960 } 4961 4962 _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, const _V& __y) 4963 { 4964 return _SimdIntOperators::_S_make_derived( 4965 _Impl::_S_bit_shift_right(__data(__x), __data(__y))); 4966 } 4967 4968 template <typename _VV = _V> 4969 _GLIBCXX_SIMD_CONSTEXPR friend _V operator<<(const _V& __x, int __y) 4970 { 4971 using _Tp = typename _VV::value_type; 4972 if (__y < 0) 4973 __invoke_ub("The behavior is undefined if the right operand of a " 4974 "shift operation is negative. [expr.shift]\nA shift by " 4975 "%d was requested", 4976 __y); 4977 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__) 4978 __invoke_ub( 4979 "The behavior is undefined if the right operand of a " 4980 "shift operation is greater than or equal to the width of the " 4981 "promoted left operand. [expr.shift]\nA shift by %d was requested", 4982 __y); 4983 return _SimdIntOperators::_S_make_derived( 4984 _Impl::_S_bit_shift_left(__data(__x), __y)); 4985 } 4986 4987 template <typename _VV = _V> 4988 _GLIBCXX_SIMD_CONSTEXPR friend _V operator>>(const _V& __x, int __y) 4989 { 4990 using _Tp = typename _VV::value_type; 4991 if (__y < 0) 4992 __invoke_ub( 4993 "The behavior is undefined if the right operand of a shift " 4994 "operation is negative. [expr.shift]\nA shift by %d was requested", 4995 __y); 4996 if (size_t(__y) >= sizeof(declval<_Tp>() << __y) * __CHAR_BIT__) 4997 __invoke_ub( 4998 "The behavior is undefined if the right operand of a shift " 4999 "operation is greater than or equal to the width of the promoted " 5000 "left operand. [expr.shift]\nA shift by %d was requested", 5001 __y); 5002 return _SimdIntOperators::_S_make_derived( 5003 _Impl::_S_bit_shift_right(__data(__x), __y)); 5004 } 5005 5006 // unary operators (for integral _Tp) 5007 _GLIBCXX_SIMD_CONSTEXPR _V operator~() const 5008 { return {__private_init, _Impl::_S_complement(__derived()._M_data)}; } 5009 }; 5010 5011 //}}}1 5012 /// @endcond 5013 5014 // simd {{{ 5015 template <typename _Tp, typename _Abi> 5016 class simd : public _SimdIntOperators< 5017 simd<_Tp, _Abi>, typename _SimdTraits<_Tp, _Abi>::_SimdImpl, 5018 conjunction<is_integral<_Tp>, 5019 typename _SimdTraits<_Tp, _Abi>::_IsValid>::value>, 5020 public _SimdTraits<_Tp, _Abi>::_SimdBase 5021 { 5022 using _Traits = _SimdTraits<_Tp, _Abi>; 5023 using _MemberType = typename _Traits::_SimdMember; 5024 using _CastType = typename _Traits::_SimdCastType; 5025 static constexpr _Tp* _S_type_tag = nullptr; 5026 friend typename _Traits::_SimdBase; 5027 5028 public: 5029 using _Impl = typename _Traits::_SimdImpl; 5030 friend _Impl; 5031 friend _SimdIntOperators<simd, _Impl, true>; 5032 5033 using value_type = _Tp; 5034 using reference = _SmartReference<_MemberType, _Impl, value_type>; 5035 using mask_type = simd_mask<_Tp, _Abi>; 5036 using abi_type = _Abi; 5037 5038 static constexpr size_t size() 5039 { return __size_or_zero_v<_Tp, _Abi>; } 5040 5041 _GLIBCXX_SIMD_CONSTEXPR simd() = default; 5042 _GLIBCXX_SIMD_CONSTEXPR simd(const simd&) = default; 5043 _GLIBCXX_SIMD_CONSTEXPR simd(simd&&) noexcept = default; 5044 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(const simd&) = default; 5045 _GLIBCXX_SIMD_CONSTEXPR simd& operator=(simd&&) noexcept = default; 5046 5047 // implicit broadcast constructor 5048 template <typename _Up, 5049 typename = enable_if_t<!is_same_v<__remove_cvref_t<_Up>, bool>>> 5050 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR 5051 simd(_ValuePreservingOrInt<_Up, value_type>&& __x) 5052 : _M_data( 5053 _Impl::_S_broadcast(static_cast<value_type>(static_cast<_Up&&>(__x)))) 5054 {} 5055 5056 // implicit type conversion constructor (convert from fixed_size to 5057 // fixed_size) 5058 template <typename _Up> 5059 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR 5060 simd(const simd<_Up, simd_abi::fixed_size<size()>>& __x, 5061 enable_if_t< 5062 conjunction< 5063 is_same<simd_abi::fixed_size<size()>, abi_type>, 5064 negation<__is_narrowing_conversion<_Up, value_type>>, 5065 __converts_to_higher_integer_rank<_Up, value_type>>::value, 5066 void*> = nullptr) 5067 : simd{static_cast<array<_Up, size()>>(__x).data(), vector_aligned} {} 5068 5069 // explicit type conversion constructor 5070 #ifdef _GLIBCXX_SIMD_ENABLE_STATIC_CAST 5071 template <typename _Up, typename _A2, 5072 typename = decltype(static_simd_cast<simd>( 5073 declval<const simd<_Up, _A2>&>()))> 5074 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR 5075 simd(const simd<_Up, _A2>& __x) 5076 : simd(static_simd_cast<simd>(__x)) {} 5077 #endif // _GLIBCXX_SIMD_ENABLE_STATIC_CAST 5078 5079 // generator constructor 5080 template <typename _Fp> 5081 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR 5082 simd(_Fp&& __gen, _ValuePreservingOrInt<decltype(declval<_Fp>()( 5083 declval<_SizeConstant<0>&>())), 5084 value_type>* = nullptr) 5085 : _M_data(_Impl::_S_generator(static_cast<_Fp&&>(__gen), _S_type_tag)) {} 5086 5087 // load constructor 5088 template <typename _Up, typename _Flags> 5089 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR 5090 simd(const _Up* __mem, _IsSimdFlagType<_Flags>) 5091 : _M_data( 5092 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag)) 5093 {} 5094 5095 // loads [simd.load] 5096 template <typename _Up, typename _Flags> 5097 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void 5098 copy_from(const _Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>) 5099 { 5100 _M_data = static_cast<decltype(_M_data)>( 5101 _Impl::_S_load(_Flags::template _S_apply<simd>(__mem), _S_type_tag)); 5102 } 5103 5104 // stores [simd.store] 5105 template <typename _Up, typename _Flags> 5106 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR void 5107 copy_to(_Vectorizable<_Up>* __mem, _IsSimdFlagType<_Flags>) const 5108 { 5109 _Impl::_S_store(_M_data, _Flags::template _S_apply<simd>(__mem), 5110 _S_type_tag); 5111 } 5112 5113 // scalar access 5114 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR reference 5115 operator[](size_t __i) 5116 { return {_M_data, int(__i)}; } 5117 5118 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR value_type 5119 operator[]([[maybe_unused]] size_t __i) const 5120 { 5121 if constexpr (__is_scalar_abi<_Abi>()) 5122 { 5123 _GLIBCXX_DEBUG_ASSERT(__i == 0); 5124 return _M_data; 5125 } 5126 else 5127 return _M_data[__i]; 5128 } 5129 5130 // increment and decrement: 5131 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd& 5132 operator++() 5133 { 5134 _Impl::_S_increment(_M_data); 5135 return *this; 5136 } 5137 5138 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd 5139 operator++(int) 5140 { 5141 simd __r = *this; 5142 _Impl::_S_increment(_M_data); 5143 return __r; 5144 } 5145 5146 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd& 5147 operator--() 5148 { 5149 _Impl::_S_decrement(_M_data); 5150 return *this; 5151 } 5152 5153 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd 5154 operator--(int) 5155 { 5156 simd __r = *this; 5157 _Impl::_S_decrement(_M_data); 5158 return __r; 5159 } 5160 5161 // unary operators (for any _Tp) 5162 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR mask_type 5163 operator!() const 5164 { return {__private_init, _Impl::_S_negate(_M_data)}; } 5165 5166 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd 5167 operator+() const 5168 { return *this; } 5169 5170 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR simd 5171 operator-() const 5172 { return {__private_init, _Impl::_S_unary_minus(_M_data)}; } 5173 5174 // access to internal representation (suggested extension) 5175 _GLIBCXX_SIMD_ALWAYS_INLINE explicit _GLIBCXX_SIMD_CONSTEXPR 5176 simd(_CastType __init) : _M_data(__init) {} 5177 5178 // compound assignment [simd.cassign] 5179 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd& 5180 operator+=(simd& __lhs, const simd& __x) 5181 { return __lhs = __lhs + __x; } 5182 5183 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd& 5184 operator-=(simd& __lhs, const simd& __x) 5185 { return __lhs = __lhs - __x; } 5186 5187 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd& 5188 operator*=(simd& __lhs, const simd& __x) 5189 { return __lhs = __lhs * __x; } 5190 5191 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd& 5192 operator/=(simd& __lhs, const simd& __x) 5193 { return __lhs = __lhs / __x; } 5194 5195 // binary operators [simd.binary] 5196 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd 5197 operator+(const simd& __x, const simd& __y) 5198 { return {__private_init, _Impl::_S_plus(__x._M_data, __y._M_data)}; } 5199 5200 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd 5201 operator-(const simd& __x, const simd& __y) 5202 { return {__private_init, _Impl::_S_minus(__x._M_data, __y._M_data)}; } 5203 5204 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd 5205 operator*(const simd& __x, const simd& __y) 5206 { return {__private_init, _Impl::_S_multiplies(__x._M_data, __y._M_data)}; } 5207 5208 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd 5209 operator/(const simd& __x, const simd& __y) 5210 { return {__private_init, _Impl::_S_divides(__x._M_data, __y._M_data)}; } 5211 5212 // compares [simd.comparison] 5213 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type 5214 operator==(const simd& __x, const simd& __y) 5215 { return simd::_S_make_mask(_Impl::_S_equal_to(__x._M_data, __y._M_data)); } 5216 5217 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type 5218 operator!=(const simd& __x, const simd& __y) 5219 { 5220 return simd::_S_make_mask( 5221 _Impl::_S_not_equal_to(__x._M_data, __y._M_data)); 5222 } 5223 5224 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type 5225 operator<(const simd& __x, const simd& __y) 5226 { return simd::_S_make_mask(_Impl::_S_less(__x._M_data, __y._M_data)); } 5227 5228 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type 5229 operator<=(const simd& __x, const simd& __y) 5230 { 5231 return simd::_S_make_mask(_Impl::_S_less_equal(__x._M_data, __y._M_data)); 5232 } 5233 5234 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type 5235 operator>(const simd& __x, const simd& __y) 5236 { return simd::_S_make_mask(_Impl::_S_less(__y._M_data, __x._M_data)); } 5237 5238 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend mask_type 5239 operator>=(const simd& __x, const simd& __y) 5240 { 5241 return simd::_S_make_mask(_Impl::_S_less_equal(__y._M_data, __x._M_data)); 5242 } 5243 5244 // operator?: overloads (suggested extension) {{{ 5245 #ifdef __GXX_CONDITIONAL_IS_OVERLOADABLE__ 5246 _GLIBCXX_SIMD_ALWAYS_INLINE _GLIBCXX_SIMD_CONSTEXPR friend simd 5247 operator?:(const mask_type& __k, const simd& __where_true, 5248 const simd& __where_false) 5249 { 5250 auto __ret = __where_false; 5251 _Impl::_S_masked_assign(__data(__k), __data(__ret), __data(__where_true)); 5252 return __ret; 5253 } 5254 5255 #endif // __GXX_CONDITIONAL_IS_OVERLOADABLE__ 5256 // }}} 5257 5258 // "private" because of the first arguments's namespace 5259 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR 5260 simd(_PrivateInit, const _MemberType& __init) 5261 : _M_data(__init) {} 5262 5263 // "private" because of the first arguments's namespace 5264 _GLIBCXX_SIMD_INTRINSIC 5265 simd(_BitsetInit, bitset<size()> __init) : _M_data() 5266 { where(mask_type(__bitset_init, __init), *this) = ~*this; } 5267 5268 _GLIBCXX_SIMD_INTRINSIC constexpr bool 5269 _M_is_constprop() const 5270 { 5271 if constexpr (__is_scalar_abi<_Abi>()) 5272 return __builtin_constant_p(_M_data); 5273 else 5274 return _M_data._M_is_constprop(); 5275 } 5276 5277 private: 5278 _GLIBCXX_SIMD_INTRINSIC static constexpr mask_type 5279 _S_make_mask(typename mask_type::_MemberType __k) 5280 { return {__private_init, __k}; } 5281 5282 friend const auto& __data<value_type, abi_type>(const simd&); 5283 friend auto& __data<value_type, abi_type>(simd&); 5284 alignas(_Traits::_S_simd_align) _MemberType _M_data; 5285 }; 5286 5287 // }}} 5288 /// @cond undocumented 5289 // __data {{{ 5290 template <typename _Tp, typename _Ap> 5291 _GLIBCXX_SIMD_INTRINSIC constexpr const auto& 5292 __data(const simd<_Tp, _Ap>& __x) 5293 { return __x._M_data; } 5294 5295 template <typename _Tp, typename _Ap> 5296 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 5297 __data(simd<_Tp, _Ap>& __x) 5298 { return __x._M_data; } 5299 5300 // }}} 5301 namespace __float_bitwise_operators { //{{{ 5302 template <typename _Tp, typename _Ap> 5303 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap> 5304 operator^(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b) 5305 { return {__private_init, _Ap::_SimdImpl::_S_bit_xor(__data(__a), __data(__b))}; } 5306 5307 template <typename _Tp, typename _Ap> 5308 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap> 5309 operator|(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b) 5310 { return {__private_init, _Ap::_SimdImpl::_S_bit_or(__data(__a), __data(__b))}; } 5311 5312 template <typename _Tp, typename _Ap> 5313 _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_SIMD_CONSTEXPR simd<_Tp, _Ap> 5314 operator&(const simd<_Tp, _Ap>& __a, const simd<_Tp, _Ap>& __b) 5315 { return {__private_init, _Ap::_SimdImpl::_S_bit_and(__data(__a), __data(__b))}; } 5316 } // namespace __float_bitwise_operators }}} 5317 /// @endcond 5318 5319 /// @} 5320 _GLIBCXX_SIMD_END_NAMESPACE 5321 5322 #endif // __cplusplus >= 201703L 5323 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_H 5324 5325 // vim: foldmethod=marker foldmarker={{{,}}}
Contact us
|
About us
|
Term of use
|
Copyright © 2000-2026 MyWebUniversity.com ™
