Where Online Learning is simpler!
The C and C++ Include Header Files
/usr/include/c++/13/experimental/bits/simd_fixed_size.h
$ cat -n /usr/include/c++/13/experimental/bits/simd_fixed_size.h 1 // Simd fixed_size ABI specific implementations -*- C++ -*- 2 3 // Copyright (C) 2020-2023 Free Software Foundation, Inc. 4 // 5 // This file is part of the GNU ISO C++ Library. This library is free 6 // software; you can redistribute it and/or modify it under the 7 // terms of the GNU General Public License as published by the 8 // Free Software Foundation; either version 3, or (at your option) 9 // any later version. 10 11 // This library is distributed in the hope that it will be useful, 12 // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 // GNU General Public License for more details. 15 16 // Under Section 7 of GPL version 3, you are granted additional 17 // permissions described in the GCC Runtime Library Exception, version 18 // 3.1, as published by the Free Software Foundation. 19 20 // You should have received a copy of the GNU General Public License and 21 // a copy of the GCC Runtime Library Exception along with this program; 22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 // <http://www.gnu.org/licenses/>. 24 25 /* 26 * The fixed_size ABI gives the following guarantees: 27 * - simd objects are passed via the stack 28 * - memory layout of `simd<_Tp, _Np>` is equivalent to `array<_Tp, _Np>` 29 * - alignment of `simd<_Tp, _Np>` is `_Np * sizeof(_Tp)` if _Np is __a 30 * power-of-2 value, otherwise `std::__bit_ceil(_Np * sizeof(_Tp))` (Note: 31 * if the alignment were to exceed the system/compiler maximum, it is bounded 32 * to that maximum) 33 * - simd_mask objects are passed like bitset<_Np> 34 * - memory layout of `simd_mask<_Tp, _Np>` is equivalent to `bitset<_Np>` 35 * - alignment of `simd_mask<_Tp, _Np>` is equal to the alignment of 36 * `bitset<_Np>` 37 */ 38 39 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_ 40 #define _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_ 41 42 #if __cplusplus >= 201703L 43 44 #include <array> 45 46 _GLIBCXX_SIMD_BEGIN_NAMESPACE 47 48 // __simd_tuple_element {{{ 49 template <size_t _I, typename _Tp> 50 struct __simd_tuple_element; 51 52 template <typename _Tp, typename _A0, typename... _As> 53 struct __simd_tuple_element<0, _SimdTuple<_Tp, _A0, _As...>> 54 { using type = simd<_Tp, _A0>; }; 55 56 template <size_t _I, typename _Tp, typename _A0, typename... _As> 57 struct __simd_tuple_element<_I, _SimdTuple<_Tp, _A0, _As...>> 58 { using type = typename __simd_tuple_element<_I - 1, _SimdTuple<_Tp, _As...>>::type; }; 59 60 template <size_t _I, typename _Tp> 61 using __simd_tuple_element_t = typename __simd_tuple_element<_I, _Tp>::type; 62 63 // }}} 64 // __simd_tuple_concat {{{ 65 66 template <typename _Tp, typename... _A0s, typename... _A1s> 67 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0s..., _A1s...> 68 __simd_tuple_concat(const _SimdTuple<_Tp, _A0s...>& __left, 69 const _SimdTuple<_Tp, _A1s...>& __right) 70 { 71 if constexpr (sizeof...(_A0s) == 0) 72 return __right; 73 else if constexpr (sizeof...(_A1s) == 0) 74 return __left; 75 else 76 return {__left.first, __simd_tuple_concat(__left.second, __right)}; 77 } 78 79 template <typename _Tp, typename _A10, typename... _A1s> 80 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, simd_abi::scalar, _A10, _A1s...> 81 __simd_tuple_concat(const _Tp& __left, const _SimdTuple<_Tp, _A10, _A1s...>& __right) 82 { return {__left, __right}; } 83 84 // }}} 85 // __simd_tuple_pop_front {{{ 86 // Returns the next _SimdTuple in __x that has _Np elements less. 87 // Precondition: _Np must match the number of elements in __first (recursively) 88 template <size_t _Np, typename _Tp> 89 _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto) 90 __simd_tuple_pop_front(_Tp&& __x) 91 { 92 if constexpr (_Np == 0) 93 return static_cast<_Tp&&>(__x); 94 else 95 { 96 using _Up = __remove_cvref_t<_Tp>; 97 static_assert(_Np >= _Up::_S_first_size); 98 return __simd_tuple_pop_front<_Np - _Up::_S_first_size>(__x.second); 99 } 100 } 101 102 // }}} 103 // __get_simd_at<_Np> {{{1 104 struct __as_simd {}; 105 106 struct __as_simd_tuple {}; 107 108 template <typename _Tp, typename _A0, typename... _Abis> 109 _GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A0> 110 __simd_tuple_get_impl(__as_simd, const _SimdTuple<_Tp, _A0, _Abis...>& __t, _SizeConstant<0>) 111 { return {__private_init, __t.first}; } 112 113 template <typename _Tp, typename _A0, typename... _Abis> 114 _GLIBCXX_SIMD_INTRINSIC constexpr const auto& 115 __simd_tuple_get_impl(__as_simd_tuple, const _SimdTuple<_Tp, _A0, _Abis...>& __t, 116 _SizeConstant<0>) 117 { return __t.first; } 118 119 template <typename _Tp, typename _A0, typename... _Abis> 120 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 121 __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _A0, _Abis...>& __t, _SizeConstant<0>) 122 { return __t.first; } 123 124 template <typename _R, size_t _Np, typename _Tp, typename... _Abis> 125 _GLIBCXX_SIMD_INTRINSIC constexpr auto 126 __simd_tuple_get_impl(_R, const _SimdTuple<_Tp, _Abis...>& __t, _SizeConstant<_Np>) 127 { return __simd_tuple_get_impl(_R(), __t.second, _SizeConstant<_Np - 1>()); } 128 129 template <size_t _Np, typename _Tp, typename... _Abis> 130 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 131 __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _Abis...>& __t, _SizeConstant<_Np>) 132 { return __simd_tuple_get_impl(__as_simd_tuple(), __t.second, _SizeConstant<_Np - 1>()); } 133 134 template <size_t _Np, typename _Tp, typename... _Abis> 135 _GLIBCXX_SIMD_INTRINSIC constexpr auto 136 __get_simd_at(const _SimdTuple<_Tp, _Abis...>& __t) 137 { return __simd_tuple_get_impl(__as_simd(), __t, _SizeConstant<_Np>()); } 138 139 // }}} 140 // __get_tuple_at<_Np> {{{ 141 template <size_t _Np, typename _Tp, typename... _Abis> 142 _GLIBCXX_SIMD_INTRINSIC constexpr auto 143 __get_tuple_at(const _SimdTuple<_Tp, _Abis...>& __t) 144 { return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>()); } 145 146 template <size_t _Np, typename _Tp, typename... _Abis> 147 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 148 __get_tuple_at(_SimdTuple<_Tp, _Abis...>& __t) 149 { return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>()); } 150 151 // __tuple_element_meta {{{1 152 template <typename _Tp, typename _Abi, size_t _Offset> 153 struct __tuple_element_meta : public _Abi::_SimdImpl 154 { 155 static_assert(is_same_v<typename _Abi::_SimdImpl::abi_type, 156 _Abi>); // this fails e.g. when _SimdImpl is an 157 // alias for _SimdImplBuiltin<_DifferentAbi> 158 using value_type = _Tp; 159 using abi_type = _Abi; 160 using _Traits = _SimdTraits<_Tp, _Abi>; 161 using _MaskImpl = typename _Abi::_MaskImpl; 162 using _MaskMember = typename _Traits::_MaskMember; 163 using simd_type = simd<_Tp, _Abi>; 164 static constexpr size_t _S_offset = _Offset; 165 static constexpr size_t _S_size() { return simd_size<_Tp, _Abi>::value; } 166 static constexpr _MaskImpl _S_mask_impl = {}; 167 168 template <size_t _Np, bool _Sanitized> 169 _GLIBCXX_SIMD_INTRINSIC static constexpr auto 170 _S_submask(_BitMask<_Np, _Sanitized> __bits) 171 { return __bits.template _M_extract<_Offset, _S_size()>(); } 172 173 template <size_t _Np, bool _Sanitized> 174 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 175 _S_make_mask(_BitMask<_Np, _Sanitized> __bits) 176 { 177 return _MaskImpl::template _S_convert<_Tp>( 178 __bits.template _M_extract<_Offset, _S_size()>()._M_sanitized()); 179 } 180 181 _GLIBCXX_SIMD_INTRINSIC static constexpr _ULLong 182 _S_mask_to_shifted_ullong(_MaskMember __k) 183 { return _MaskImpl::_S_to_bits(__k).to_ullong() << _Offset; } 184 }; 185 186 template <size_t _Offset, typename _Tp, typename _Abi, typename... _As> 187 _GLIBCXX_SIMD_INTRINSIC constexpr 188 __tuple_element_meta<_Tp, _Abi, _Offset> 189 __make_meta(const _SimdTuple<_Tp, _Abi, _As...>&) 190 { return {}; } 191 192 // }}}1 193 // _WithOffset wrapper class {{{ 194 template <size_t _Offset, typename _Base> 195 struct _WithOffset : public _Base 196 { 197 static inline constexpr size_t _S_offset = _Offset; 198 199 _GLIBCXX_SIMD_INTRINSIC char* 200 _M_as_charptr() 201 { return reinterpret_cast<char*>(this) + _S_offset * sizeof(typename _Base::value_type); } 202 203 _GLIBCXX_SIMD_INTRINSIC const char* 204 _M_as_charptr() const 205 { return reinterpret_cast<const char*>(this) + _S_offset * sizeof(typename _Base::value_type); } 206 }; 207 208 // make _WithOffset<_WithOffset> ill-formed to use: 209 template <size_t _O0, size_t _O1, typename _Base> 210 struct _WithOffset<_O0, _WithOffset<_O1, _Base>> {}; 211 212 template <size_t _Offset, typename _Tp> 213 _GLIBCXX_SIMD_INTRINSIC 214 decltype(auto) 215 __add_offset(_Tp& __base) 216 { return static_cast<_WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(__base); } 217 218 template <size_t _Offset, typename _Tp> 219 _GLIBCXX_SIMD_INTRINSIC 220 decltype(auto) 221 __add_offset(const _Tp& __base) 222 { return static_cast<const _WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(__base); } 223 224 template <size_t _Offset, size_t _ExistingOffset, typename _Tp> 225 _GLIBCXX_SIMD_INTRINSIC 226 decltype(auto) 227 __add_offset(_WithOffset<_ExistingOffset, _Tp>& __base) 228 { return static_cast<_WithOffset<_Offset + _ExistingOffset, _Tp>&>(static_cast<_Tp&>(__base)); } 229 230 template <size_t _Offset, size_t _ExistingOffset, typename _Tp> 231 _GLIBCXX_SIMD_INTRINSIC 232 decltype(auto) 233 __add_offset(const _WithOffset<_ExistingOffset, _Tp>& __base) 234 { 235 return static_cast<const _WithOffset<_Offset + _ExistingOffset, _Tp>&>( 236 static_cast<const _Tp&>(__base)); 237 } 238 239 template <typename _Tp> 240 constexpr inline size_t __offset = 0; 241 242 template <size_t _Offset, typename _Tp> 243 constexpr inline size_t __offset<_WithOffset<_Offset, _Tp>> 244 = _WithOffset<_Offset, _Tp>::_S_offset; 245 246 template <typename _Tp> 247 constexpr inline size_t __offset<const _Tp> = __offset<_Tp>; 248 249 template <typename _Tp> 250 constexpr inline size_t __offset<_Tp&> = __offset<_Tp>; 251 252 template <typename _Tp> 253 constexpr inline size_t __offset<_Tp&&> = __offset<_Tp>; 254 255 // }}} 256 // _SimdTuple specializations {{{1 257 // empty {{{2 258 template <typename _Tp> 259 struct _SimdTuple<_Tp> 260 { 261 using value_type = _Tp; 262 static constexpr size_t _S_tuple_size = 0; 263 static constexpr size_t _S_size() { return 0; } 264 }; 265 266 // _SimdTupleData {{{2 267 template <typename _FirstType, typename _SecondType> 268 struct _SimdTupleData 269 { 270 _FirstType first; 271 _SecondType second; 272 273 _GLIBCXX_SIMD_INTRINSIC 274 constexpr bool 275 _M_is_constprop() const 276 { 277 if constexpr (is_class_v<_FirstType>) 278 return first._M_is_constprop() && second._M_is_constprop(); 279 else 280 return __builtin_constant_p(first) && second._M_is_constprop(); 281 } 282 }; 283 284 template <typename _FirstType, typename _Tp> 285 struct _SimdTupleData<_FirstType, _SimdTuple<_Tp>> 286 { 287 _FirstType first; 288 static constexpr _SimdTuple<_Tp> second = {}; 289 290 _GLIBCXX_SIMD_INTRINSIC 291 constexpr bool 292 _M_is_constprop() const 293 { 294 if constexpr (is_class_v<_FirstType>) 295 return first._M_is_constprop(); 296 else 297 return __builtin_constant_p(first); 298 } 299 }; 300 301 // 1 or more {{{2 302 template <typename _Tp, typename _Abi0, typename... _Abis> 303 struct _SimdTuple<_Tp, _Abi0, _Abis...> 304 : _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember, 305 _SimdTuple<_Tp, _Abis...>> 306 { 307 static_assert(!__is_fixed_size_abi_v<_Abi0>); 308 using value_type = _Tp; 309 using _FirstType = typename _SimdTraits<_Tp, _Abi0>::_SimdMember; 310 using _FirstAbi = _Abi0; 311 using _SecondType = _SimdTuple<_Tp, _Abis...>; 312 static constexpr size_t _S_tuple_size = sizeof...(_Abis) + 1; 313 314 static constexpr size_t _S_size() 315 { return simd_size_v<_Tp, _Abi0> + _SecondType::_S_size(); } 316 317 static constexpr size_t _S_first_size = simd_size_v<_Tp, _Abi0>; 318 static constexpr bool _S_is_homogeneous = (is_same_v<_Abi0, _Abis> && ...); 319 320 using _Base = _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember, 321 _SimdTuple<_Tp, _Abis...>>; 322 using _Base::first; 323 using _Base::second; 324 325 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple() = default; 326 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(const _SimdTuple&) = default; 327 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple& operator=(const _SimdTuple&) 328 = default; 329 330 template <typename _Up> 331 _GLIBCXX_SIMD_INTRINSIC constexpr 332 _SimdTuple(_Up&& __x) 333 : _Base{static_cast<_Up&&>(__x)} {} 334 335 template <typename _Up, typename _Up2> 336 _GLIBCXX_SIMD_INTRINSIC constexpr 337 _SimdTuple(_Up&& __x, _Up2&& __y) 338 : _Base{static_cast<_Up&&>(__x), static_cast<_Up2&&>(__y)} {} 339 340 template <typename _Up> 341 _GLIBCXX_SIMD_INTRINSIC constexpr 342 _SimdTuple(_Up&& __x, _SimdTuple<_Tp>) 343 : _Base{static_cast<_Up&&>(__x)} {} 344 345 _GLIBCXX_SIMD_INTRINSIC char* 346 _M_as_charptr() 347 { return reinterpret_cast<char*>(this); } 348 349 _GLIBCXX_SIMD_INTRINSIC const char* 350 _M_as_charptr() const 351 { return reinterpret_cast<const char*>(this); } 352 353 template <size_t _Np> 354 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 355 _M_at() 356 { 357 if constexpr (_Np == 0) 358 return first; 359 else 360 return second.template _M_at<_Np - 1>(); 361 } 362 363 template <size_t _Np> 364 _GLIBCXX_SIMD_INTRINSIC constexpr const auto& 365 _M_at() const 366 { 367 if constexpr (_Np == 0) 368 return first; 369 else 370 return second.template _M_at<_Np - 1>(); 371 } 372 373 template <size_t _Np> 374 _GLIBCXX_SIMD_INTRINSIC constexpr auto 375 _M_simd_at() const 376 { 377 if constexpr (_Np == 0) 378 return simd<_Tp, _Abi0>(__private_init, first); 379 else 380 return second.template _M_simd_at<_Np - 1>(); 381 } 382 383 template <size_t _Offset = 0, typename _Fp> 384 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple 385 _S_generate(_Fp&& __gen, _SizeConstant<_Offset> = {}) 386 { 387 auto&& __first = __gen(__tuple_element_meta<_Tp, _Abi0, _Offset>()); 388 if constexpr (_S_tuple_size == 1) 389 return {__first}; 390 else 391 return {__first, 392 _SecondType::_S_generate( 393 static_cast<_Fp&&>(__gen), 394 _SizeConstant<_Offset + simd_size_v<_Tp, _Abi0>>())}; 395 } 396 397 template <size_t _Offset = 0, typename _Fp, typename... _More> 398 _GLIBCXX_SIMD_INTRINSIC _SimdTuple 399 _M_apply_wrapped(_Fp&& __fun, const _More&... __more) const 400 { 401 auto&& __first 402 = __fun(__make_meta<_Offset>(*this), first, __more.first...); 403 if constexpr (_S_tuple_size == 1) 404 return {__first}; 405 else 406 return { 407 __first, 408 second.template _M_apply_wrapped<_Offset + simd_size_v<_Tp, _Abi0>>( 409 static_cast<_Fp&&>(__fun), __more.second...)}; 410 } 411 412 template <typename _Tup> 413 _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto) 414 _M_extract_argument(_Tup&& __tup) const 415 { 416 using _TupT = typename __remove_cvref_t<_Tup>::value_type; 417 if constexpr (is_same_v<_SimdTuple, __remove_cvref_t<_Tup>>) 418 return __tup.first; 419 else if (__builtin_is_constant_evaluated()) 420 return __fixed_size_storage_t<_TupT, _S_first_size>::_S_generate( 421 [&](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 422 return __meta._S_generator( 423 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 424 return __tup[__i]; 425 }, static_cast<_TupT*>(nullptr)); 426 }); 427 else 428 return [&]() { // not always_inline; allow the compiler to decide 429 __fixed_size_storage_t<_TupT, _S_first_size> __r; 430 __builtin_memcpy(__r._M_as_charptr(), __tup._M_as_charptr(), 431 sizeof(__r)); 432 return __r; 433 }(); 434 } 435 436 template <typename _Tup> 437 _GLIBCXX_SIMD_INTRINSIC constexpr auto& 438 _M_skip_argument(_Tup&& __tup) const 439 { 440 static_assert(_S_tuple_size > 1); 441 using _Up = __remove_cvref_t<_Tup>; 442 constexpr size_t __off = __offset<_Up>; 443 if constexpr (_S_first_size == _Up::_S_first_size && __off == 0) 444 return __tup.second; 445 else if constexpr (_S_first_size > _Up::_S_first_size 446 && _S_first_size % _Up::_S_first_size == 0 447 && __off == 0) 448 return __simd_tuple_pop_front<_S_first_size>(__tup); 449 else if constexpr (_S_first_size + __off < _Up::_S_first_size) 450 return __add_offset<_S_first_size>(__tup); 451 else if constexpr (_S_first_size + __off == _Up::_S_first_size) 452 return __tup.second; 453 else 454 __assert_unreachable<_Tup>(); 455 } 456 457 template <size_t _Offset, typename... _More> 458 _GLIBCXX_SIMD_INTRINSIC constexpr void 459 _M_assign_front(const _SimdTuple<_Tp, _Abi0, _More...>& __x) & 460 { 461 static_assert(_Offset == 0); 462 first = __x.first; 463 if constexpr (sizeof...(_More) > 0) 464 { 465 static_assert(sizeof...(_Abis) >= sizeof...(_More)); 466 second.template _M_assign_front<0>(__x.second); 467 } 468 } 469 470 template <size_t _Offset> 471 _GLIBCXX_SIMD_INTRINSIC constexpr void 472 _M_assign_front(const _FirstType& __x) & 473 { 474 static_assert(_Offset == 0); 475 first = __x; 476 } 477 478 template <size_t _Offset, typename... _As> 479 _GLIBCXX_SIMD_INTRINSIC constexpr void 480 _M_assign_front(const _SimdTuple<_Tp, _As...>& __x) & 481 { 482 __builtin_memcpy(_M_as_charptr() + _Offset * sizeof(value_type), 483 __x._M_as_charptr(), 484 sizeof(_Tp) * _SimdTuple<_Tp, _As...>::_S_size()); 485 } 486 487 /* 488 * Iterate over the first objects in this _SimdTuple and call __fun for each 489 * of them. If additional arguments are passed via __more, chunk them into 490 * _SimdTuple or __vector_type_t objects of the same number of values. 491 */ 492 template <typename _Fp, typename... _More> 493 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple 494 _M_apply_per_chunk(_Fp&& __fun, _More&&... __more) const 495 { 496 if constexpr ((... 497 || conjunction_v< 498 is_lvalue_reference<_More>, 499 negation<is_const<remove_reference_t<_More>>>>) ) 500 { 501 // need to write back at least one of __more after calling __fun 502 auto&& __first = [&](auto... __args) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 503 auto __r = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first, 504 __args...); 505 [[maybe_unused]] auto&& __ignore_me = {( 506 [](auto&& __dst, const auto& __src) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 507 if constexpr (is_assignable_v<decltype(__dst), 508 decltype(__dst)>) 509 { 510 __dst.template _M_assign_front<__offset<decltype(__dst)>>( 511 __src); 512 } 513 }(static_cast<_More&&>(__more), __args), 514 0)...}; 515 return __r; 516 }(_M_extract_argument(__more)...); 517 if constexpr (_S_tuple_size == 1) 518 return {__first}; 519 else 520 return {__first, 521 second._M_apply_per_chunk(static_cast<_Fp&&>(__fun), 522 _M_skip_argument(__more)...)}; 523 } 524 else 525 { 526 auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first, 527 _M_extract_argument(__more)...); 528 if constexpr (_S_tuple_size == 1) 529 return {__first}; 530 else 531 return {__first, 532 second._M_apply_per_chunk(static_cast<_Fp&&>(__fun), 533 _M_skip_argument(__more)...)}; 534 } 535 } 536 537 template <typename _R = _Tp, typename _Fp, typename... _More> 538 _GLIBCXX_SIMD_INTRINSIC constexpr auto 539 _M_apply_r(_Fp&& __fun, const _More&... __more) const 540 { 541 auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first, 542 __more.first...); 543 if constexpr (_S_tuple_size == 1) 544 return __first; 545 else 546 return __simd_tuple_concat<_R>( 547 __first, second.template _M_apply_r<_R>(static_cast<_Fp&&>(__fun), 548 __more.second...)); 549 } 550 551 template <typename _Fp, typename... _More> 552 _GLIBCXX_SIMD_INTRINSIC constexpr friend _SanitizedBitMask<_S_size()> 553 _M_test(const _Fp& __fun, const _SimdTuple& __x, const _More&... __more) 554 { 555 const _SanitizedBitMask<_S_first_size> __first 556 = _Abi0::_MaskImpl::_S_to_bits( 557 __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first, 558 __more.first...)); 559 if constexpr (_S_tuple_size == 1) 560 return __first; 561 else 562 return _M_test(__fun, __x.second, __more.second...) 563 ._M_prepend(__first); 564 } 565 566 template <typename _Up, _Up _I> 567 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp 568 operator[](integral_constant<_Up, _I>) const noexcept 569 { 570 if constexpr (_I < simd_size_v<_Tp, _Abi0>) 571 return _M_subscript_read(_I); 572 else 573 return second[integral_constant<_Up, _I - simd_size_v<_Tp, _Abi0>>()]; 574 } 575 576 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp 577 operator[](size_t __i) const noexcept 578 { 579 if constexpr (_S_tuple_size == 1) 580 return _M_subscript_read(__i); 581 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS 582 else if (not __builtin_is_constant_evaluated()) 583 return reinterpret_cast<const __may_alias<_Tp>*>(this)[__i]; 584 #endif 585 else if constexpr (__is_scalar_abi<_Abi0>()) 586 { 587 const _Tp* ptr = &first; 588 return ptr[__i]; 589 } 590 else 591 return __i < simd_size_v<_Tp, _Abi0> ? _M_subscript_read(__i) 592 : second[__i - simd_size_v<_Tp, _Abi0>]; 593 } 594 595 _GLIBCXX_SIMD_INTRINSIC constexpr void 596 _M_set(size_t __i, _Tp __val) noexcept 597 { 598 if constexpr (_S_tuple_size == 1) 599 return _M_subscript_write(__i, __val); 600 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS 601 else if (not __builtin_is_constant_evaluated()) 602 reinterpret_cast<__may_alias<_Tp>*>(this)[__i] = __val; 603 #endif 604 else if (__i < simd_size_v<_Tp, _Abi0>) 605 _M_subscript_write(__i, __val); 606 else 607 second._M_set(__i - simd_size_v<_Tp, _Abi0>, __val); 608 } 609 610 private: 611 // _M_subscript_read/_write {{{ 612 _GLIBCXX_SIMD_INTRINSIC constexpr _Tp 613 _M_subscript_read([[maybe_unused]] size_t __i) const noexcept 614 { 615 if constexpr (__is_vectorizable_v<_FirstType>) 616 return first; 617 else 618 return first[__i]; 619 } 620 621 _GLIBCXX_SIMD_INTRINSIC constexpr void 622 _M_subscript_write([[maybe_unused]] size_t __i, _Tp __y) noexcept 623 { 624 if constexpr (__is_vectorizable_v<_FirstType>) 625 first = __y; 626 else 627 first._M_set(__i, __y); 628 } 629 630 // }}} 631 }; 632 633 // __make_simd_tuple {{{1 634 template <typename _Tp, typename _A0> 635 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0> 636 __make_simd_tuple(simd<_Tp, _A0> __x0) 637 { return {__data(__x0)}; } 638 639 template <typename _Tp, typename _A0, typename... _As> 640 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0, _As...> 641 __make_simd_tuple(const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs) 642 { return {__data(__x0), __make_simd_tuple(__xs...)}; } 643 644 template <typename _Tp, typename _A0> 645 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0> 646 __make_simd_tuple(const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0) 647 { return {__arg0}; } 648 649 template <typename _Tp, typename _A0, typename _A1, typename... _Abis> 650 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0, _A1, _Abis...> 651 __make_simd_tuple( 652 const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0, 653 const typename _SimdTraits<_Tp, _A1>::_SimdMember& __arg1, 654 const typename _SimdTraits<_Tp, _Abis>::_SimdMember&... __args) 655 { return {__arg0, __make_simd_tuple<_Tp, _A1, _Abis...>(__arg1, __args...)}; } 656 657 // __to_simd_tuple {{{1 658 template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX> 659 _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np> 660 __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX); 661 662 template <typename _Tp, size_t _Np, 663 size_t _Offset = 0, // skip this many elements in __from0 664 typename _R = __fixed_size_storage_t<_Tp, _Np>, typename _V0, 665 typename _V0VT = _VectorTraits<_V0>, typename... _VX> 666 _GLIBCXX_SIMD_INTRINSIC _R constexpr __to_simd_tuple(const _V0 __from0, const _VX... __fromX) 667 { 668 static_assert(is_same_v<typename _V0VT::value_type, _Tp>); 669 static_assert(_Offset < _V0VT::_S_full_size); 670 using _R0 = __vector_type_t<_Tp, _R::_S_first_size>; 671 if constexpr (_R::_S_tuple_size == 1) 672 { 673 if constexpr (_Np == 1) 674 return _R{__from0[_Offset]}; 675 else if constexpr (_Offset == 0 && _V0VT::_S_full_size >= _Np) 676 return _R{__intrin_bitcast<_R0>(__from0)}; 677 else if constexpr (_Offset * 2 == _V0VT::_S_full_size 678 && _V0VT::_S_full_size / 2 >= _Np) 679 return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0))}; 680 else if constexpr (_Offset * 4 == _V0VT::_S_full_size 681 && _V0VT::_S_full_size / 4 >= _Np) 682 return _R{__intrin_bitcast<_R0>(__extract_part<1, 4>(__from0))}; 683 else 684 __assert_unreachable<_Tp>(); 685 } 686 else 687 { 688 if constexpr (1 == _R::_S_first_size) 689 { // extract one scalar and recurse 690 if constexpr (_Offset + 1 < _V0VT::_S_full_size) 691 return _R{__from0[_Offset], 692 __to_simd_tuple<_Tp, _Np - 1, _Offset + 1>(__from0, 693 __fromX...)}; 694 else 695 return _R{__from0[_Offset], 696 __to_simd_tuple<_Tp, _Np - 1, 0>(__fromX...)}; 697 } 698 699 // place __from0 into _R::first and recurse for __fromX -> _R::second 700 else if constexpr (_V0VT::_S_full_size == _R::_S_first_size 701 && _Offset == 0) 702 return _R{__from0, 703 __to_simd_tuple<_Tp, _Np - _R::_S_first_size>(__fromX...)}; 704 705 // place lower part of __from0 into _R::first and recurse with _Offset 706 else if constexpr (_V0VT::_S_full_size > _R::_S_first_size 707 && _Offset == 0) 708 return _R{__intrin_bitcast<_R0>(__from0), 709 __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 710 _R::_S_first_size>(__from0, __fromX...)}; 711 712 // place lower part of second quarter of __from0 into _R::first and 713 // recurse with _Offset 714 else if constexpr (_Offset * 4 == _V0VT::_S_full_size 715 && _V0VT::_S_full_size >= 4 * _R::_S_first_size) 716 return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)), 717 __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 718 _Offset + _R::_S_first_size>(__from0, 719 __fromX...)}; 720 721 // place lower half of high half of __from0 into _R::first and recurse 722 // with _Offset 723 else if constexpr (_Offset * 2 == _V0VT::_S_full_size 724 && _V0VT::_S_full_size >= 4 * _R::_S_first_size) 725 return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)), 726 __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 727 _Offset + _R::_S_first_size>(__from0, 728 __fromX...)}; 729 730 // place high half of __from0 into _R::first and recurse with __fromX 731 else if constexpr (_Offset * 2 == _V0VT::_S_full_size 732 && _V0VT::_S_full_size / 2 >= _R::_S_first_size) 733 return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0)), 734 __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 0>( 735 __fromX...)}; 736 737 // ill-formed if some unforseen pattern is needed 738 else 739 __assert_unreachable<_Tp>(); 740 } 741 } 742 743 template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX> 744 _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np> 745 __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX) 746 { 747 if constexpr (is_same_v<_Tp, _V>) 748 { 749 static_assert( 750 sizeof...(_VX) == 0, 751 "An array of scalars must be the last argument to __to_simd_tuple"); 752 return __call_with_subscripts( 753 __from, make_index_sequence<_NV>(), 754 [&](const auto... __args) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 755 return __simd_tuple_concat( 756 _SimdTuple<_Tp, simd_abi::scalar>{__args}..., _SimdTuple<_Tp>()); 757 }); 758 } 759 else 760 return __call_with_subscripts( 761 __from, make_index_sequence<_NV>(), 762 [&](const auto... __args) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 763 return __to_simd_tuple<_Tp, _Np>(__args..., __fromX...); 764 }); 765 } 766 767 template <size_t, typename _Tp> 768 using __to_tuple_helper = _Tp; 769 770 template <typename _Tp, typename _A0, size_t _NOut, size_t _Np, 771 size_t... _Indexes> 772 _GLIBCXX_SIMD_INTRINSIC __fixed_size_storage_t<_Tp, _NOut> 773 __to_simd_tuple_impl(index_sequence<_Indexes...>, 774 const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args) 775 { 776 return __make_simd_tuple<_Tp, __to_tuple_helper<_Indexes, _A0>...>( 777 __args[_Indexes]...); 778 } 779 780 template <typename _Tp, typename _A0, size_t _NOut, size_t _Np, 781 typename _R = __fixed_size_storage_t<_Tp, _NOut>> 782 _GLIBCXX_SIMD_INTRINSIC _R 783 __to_simd_tuple_sized( 784 const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args) 785 { 786 static_assert(_Np * simd_size_v<_Tp, _A0> >= _NOut); 787 return __to_simd_tuple_impl<_Tp, _A0, _NOut>( 788 make_index_sequence<_R::_S_tuple_size>(), __args); 789 } 790 791 // __optimize_simd_tuple {{{1 792 template <typename _Tp> 793 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp> 794 __optimize_simd_tuple(const _SimdTuple<_Tp>) 795 { return {}; } 796 797 template <typename _Tp, typename _Ap> 798 _GLIBCXX_SIMD_INTRINSIC constexpr const _SimdTuple<_Tp, _Ap>& 799 __optimize_simd_tuple(const _SimdTuple<_Tp, _Ap>& __x) 800 { return __x; } 801 802 template <typename _Tp, typename _A0, typename _A1, typename... _Abis, 803 typename _R = __fixed_size_storage_t< 804 _Tp, _SimdTuple<_Tp, _A0, _A1, _Abis...>::_S_size()>> 805 _GLIBCXX_SIMD_INTRINSIC constexpr _R 806 __optimize_simd_tuple(const _SimdTuple<_Tp, _A0, _A1, _Abis...>& __x) 807 { 808 using _Tup = _SimdTuple<_Tp, _A0, _A1, _Abis...>; 809 if constexpr (is_same_v<_R, _Tup>) 810 return __x; 811 else if constexpr (is_same_v<typename _R::_FirstType, 812 typename _Tup::_FirstType>) 813 return {__x.first, __optimize_simd_tuple(__x.second)}; 814 else if constexpr (__is_scalar_abi<_A0>() 815 || _A0::template _S_is_partial<_Tp>) 816 return {__generate_from_n_evaluations<_R::_S_first_size, 817 typename _R::_FirstType>( 818 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }), 819 __optimize_simd_tuple( 820 __simd_tuple_pop_front<_R::_S_first_size>(__x))}; 821 else if constexpr (is_same_v<_A0, _A1> 822 && _R::_S_first_size == simd_size_v<_Tp, _A0> + simd_size_v<_Tp, _A1>) 823 return {__concat(__x.template _M_at<0>(), __x.template _M_at<1>()), 824 __optimize_simd_tuple(__x.second.second)}; 825 else if constexpr (sizeof...(_Abis) >= 2 826 && _R::_S_first_size == (4 * simd_size_v<_Tp, _A0>) 827 && simd_size_v<_Tp, _A0> == __simd_tuple_element_t< 828 (sizeof...(_Abis) >= 2 ? 3 : 0), _Tup>::size()) 829 return { 830 __concat(__concat(__x.template _M_at<0>(), __x.template _M_at<1>()), 831 __concat(__x.template _M_at<2>(), __x.template _M_at<3>())), 832 __optimize_simd_tuple(__x.second.second.second.second)}; 833 else 834 { 835 static_assert(sizeof(_R) == sizeof(__x)); 836 _R __r; 837 __builtin_memcpy(__r._M_as_charptr(), __x._M_as_charptr(), 838 sizeof(_Tp) * _R::_S_size()); 839 return __r; 840 } 841 } 842 843 // __for_each(const _SimdTuple &, Fun) {{{1 844 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp> 845 _GLIBCXX_SIMD_INTRINSIC constexpr void 846 __for_each(const _SimdTuple<_Tp, _A0>& __t, _Fp&& __fun) 847 { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); } 848 849 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1, 850 typename... _As, typename _Fp> 851 _GLIBCXX_SIMD_INTRINSIC constexpr void 852 __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun) 853 { 854 __fun(__make_meta<_Offset>(__t), __t.first); 855 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second, 856 static_cast<_Fp&&>(__fun)); 857 } 858 859 // __for_each(_SimdTuple &, Fun) {{{1 860 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp> 861 _GLIBCXX_SIMD_INTRINSIC constexpr void 862 __for_each(_SimdTuple<_Tp, _A0>& __t, _Fp&& __fun) 863 { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); } 864 865 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1, 866 typename... _As, typename _Fp> 867 _GLIBCXX_SIMD_INTRINSIC constexpr void 868 __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun) 869 { 870 __fun(__make_meta<_Offset>(__t), __t.first); 871 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second, 872 static_cast<_Fp&&>(__fun)); 873 } 874 875 // __for_each(_SimdTuple &, const _SimdTuple &, Fun) {{{1 876 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp> 877 _GLIBCXX_SIMD_INTRINSIC constexpr void 878 __for_each(_SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b, _Fp&& __fun) 879 { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first); } 880 881 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1, 882 typename... _As, typename _Fp> 883 _GLIBCXX_SIMD_INTRINSIC constexpr void 884 __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __a, 885 const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun) 886 { 887 __fun(__make_meta<_Offset>(__a), __a.first, __b.first); 888 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second, 889 static_cast<_Fp&&>(__fun)); 890 } 891 892 // __for_each(const _SimdTuple &, const _SimdTuple &, Fun) {{{1 893 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp> 894 _GLIBCXX_SIMD_INTRINSIC constexpr void 895 __for_each(const _SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b, _Fp&& __fun) 896 { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first); } 897 898 template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1, 899 typename... _As, typename _Fp> 900 _GLIBCXX_SIMD_INTRINSIC constexpr void 901 __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __a, 902 const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun) 903 { 904 __fun(__make_meta<_Offset>(__a), __a.first, __b.first); 905 __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second, 906 static_cast<_Fp&&>(__fun)); 907 } 908 909 // }}}1 910 // __extract_part(_SimdTuple) {{{ 911 template <int _Index, int _Total, int _Combine, typename _Tp, typename _A0, typename... _As> 912 _GLIBCXX_SIMD_INTRINSIC constexpr auto // __vector_type_t or _SimdTuple 913 __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x) 914 { 915 // worst cases: 916 // (a) 4, 4, 4 => 3, 3, 3, 3 (_Total = 4) 917 // (b) 2, 2, 2 => 3, 3 (_Total = 2) 918 // (c) 4, 2 => 2, 2, 2 (_Total = 3) 919 using _Tuple = _SimdTuple<_Tp, _A0, _As...>; 920 static_assert(_Index + _Combine <= _Total && _Index >= 0 && _Total >= 1); 921 constexpr size_t _Np = _Tuple::_S_size(); 922 static_assert(_Np >= _Total && _Np % _Total == 0); 923 constexpr size_t __values_per_part = _Np / _Total; 924 [[maybe_unused]] constexpr size_t __values_to_skip 925 = _Index * __values_per_part; 926 constexpr size_t __return_size = __values_per_part * _Combine; 927 using _RetAbi = simd_abi::deduce_t<_Tp, __return_size>; 928 929 // handle (optimize) the simple cases 930 if constexpr (__return_size == 1) 931 return __x[integral_constant<size_t, __values_to_skip>()]; 932 else if constexpr (_Index == 0 && _Tuple::_S_first_size == __return_size) 933 return __x.first._M_data; 934 else if constexpr (_Index == 0 && _Total == _Combine) 935 return __x; 936 else if constexpr (_Index == 0 && _Tuple::_S_first_size >= __return_size) 937 return __intrin_bitcast<__vector_type_t<_Tp, __return_size>>( 938 __as_vector(__x.first)); 939 940 // recurse to skip unused data members at the beginning of _SimdTuple 941 else if constexpr (__values_to_skip >= _Tuple::_S_first_size) 942 { // recurse 943 if constexpr (_Tuple::_S_first_size % __values_per_part == 0) 944 { 945 constexpr int __parts_in_first 946 = _Tuple::_S_first_size / __values_per_part; 947 return __extract_part<_Index - __parts_in_first, 948 _Total - __parts_in_first, _Combine>( 949 __x.second); 950 } 951 else 952 return __extract_part<__values_to_skip - _Tuple::_S_first_size, 953 _Np - _Tuple::_S_first_size, __return_size>( 954 __x.second); 955 } 956 957 // extract from multiple _SimdTuple data members 958 else if constexpr (__return_size > _Tuple::_S_first_size - __values_to_skip) 959 { 960 #ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS 961 const __may_alias<_Tp>* const element_ptr 962 = reinterpret_cast<const __may_alias<_Tp>*>(&__x) + __values_to_skip; 963 return __as_vector(simd<_Tp, _RetAbi>(element_ptr, element_aligned)); 964 #else 965 [[maybe_unused]] constexpr size_t __offset = __values_to_skip; 966 return __as_vector(simd<_Tp, _RetAbi>( 967 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 968 constexpr _SizeConstant<__i + __offset> __k; 969 return __x[__k]; 970 })); 971 #endif 972 } 973 974 // all of the return values are in __x.first 975 else if constexpr (_Tuple::_S_first_size % __values_per_part == 0) 976 return __extract_part<_Index, _Tuple::_S_first_size / __values_per_part, 977 _Combine>(__x.first); 978 else 979 return __extract_part<__values_to_skip, _Tuple::_S_first_size, 980 _Combine * __values_per_part>(__x.first); 981 } 982 983 // }}} 984 // __fixed_size_storage_t<_Tp, _Np>{{{ 985 template <typename _Tp, int _Np, typename _Tuple, 986 typename _Next = simd<_Tp, _AllNativeAbis::_BestAbi<_Tp, _Np>>, 987 int _Remain = _Np - int(_Next::size())> 988 struct __fixed_size_storage_builder; 989 990 template <typename _Tp, int _Np> 991 struct __fixed_size_storage 992 : public __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp>> {}; 993 994 template <typename _Tp, int _Np, typename... _As, typename _Next> 995 struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next, 996 0> 997 { using type = _SimdTuple<_Tp, _As..., typename _Next::abi_type>; }; 998 999 template <typename _Tp, int _Np, typename... _As, typename _Next, int _Remain> 1000 struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next, 1001 _Remain> 1002 { 1003 using type = typename __fixed_size_storage_builder< 1004 _Tp, _Remain, _SimdTuple<_Tp, _As..., typename _Next::abi_type>>::type; 1005 }; 1006 1007 // }}} 1008 // __autocvt_to_simd {{{ 1009 template <typename _Tp, bool = is_arithmetic_v<__remove_cvref_t<_Tp>>> 1010 struct __autocvt_to_simd 1011 { 1012 _Tp _M_data; 1013 using _TT = __remove_cvref_t<_Tp>; 1014 1015 _GLIBCXX_SIMD_INTRINSIC constexpr 1016 operator _TT() 1017 { return _M_data; } 1018 1019 _GLIBCXX_SIMD_INTRINSIC constexpr 1020 operator _TT&() 1021 { 1022 static_assert(is_lvalue_reference<_Tp>::value, ""); 1023 static_assert(!is_const<_Tp>::value, ""); 1024 return _M_data; 1025 } 1026 1027 _GLIBCXX_SIMD_INTRINSIC constexpr 1028 operator _TT*() 1029 { 1030 static_assert(is_lvalue_reference<_Tp>::value, ""); 1031 static_assert(!is_const<_Tp>::value, ""); 1032 return &_M_data; 1033 } 1034 1035 _GLIBCXX_SIMD_INTRINSIC constexpr 1036 __autocvt_to_simd(_Tp dd) : _M_data(dd) {} 1037 1038 template <typename _Abi> 1039 _GLIBCXX_SIMD_INTRINSIC constexpr 1040 operator simd<typename _TT::value_type, _Abi>() 1041 { return {__private_init, _M_data}; } 1042 1043 template <typename _Abi> 1044 _GLIBCXX_SIMD_INTRINSIC constexpr 1045 operator simd<typename _TT::value_type, _Abi>&() 1046 { return *reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(&_M_data); } 1047 1048 template <typename _Abi> 1049 _GLIBCXX_SIMD_INTRINSIC constexpr 1050 operator simd<typename _TT::value_type, _Abi>*() 1051 { return reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(&_M_data); } 1052 }; 1053 1054 template <typename _Tp> 1055 __autocvt_to_simd(_Tp &&) -> __autocvt_to_simd<_Tp>; 1056 1057 template <typename _Tp> 1058 struct __autocvt_to_simd<_Tp, true> 1059 { 1060 using _TT = __remove_cvref_t<_Tp>; 1061 _Tp _M_data; 1062 fixed_size_simd<_TT, 1> _M_fd; 1063 1064 _GLIBCXX_SIMD_INTRINSIC 1065 constexpr __autocvt_to_simd(_Tp dd) : _M_data(dd), _M_fd(_M_data) {} 1066 1067 _GLIBCXX_SIMD_INTRINSIC 1068 ~__autocvt_to_simd() 1069 { _M_data = __data(_M_fd).first; } 1070 1071 _GLIBCXX_SIMD_INTRINSIC constexpr 1072 operator fixed_size_simd<_TT, 1>() 1073 { return _M_fd; } 1074 1075 _GLIBCXX_SIMD_INTRINSIC constexpr 1076 operator fixed_size_simd<_TT, 1> &() 1077 { 1078 static_assert(is_lvalue_reference<_Tp>::value, ""); 1079 static_assert(!is_const<_Tp>::value, ""); 1080 return _M_fd; 1081 } 1082 1083 _GLIBCXX_SIMD_INTRINSIC constexpr 1084 operator fixed_size_simd<_TT, 1> *() 1085 { 1086 static_assert(is_lvalue_reference<_Tp>::value, ""); 1087 static_assert(!is_const<_Tp>::value, ""); 1088 return &_M_fd; 1089 } 1090 }; 1091 1092 // }}} 1093 1094 struct _CommonImplFixedSize; 1095 template <int _Np, typename = __detail::__odr_helper> struct _SimdImplFixedSize; 1096 template <int _Np, typename = __detail::__odr_helper> struct _MaskImplFixedSize; 1097 // simd_abi::_Fixed {{{ 1098 template <int _Np> 1099 struct simd_abi::_Fixed 1100 { 1101 template <typename _Tp> static constexpr size_t _S_size = _Np; 1102 template <typename _Tp> static constexpr size_t _S_full_size = _Np; 1103 // validity traits {{{ 1104 struct _IsValidAbiTag : public __bool_constant<(_Np > 0)> {}; 1105 1106 template <typename _Tp> 1107 struct _IsValidSizeFor 1108 : __bool_constant<(_Np <= simd_abi::max_fixed_size<_Tp>)> {}; 1109 1110 template <typename _Tp> 1111 struct _IsValid : conjunction<_IsValidAbiTag, __is_vectorizable<_Tp>, 1112 _IsValidSizeFor<_Tp>> {}; 1113 1114 template <typename _Tp> 1115 static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value; 1116 1117 // }}} 1118 // _S_masked {{{ 1119 _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np> 1120 _S_masked(_BitMask<_Np> __x) 1121 { return __x._M_sanitized(); } 1122 1123 _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np> 1124 _S_masked(_SanitizedBitMask<_Np> __x) 1125 { return __x; } 1126 1127 // }}} 1128 // _*Impl {{{ 1129 using _CommonImpl = _CommonImplFixedSize; 1130 using _SimdImpl = _SimdImplFixedSize<_Np>; 1131 using _MaskImpl = _MaskImplFixedSize<_Np>; 1132 1133 // }}} 1134 // __traits {{{ 1135 template <typename _Tp, bool = _S_is_valid_v<_Tp>> 1136 struct __traits : _InvalidTraits {}; 1137 1138 template <typename _Tp> 1139 struct __traits<_Tp, true> 1140 { 1141 using _IsValid = true_type; 1142 using _SimdImpl = _SimdImplFixedSize<_Np>; 1143 using _MaskImpl = _MaskImplFixedSize<_Np>; 1144 1145 // simd and simd_mask member types {{{ 1146 using _SimdMember = __fixed_size_storage_t<_Tp, _Np>; 1147 using _MaskMember = _SanitizedBitMask<_Np>; 1148 1149 static constexpr size_t _S_simd_align 1150 = std::__bit_ceil(_Np * sizeof(_Tp)); 1151 1152 static constexpr size_t _S_mask_align = alignof(_MaskMember); 1153 1154 // }}} 1155 // _SimdBase / base class for simd, providing extra conversions {{{ 1156 struct _SimdBase 1157 { 1158 // The following ensures, function arguments are passed via the stack. 1159 // This is important for ABI compatibility across TU boundaries 1160 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr 1161 _SimdBase(const _SimdBase&) {} 1162 1163 _SimdBase() = default; 1164 1165 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit 1166 operator const _SimdMember &() const 1167 { return static_cast<const simd<_Tp, _Fixed>*>(this)->_M_data; } 1168 1169 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit 1170 operator array<_Tp, _Np>() const 1171 { 1172 array<_Tp, _Np> __r; 1173 // _SimdMember can be larger because of higher alignment 1174 static_assert(sizeof(__r) <= sizeof(_SimdMember), ""); 1175 __builtin_memcpy(__r.data(), &static_cast<const _SimdMember&>(*this), 1176 sizeof(__r)); 1177 return __r; 1178 } 1179 }; 1180 1181 // }}} 1182 // _MaskBase {{{ 1183 // empty. The bitset interface suffices 1184 struct _MaskBase {}; 1185 1186 // }}} 1187 // _SimdCastType {{{ 1188 struct _SimdCastType 1189 { 1190 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr 1191 _SimdCastType(const array<_Tp, _Np>&); 1192 1193 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr 1194 _SimdCastType(const _SimdMember& dd) : _M_data(dd) {} 1195 1196 _GLIBCXX_SIMD_ALWAYS_INLINE constexpr explicit 1197 operator const _SimdMember &() const { return _M_data; } 1198 1199 private: 1200 const _SimdMember& _M_data; 1201 }; 1202 1203 // }}} 1204 // _MaskCastType {{{ 1205 class _MaskCastType 1206 { 1207 _MaskCastType() = delete; 1208 }; 1209 // }}} 1210 }; 1211 // }}} 1212 }; 1213 1214 // }}} 1215 // _CommonImplFixedSize {{{ 1216 struct _CommonImplFixedSize 1217 { 1218 // _S_store {{{ 1219 template <typename _Tp, typename... _As> 1220 _GLIBCXX_SIMD_INTRINSIC static void 1221 _S_store(const _SimdTuple<_Tp, _As...>& __x, void* __addr) 1222 { 1223 constexpr size_t _Np = _SimdTuple<_Tp, _As...>::_S_size(); 1224 __builtin_memcpy(__addr, &__x, _Np * sizeof(_Tp)); 1225 } 1226 1227 // }}} 1228 }; 1229 1230 // }}} 1231 // _SimdImplFixedSize {{{1 1232 // fixed_size should not inherit from _SimdMathFallback in order for 1233 // specializations in the used _SimdTuple Abis to get used 1234 template <int _Np, typename> 1235 struct _SimdImplFixedSize 1236 { 1237 // member types {{{2 1238 using _MaskMember = _SanitizedBitMask<_Np>; 1239 1240 template <typename _Tp> 1241 using _SimdMember = __fixed_size_storage_t<_Tp, _Np>; 1242 1243 template <typename _Tp> 1244 static constexpr size_t _S_tuple_size = _SimdMember<_Tp>::_S_tuple_size; 1245 1246 template <typename _Tp> 1247 using _Simd = simd<_Tp, simd_abi::fixed_size<_Np>>; 1248 1249 template <typename _Tp> 1250 using _TypeTag = _Tp*; 1251 1252 // broadcast {{{2 1253 template <typename _Tp> 1254 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp> 1255 _S_broadcast(_Tp __x) noexcept 1256 { 1257 return _SimdMember<_Tp>::_S_generate( 1258 [&](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1259 return __meta._S_broadcast(__x); 1260 }); 1261 } 1262 1263 // _S_generator {{{2 1264 template <typename _Fp, typename _Tp> 1265 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp> 1266 _S_generator(_Fp&& __gen, _TypeTag<_Tp>) 1267 { 1268 return _SimdMember<_Tp>::_S_generate( 1269 [&__gen](auto __meta) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1270 return __meta._S_generator( 1271 [&](auto __i) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1272 return __i < _Np ? __gen(_SizeConstant<__meta._S_offset + __i>()) 1273 : 0; 1274 }, 1275 _TypeTag<_Tp>()); 1276 }); 1277 } 1278 1279 // _S_load {{{2 1280 template <typename _Tp, typename _Up> 1281 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdMember<_Tp> 1282 _S_load(const _Up* __mem, _TypeTag<_Tp>) noexcept 1283 { 1284 return _SimdMember<_Tp>::_S_generate( 1285 [&](auto __meta) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1286 return __meta._S_load(&__mem[__meta._S_offset], _TypeTag<_Tp>()); 1287 }); 1288 } 1289 1290 // _S_masked_load {{{2 1291 template <typename _Tp, typename... _As, typename _Up> 1292 _GLIBCXX_SIMD_INTRINSIC static _SimdTuple<_Tp, _As...> 1293 _S_masked_load(const _SimdTuple<_Tp, _As...>& __old, 1294 const _MaskMember __bits, const _Up* __mem) noexcept 1295 { 1296 auto __merge = __old; 1297 __for_each(__merge, [&](auto __meta, auto& __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1298 if (__meta._S_submask(__bits).any()) 1299 #pragma GCC diagnostic push 1300 // Dereferencing __mem + __meta._S_offset could be UB ([expr.add]/4.3). 1301 // It is the responsibility of the caller of the masked load (via the mask's value) to 1302 // avoid UB. Consequently, the compiler may assume this branch is unreachable, if the 1303 // pointer arithmetic is UB. 1304 #pragma GCC diagnostic ignored "-Warray-bounds" 1305 __native 1306 = __meta._S_masked_load(__native, __meta._S_make_mask(__bits), 1307 __mem + __meta._S_offset); 1308 #pragma GCC diagnostic pop 1309 }); 1310 return __merge; 1311 } 1312 1313 // _S_store {{{2 1314 template <typename _Tp, typename _Up> 1315 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1316 _S_store(const _SimdMember<_Tp>& __v, _Up* __mem, _TypeTag<_Tp>) noexcept 1317 { 1318 __for_each(__v, [&](auto __meta, auto __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1319 __meta._S_store(__native, &__mem[__meta._S_offset], _TypeTag<_Tp>()); 1320 }); 1321 } 1322 1323 // _S_masked_store {{{2 1324 template <typename _Tp, typename... _As, typename _Up> 1325 _GLIBCXX_SIMD_INTRINSIC static void 1326 _S_masked_store(const _SimdTuple<_Tp, _As...>& __v, _Up* __mem, 1327 const _MaskMember __bits) noexcept 1328 { 1329 __for_each(__v, [&](auto __meta, auto __native) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1330 if (__meta._S_submask(__bits).any()) 1331 #pragma GCC diagnostic push 1332 // __mem + __mem._S_offset could be UB ([expr.add]/4.3, but it punts 1333 // the responsibility for avoiding UB to the caller of the masked 1334 // store via the mask. Consequently, the compiler may assume this 1335 // branch is unreachable, if the pointer arithmetic is UB. 1336 #pragma GCC diagnostic ignored "-Warray-bounds" 1337 __meta._S_masked_store(__native, __mem + __meta._S_offset, 1338 __meta._S_make_mask(__bits)); 1339 #pragma GCC diagnostic pop 1340 }); 1341 } 1342 1343 // negation {{{2 1344 template <typename _Tp, typename... _As> 1345 static constexpr inline _MaskMember 1346 _S_negate(const _SimdTuple<_Tp, _As...>& __x) noexcept 1347 { 1348 _MaskMember __bits = 0; 1349 __for_each( 1350 __x, [&__bits](auto __meta, auto __native) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1351 __bits 1352 |= __meta._S_mask_to_shifted_ullong(__meta._S_negate(__native)); 1353 }); 1354 return __bits; 1355 } 1356 1357 // reductions {{{2 1358 template <typename _Tp, typename _BinaryOperation> 1359 static constexpr inline _Tp _S_reduce(const _Simd<_Tp>& __x, 1360 const _BinaryOperation& __binary_op) 1361 { 1362 using _Tup = _SimdMember<_Tp>; 1363 const _Tup& __tup = __data(__x); 1364 if constexpr (_Tup::_S_tuple_size == 1) 1365 return _Tup::_FirstAbi::_SimdImpl::_S_reduce( 1366 __tup.template _M_simd_at<0>(), __binary_op); 1367 else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 2 1368 && _Tup::_SecondType::_S_size() == 1) 1369 { 1370 return __binary_op(simd<_Tp, simd_abi::scalar>( 1371 reduce(__tup.template _M_simd_at<0>(), 1372 __binary_op)), 1373 __tup.template _M_simd_at<1>())[0]; 1374 } 1375 else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 4 1376 && _Tup::_SecondType::_S_size() == 2) 1377 { 1378 return __binary_op( 1379 simd<_Tp, simd_abi::scalar>( 1380 reduce(__tup.template _M_simd_at<0>(), __binary_op)), 1381 simd<_Tp, simd_abi::scalar>( 1382 reduce(__tup.template _M_simd_at<1>(), __binary_op)))[0]; 1383 } 1384 else 1385 { 1386 const auto& __x2 = __call_with_n_evaluations< 1387 __div_roundup(_Tup::_S_tuple_size, 2)>( 1388 [](auto __first_simd, auto... __remaining) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1389 if constexpr (sizeof...(__remaining) == 0) 1390 return __first_simd; 1391 else 1392 { 1393 using _Tup2 1394 = _SimdTuple<_Tp, 1395 typename decltype(__first_simd)::abi_type, 1396 typename decltype(__remaining)::abi_type...>; 1397 return fixed_size_simd<_Tp, _Tup2::_S_size()>( 1398 __private_init, 1399 __make_simd_tuple(__first_simd, __remaining...)); 1400 } 1401 }, 1402 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1403 auto __left = __tup.template _M_simd_at<2 * __i>(); 1404 if constexpr (2 * __i + 1 == _Tup::_S_tuple_size) 1405 return __left; 1406 else 1407 { 1408 auto __right = __tup.template _M_simd_at<2 * __i + 1>(); 1409 using _LT = decltype(__left); 1410 using _RT = decltype(__right); 1411 if constexpr (_LT::size() == _RT::size()) 1412 return __binary_op(__left, __right); 1413 else 1414 { 1415 _GLIBCXX_SIMD_USE_CONSTEXPR_API 1416 typename _LT::mask_type __k( 1417 __private_init, 1418 [](auto __j) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1419 return __j < _RT::size(); 1420 }); 1421 _LT __ext_right = __left; 1422 where(__k, __ext_right) 1423 = __proposed::resizing_simd_cast<_LT>(__right); 1424 where(__k, __left) = __binary_op(__left, __ext_right); 1425 return __left; 1426 } 1427 } 1428 }); 1429 return reduce(__x2, __binary_op); 1430 } 1431 } 1432 1433 // _S_min, _S_max {{{2 1434 template <typename _Tp, typename... _As> 1435 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> 1436 _S_min(const _SimdTuple<_Tp, _As...>& __a, const _SimdTuple<_Tp, _As...>& __b) 1437 { 1438 return __a._M_apply_per_chunk( 1439 [](auto __impl, auto __aa, auto __bb) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1440 return __impl._S_min(__aa, __bb); 1441 }, 1442 __b); 1443 } 1444 1445 template <typename _Tp, typename... _As> 1446 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> 1447 _S_max(const _SimdTuple<_Tp, _As...>& __a, const _SimdTuple<_Tp, _As...>& __b) 1448 { 1449 return __a._M_apply_per_chunk( 1450 [](auto __impl, auto __aa, auto __bb) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1451 return __impl._S_max(__aa, __bb); 1452 }, 1453 __b); 1454 } 1455 1456 // _S_complement {{{2 1457 template <typename _Tp, typename... _As> 1458 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> 1459 _S_complement(const _SimdTuple<_Tp, _As...>& __x) noexcept 1460 { 1461 return __x._M_apply_per_chunk( 1462 [](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1463 return __impl._S_complement(__xx); 1464 }); 1465 } 1466 1467 // _S_unary_minus {{{2 1468 template <typename _Tp, typename... _As> 1469 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> 1470 _S_unary_minus(const _SimdTuple<_Tp, _As...>& __x) noexcept 1471 { 1472 return __x._M_apply_per_chunk( 1473 [](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1474 return __impl._S_unary_minus(__xx); 1475 }); 1476 } 1477 1478 // arithmetic operators {{{2 1479 1480 #define _GLIBCXX_SIMD_FIXED_OP(name_, op_) \ 1481 template <typename _Tp, typename... _As> \ 1482 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> name_( \ 1483 const _SimdTuple<_Tp, _As...>& __x, const _SimdTuple<_Tp, _As...>& __y) \ 1484 { \ 1485 return __x._M_apply_per_chunk( \ 1486 [](auto __impl, auto __xx, auto __yy) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \ 1487 return __impl.name_(__xx, __yy); \ 1488 }, \ 1489 __y); \ 1490 } 1491 1492 _GLIBCXX_SIMD_FIXED_OP(_S_plus, +) 1493 _GLIBCXX_SIMD_FIXED_OP(_S_minus, -) 1494 _GLIBCXX_SIMD_FIXED_OP(_S_multiplies, *) 1495 _GLIBCXX_SIMD_FIXED_OP(_S_divides, /) 1496 _GLIBCXX_SIMD_FIXED_OP(_S_modulus, %) 1497 _GLIBCXX_SIMD_FIXED_OP(_S_bit_and, &) 1498 _GLIBCXX_SIMD_FIXED_OP(_S_bit_or, |) 1499 _GLIBCXX_SIMD_FIXED_OP(_S_bit_xor, ^) 1500 _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_left, <<) 1501 _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_right, >>) 1502 #undef _GLIBCXX_SIMD_FIXED_OP 1503 1504 template <typename _Tp, typename... _As> 1505 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> 1506 _S_bit_shift_left(const _SimdTuple<_Tp, _As...>& __x, int __y) 1507 { 1508 return __x._M_apply_per_chunk( 1509 [__y](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1510 return __impl._S_bit_shift_left(__xx, __y); 1511 }); 1512 } 1513 1514 template <typename _Tp, typename... _As> 1515 _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple<_Tp, _As...> 1516 _S_bit_shift_right(const _SimdTuple<_Tp, _As...>& __x, int __y) 1517 { 1518 return __x._M_apply_per_chunk( 1519 [__y](auto __impl, auto __xx) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1520 return __impl._S_bit_shift_right(__xx, __y); 1521 }); 1522 } 1523 1524 // math {{{2 1525 #define _GLIBCXX_SIMD_APPLY_ON_TUPLE(_RetTp, __name) \ 1526 template <typename _Tp, typename... _As, typename... _More> \ 1527 static inline __fixed_size_storage_t<_RetTp, _Np> \ 1528 _S_##__name(const _SimdTuple<_Tp, _As...>& __x, \ 1529 const _More&... __more) \ 1530 { \ 1531 if constexpr (sizeof...(_More) == 0) \ 1532 { \ 1533 if constexpr (is_same_v<_Tp, _RetTp>) \ 1534 return __x._M_apply_per_chunk( \ 1535 [](auto __impl, auto __xx) \ 1536 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \ 1537 { \ 1538 using _V = typename decltype(__impl)::simd_type; \ 1539 return __data(__name(_V(__private_init, __xx))); \ 1540 }); \ 1541 else \ 1542 return __optimize_simd_tuple( \ 1543 __x.template _M_apply_r<_RetTp>( \ 1544 [](auto __impl, auto __xx) \ 1545 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \ 1546 { return __impl._S_##__name(__xx); })); \ 1547 } \ 1548 else if constexpr ( \ 1549 is_same_v< \ 1550 _Tp, \ 1551 _RetTp> && (... && is_same_v<_SimdTuple<_Tp, _As...>, _More>) ) \ 1552 return __x._M_apply_per_chunk( \ 1553 [](auto __impl, auto __xx, auto... __pack) \ 1554 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \ 1555 { \ 1556 using _V = typename decltype(__impl)::simd_type; \ 1557 return __data(__name(_V(__private_init, __xx), \ 1558 _V(__private_init, __pack)...)); \ 1559 }, __more...); \ 1560 else if constexpr (is_same_v<_Tp, _RetTp>) \ 1561 return __x._M_apply_per_chunk( \ 1562 [](auto __impl, auto __xx, auto... __pack) \ 1563 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \ 1564 { \ 1565 using _V = typename decltype(__impl)::simd_type; \ 1566 return __data(__name(_V(__private_init, __xx), \ 1567 __autocvt_to_simd(__pack)...)); \ 1568 }, __more...); \ 1569 else \ 1570 __assert_unreachable<_Tp>(); \ 1571 } 1572 1573 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acos) 1574 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asin) 1575 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan) 1576 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan2) 1577 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cos) 1578 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sin) 1579 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tan) 1580 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acosh) 1581 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asinh) 1582 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atanh) 1583 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cosh) 1584 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sinh) 1585 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tanh) 1586 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp) 1587 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp2) 1588 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, expm1) 1589 _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, ilogb) 1590 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log) 1591 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log10) 1592 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log1p) 1593 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log2) 1594 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, logb) 1595 // modf implemented in simd_math.h 1596 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, 1597 scalbn) // double scalbn(double x, int exp); 1598 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, scalbln) 1599 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cbrt) 1600 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, abs) 1601 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fabs) 1602 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, pow) 1603 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sqrt) 1604 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erf) 1605 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erfc) 1606 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, lgamma) 1607 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tgamma) 1608 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, trunc) 1609 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ceil) 1610 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, floor) 1611 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nearbyint) 1612 1613 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, rint) 1614 _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lrint) 1615 _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llrint) 1616 1617 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, round) 1618 _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lround) 1619 _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llround) 1620 1621 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ldexp) 1622 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmod) 1623 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, remainder) 1624 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, copysign) 1625 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nextafter) 1626 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fdim) 1627 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmax) 1628 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmin) 1629 _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fma) 1630 _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, fpclassify) 1631 #undef _GLIBCXX_SIMD_APPLY_ON_TUPLE 1632 1633 template <typename _Tp, typename... _Abis> 1634 static inline _SimdTuple<_Tp, _Abis...> 1635 _S_remquo(const _SimdTuple<_Tp, _Abis...>& __x, const _SimdTuple<_Tp, _Abis...>& __y, 1636 __fixed_size_storage_t<int, _SimdTuple<_Tp, _Abis...>::_S_size()>* __z) 1637 { 1638 return __x._M_apply_per_chunk( 1639 [](auto __impl, const auto __xx, const auto __yy, auto& __zz) 1640 _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 1641 { return __impl._S_remquo(__xx, __yy, &__zz); }, 1642 __y, *__z); 1643 } 1644 1645 template <typename _Tp, typename... _As> 1646 static inline _SimdTuple<_Tp, _As...> 1647 _S_frexp(const _SimdTuple<_Tp, _As...>& __x, 1648 __fixed_size_storage_t<int, _Np>& __exp) noexcept 1649 { 1650 return __x._M_apply_per_chunk( 1651 [](auto __impl, const auto& __a, auto& __b) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1652 return __data(frexp(typename decltype(__impl)::simd_type(__private_init, __a), 1653 __autocvt_to_simd(__b))); 1654 }, __exp); 1655 } 1656 1657 #define _GLIBCXX_SIMD_TEST_ON_TUPLE_(name_) \ 1658 template <typename _Tp, typename... _As> \ 1659 static inline _MaskMember \ 1660 _S_##name_(const _SimdTuple<_Tp, _As...>& __x) noexcept \ 1661 { \ 1662 return _M_test([] (auto __impl, auto __xx) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { \ 1663 return __impl._S_##name_(__xx); \ 1664 }, __x); \ 1665 } 1666 1667 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isinf) 1668 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isfinite) 1669 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnan) 1670 _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnormal) 1671 _GLIBCXX_SIMD_TEST_ON_TUPLE_(signbit) 1672 #undef _GLIBCXX_SIMD_TEST_ON_TUPLE_ 1673 1674 // _S_increment & _S_decrement{{{2 1675 template <typename... _Ts> 1676 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1677 _S_increment(_SimdTuple<_Ts...>& __x) 1678 { 1679 __for_each( 1680 __x, [](auto __meta, auto& native) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1681 __meta._S_increment(native); 1682 }); 1683 } 1684 1685 template <typename... _Ts> 1686 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1687 _S_decrement(_SimdTuple<_Ts...>& __x) 1688 { 1689 __for_each( 1690 __x, [](auto __meta, auto& native) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1691 __meta._S_decrement(native); 1692 }); 1693 } 1694 1695 // compares {{{2 1696 #define _GLIBCXX_SIMD_CMP_OPERATIONS(__cmp) \ 1697 template <typename _Tp, typename... _As> \ 1698 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember \ 1699 __cmp(const _SimdTuple<_Tp, _As...>& __x, \ 1700 const _SimdTuple<_Tp, _As...>& __y) \ 1701 { \ 1702 return _M_test([](auto __impl, auto __xx, auto __yy) \ 1703 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA \ 1704 { return __impl.__cmp(__xx, __yy); }, \ 1705 __x, __y); \ 1706 } 1707 1708 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_equal_to) 1709 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_not_equal_to) 1710 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less) 1711 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less_equal) 1712 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isless) 1713 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessequal) 1714 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreater) 1715 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreaterequal) 1716 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessgreater) 1717 _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isunordered) 1718 #undef _GLIBCXX_SIMD_CMP_OPERATIONS 1719 1720 // smart_reference access {{{2 1721 template <typename _Tp, typename... _As, typename _Up> 1722 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1723 _S_set(_SimdTuple<_Tp, _As...>& __v, int __i, _Up&& __x) noexcept 1724 { __v._M_set(__i, static_cast<_Up&&>(__x)); } 1725 1726 // _S_masked_assign {{{2 1727 template <typename _Tp, typename... _As> 1728 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1729 _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs, 1730 const __type_identity_t<_SimdTuple<_Tp, _As...>>& __rhs) 1731 { 1732 __for_each(__lhs, __rhs, 1733 [&](auto __meta, auto& __native_lhs, auto __native_rhs) 1734 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 1735 { 1736 __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs, 1737 __native_rhs); 1738 }); 1739 } 1740 1741 // Optimization for the case where the RHS is a scalar. No need to broadcast 1742 // the scalar to a simd first. 1743 template <typename _Tp, typename... _As> 1744 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1745 _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs, 1746 const __type_identity_t<_Tp> __rhs) 1747 { 1748 __for_each( 1749 __lhs, [&](auto __meta, auto& __native_lhs) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1750 __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs, 1751 __rhs); 1752 }); 1753 } 1754 1755 // _S_masked_cassign {{{2 1756 template <typename _Op, typename _Tp, typename... _As> 1757 static constexpr inline void 1758 _S_masked_cassign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs, 1759 const _SimdTuple<_Tp, _As...>& __rhs, _Op __op) 1760 { 1761 __for_each(__lhs, __rhs, 1762 [&](auto __meta, auto& __native_lhs, auto __native_rhs) 1763 constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA 1764 { 1765 __meta.template _S_masked_cassign(__meta._S_make_mask(__bits), 1766 __native_lhs, __native_rhs, __op); 1767 }); 1768 } 1769 1770 // Optimization for the case where the RHS is a scalar. No need to broadcast 1771 // the scalar to a simd first. 1772 template <typename _Op, typename _Tp, typename... _As> 1773 static constexpr inline void 1774 _S_masked_cassign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs, 1775 const _Tp& __rhs, _Op __op) 1776 { 1777 __for_each( 1778 __lhs, [&](auto __meta, auto& __native_lhs) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1779 __meta.template _S_masked_cassign(__meta._S_make_mask(__bits), 1780 __native_lhs, __rhs, __op); 1781 }); 1782 } 1783 1784 // _S_masked_unary {{{2 1785 template <template <typename> class _Op, typename _Tp, typename... _As> 1786 static constexpr inline _SimdTuple<_Tp, _As...> 1787 _S_masked_unary(const _MaskMember __bits, const _SimdTuple<_Tp, _As...>& __v) 1788 { 1789 return __v._M_apply_wrapped([&__bits](auto __meta, 1790 auto __native) constexpr { 1791 return __meta.template _S_masked_unary<_Op>(__meta._S_make_mask( 1792 __bits), 1793 __native); 1794 }); 1795 } 1796 1797 // }}}2 1798 }; 1799 1800 // _MaskImplFixedSize {{{1 1801 template <int _Np, typename> 1802 struct _MaskImplFixedSize 1803 { 1804 static_assert( 1805 sizeof(_ULLong) * __CHAR_BIT__ >= _Np, 1806 "The fixed_size implementation relies on one _ULLong being able to store " 1807 "all boolean elements."); // required in load & store 1808 1809 // member types {{{ 1810 using _Abi = simd_abi::fixed_size<_Np>; 1811 1812 using _MaskMember = _SanitizedBitMask<_Np>; 1813 1814 template <typename _Tp> 1815 using _FirstAbi = typename __fixed_size_storage_t<_Tp, _Np>::_FirstAbi; 1816 1817 template <typename _Tp> 1818 using _TypeTag = _Tp*; 1819 1820 // }}} 1821 // _S_broadcast {{{ 1822 template <typename> 1823 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1824 _S_broadcast(bool __x) 1825 { return __x ? ~_MaskMember() : _MaskMember(); } 1826 1827 // }}} 1828 // _S_load {{{ 1829 template <typename> 1830 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1831 _S_load(const bool* __mem) 1832 { 1833 if (__builtin_is_constant_evaluated()) 1834 { 1835 _MaskMember __r{}; 1836 for (size_t __i = 0; __i < _Np; ++__i) 1837 __r.set(__i, __mem[__i]); 1838 return __r; 1839 } 1840 using _Ip = __int_for_sizeof_t<bool>; 1841 // the following load uses element_aligned and relies on __mem already 1842 // carrying alignment information from when this load function was 1843 // called. 1844 const simd<_Ip, _Abi> __bools(reinterpret_cast<const __may_alias<_Ip>*>( 1845 __mem), 1846 element_aligned); 1847 return __data(__bools != 0); 1848 } 1849 1850 // }}} 1851 // _S_to_bits {{{ 1852 template <bool _Sanitized> 1853 _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np> 1854 _S_to_bits(_BitMask<_Np, _Sanitized> __x) 1855 { 1856 if constexpr (_Sanitized) 1857 return __x; 1858 else 1859 return __x._M_sanitized(); 1860 } 1861 1862 // }}} 1863 // _S_convert {{{ 1864 template <typename _Tp, typename _Up, typename _UAbi> 1865 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1866 _S_convert(simd_mask<_Up, _UAbi> __x) 1867 { 1868 return _UAbi::_MaskImpl::_S_to_bits(__data(__x)) 1869 .template _M_extract<0, _Np>(); 1870 } 1871 1872 // }}} 1873 // _S_from_bitmask {{{2 1874 template <typename _Tp> 1875 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1876 _S_from_bitmask(_MaskMember __bits, _TypeTag<_Tp>) noexcept 1877 { return __bits; } 1878 1879 // _S_load {{{2 1880 static constexpr inline _MaskMember 1881 _S_load(const bool* __mem) noexcept 1882 { 1883 // TODO: _UChar is not necessarily the best type to use here. For smaller 1884 // _Np _UShort, _UInt, _ULLong, float, and double can be more efficient. 1885 _ULLong __r = 0; 1886 using _Vs = __fixed_size_storage_t<_UChar, _Np>; 1887 __for_each(_Vs{}, [&](auto __meta, auto) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1888 __r |= __meta._S_mask_to_shifted_ullong( 1889 __meta._S_mask_impl._S_load(&__mem[__meta._S_offset], 1890 _SizeConstant<__meta._S_size()>())); 1891 }); 1892 return __r; 1893 } 1894 1895 // _S_masked_load {{{2 1896 static constexpr inline _MaskMember 1897 _S_masked_load(_MaskMember __merge, _MaskMember __mask, const bool* __mem) noexcept 1898 { 1899 _BitOps::_S_bit_iteration(__mask.to_ullong(), 1900 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 1901 __merge.set(__i, __mem[__i]); 1902 }); 1903 return __merge; 1904 } 1905 1906 // _S_store {{{2 1907 static constexpr inline void 1908 _S_store(const _MaskMember __bitmask, bool* __mem) noexcept 1909 { 1910 if constexpr (_Np == 1) 1911 __mem[0] = __bitmask[0]; 1912 else 1913 _FirstAbi<_UChar>::_CommonImpl::_S_store_bool_array(__bitmask, __mem); 1914 } 1915 1916 // _S_masked_store {{{2 1917 static constexpr inline void 1918 _S_masked_store(const _MaskMember __v, bool* __mem, const _MaskMember __k) noexcept 1919 { 1920 _BitOps::_S_bit_iteration( 1921 __k, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { __mem[__i] = __v[__i]; }); 1922 } 1923 1924 // logical and bitwise operators {{{2 1925 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1926 _S_logical_and(const _MaskMember& __x, const _MaskMember& __y) noexcept 1927 { return __x & __y; } 1928 1929 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1930 _S_logical_or(const _MaskMember& __x, const _MaskMember& __y) noexcept 1931 { return __x | __y; } 1932 1933 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1934 _S_bit_not(const _MaskMember& __x) noexcept 1935 { return ~__x; } 1936 1937 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1938 _S_bit_and(const _MaskMember& __x, const _MaskMember& __y) noexcept 1939 { return __x & __y; } 1940 1941 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1942 _S_bit_or(const _MaskMember& __x, const _MaskMember& __y) noexcept 1943 { return __x | __y; } 1944 1945 _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember 1946 _S_bit_xor(const _MaskMember& __x, const _MaskMember& __y) noexcept 1947 { return __x ^ __y; } 1948 1949 // smart_reference access {{{2 1950 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1951 _S_set(_MaskMember& __k, int __i, bool __x) noexcept 1952 { __k.set(__i, __x); } 1953 1954 // _S_masked_assign {{{2 1955 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1956 _S_masked_assign(const _MaskMember __k, _MaskMember& __lhs, const _MaskMember __rhs) 1957 { __lhs = (__lhs & ~__k) | (__rhs & __k); } 1958 1959 // Optimization for the case where the RHS is a scalar. 1960 _GLIBCXX_SIMD_INTRINSIC static constexpr void 1961 _S_masked_assign(const _MaskMember __k, _MaskMember& __lhs, const bool __rhs) 1962 { 1963 if (__rhs) 1964 __lhs |= __k; 1965 else 1966 __lhs &= ~__k; 1967 } 1968 1969 // }}}2 1970 // _S_all_of {{{ 1971 template <typename _Tp> 1972 _GLIBCXX_SIMD_INTRINSIC static constexpr bool 1973 _S_all_of(simd_mask<_Tp, _Abi> __k) 1974 { return __data(__k).all(); } 1975 1976 // }}} 1977 // _S_any_of {{{ 1978 template <typename _Tp> 1979 _GLIBCXX_SIMD_INTRINSIC static constexpr bool 1980 _S_any_of(simd_mask<_Tp, _Abi> __k) 1981 { return __data(__k).any(); } 1982 1983 // }}} 1984 // _S_none_of {{{ 1985 template <typename _Tp> 1986 _GLIBCXX_SIMD_INTRINSIC static constexpr bool 1987 _S_none_of(simd_mask<_Tp, _Abi> __k) 1988 { return __data(__k).none(); } 1989 1990 // }}} 1991 // _S_some_of {{{ 1992 template <typename _Tp> 1993 _GLIBCXX_SIMD_INTRINSIC static constexpr bool 1994 _S_some_of([[maybe_unused]] simd_mask<_Tp, _Abi> __k) 1995 { 1996 if constexpr (_Np == 1) 1997 return false; 1998 else 1999 return __data(__k).any() && !__data(__k).all(); 2000 } 2001 2002 // }}} 2003 // _S_popcount {{{ 2004 template <typename _Tp> 2005 _GLIBCXX_SIMD_INTRINSIC static constexpr int 2006 _S_popcount(simd_mask<_Tp, _Abi> __k) 2007 { return __data(__k).count(); } 2008 2009 // }}} 2010 // _S_find_first_set {{{ 2011 template <typename _Tp> 2012 _GLIBCXX_SIMD_INTRINSIC static constexpr int 2013 _S_find_first_set(simd_mask<_Tp, _Abi> __k) 2014 { return std::__countr_zero(__data(__k).to_ullong()); } 2015 2016 // }}} 2017 // _S_find_last_set {{{ 2018 template <typename _Tp> 2019 _GLIBCXX_SIMD_INTRINSIC static constexpr int 2020 _S_find_last_set(simd_mask<_Tp, _Abi> __k) 2021 { return std::__bit_width(__data(__k).to_ullong()) - 1; } 2022 2023 // }}} 2024 }; 2025 // }}}1 2026 2027 _GLIBCXX_SIMD_END_NAMESPACE 2028 #endif // __cplusplus >= 201703L 2029 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_ 2030 2031 // vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
Contact us
|
About us
|
Term of use
|
Copyright © 2000-2025 MyWebUniversity.com ™
