Where Online Learning is simpler!
The C and C++ Include Header Files
cat -n /usr/include/c++/15/experimental/bits/simd_converter.h
1 // Generic simd conversions -*- C++ -*- 2 3 // Copyright (C) 2020-2025 Free Software Foundation, Inc. 4 // 5 // This file is part of the GNU ISO C++ Library. This library is free 6 // software; you can redistribute it and/or modify it under the 7 // terms of the GNU General Public License as published by the 8 // Free Software Foundation; either version 3, or (at your option) 9 // any later version. 10 11 // This library is distributed in the hope that it will be useful, 12 // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 // GNU General Public License for more details. 15 16 // Under Section 7 of GPL version 3, you are granted additional 17 // permissions described in the GCC Runtime Library Exception, version 18 // 3.1, as published by the Free Software Foundation. 19 20 // You should have received a copy of the GNU General Public License and 21 // a copy of the GCC Runtime Library Exception along with this program; 22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 // <http://www.gnu.org/licenses/>. 24 25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ 26 #define _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ 27 28 #if __cplusplus >= 201703L 29 30 _GLIBCXX_SIMD_BEGIN_NAMESPACE 31 32 template <typename _Arg, typename _Ret, typename _To, size_t _Np> 33 _Ret __converter_fallback(_Arg __a) 34 { 35 _Ret __ret{}; 36 __execute_n_times<_Np>( 37 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 38 __ret._M_set(__i, static_cast<_To>(__a[__i])); 39 }); 40 return __ret; 41 } 42 43 // _SimdConverter scalar -> scalar {{{ 44 template <typename _From, typename _To> 45 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::scalar, 46 enable_if_t<!is_same_v<_From, _To>>> 47 { 48 _GLIBCXX_SIMD_INTRINSIC constexpr _To operator()(_From __a) const noexcept 49 { return static_cast<_To>(__a); } 50 }; 51 52 // }}} 53 // _SimdConverter scalar -> "native" {{{ 54 template <typename _From, typename _To, typename _Abi> 55 struct _SimdConverter<_From, simd_abi::scalar, _To, _Abi, 56 enable_if_t<!is_same_v<_Abi, simd_abi::scalar>>> 57 { 58 using _Ret = typename _Abi::template __traits<_To>::_SimdMember; 59 60 template <typename... _More> 61 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 62 operator()(_From __a, _More... __more) const noexcept 63 { 64 static_assert(sizeof...(_More) + 1 == _Abi::template _S_size<_To>); 65 static_assert(conjunction_v<is_same<_From, _More>...>); 66 return __make_vector<_To>(__a, __more...); 67 } 68 }; 69 70 // }}} 71 // _SimdConverter "native non-sve 1" -> "native non-sve 2" {{{ 72 template <typename _From, typename _To, typename _AFrom, typename _ATo> 73 struct _SimdConverter< 74 _From, _AFrom, _To, _ATo, 75 enable_if_t<!disjunction_v< 76 __is_fixed_size_abi<_AFrom>, __is_fixed_size_abi<_ATo>, 77 is_same<_AFrom, simd_abi::scalar>, is_same<_ATo, simd_abi::scalar>, 78 conjunction<is_same<_From, _To>, is_same<_AFrom, _ATo>>> 79 && !(__is_sve_abi<_AFrom>() || __is_sve_abi<_ATo>())>> 80 { 81 using _Arg = typename _AFrom::template __traits<_From>::_SimdMember; 82 using _Ret = typename _ATo::template __traits<_To>::_SimdMember; 83 using _V = __vector_type_t<_To, simd_size_v<_To, _ATo>>; 84 85 template <typename... _More> 86 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 87 operator()(_Arg __a, _More... __more) const noexcept 88 { return __vector_convert<_V>(__a, __more...); } 89 }; 90 91 // }}} 92 // _SimdConverter "native 1" -> "native 2" {{{ 93 template <typename _From, typename _To, typename _AFrom, typename _ATo> 94 struct _SimdConverter< 95 _From, _AFrom, _To, _ATo, 96 enable_if_t<!disjunction_v< 97 __is_fixed_size_abi<_AFrom>, __is_fixed_size_abi<_ATo>, 98 is_same<_AFrom, simd_abi::scalar>, is_same<_ATo, simd_abi::scalar>, 99 conjunction<is_same<_From, _To>, is_same<_AFrom, _ATo>>> 100 && (__is_sve_abi<_AFrom>() || __is_sve_abi<_ATo>()) 101 >> 102 { 103 using _Arg = typename _AFrom::template __traits<_From>::_SimdMember; 104 using _Ret = typename _ATo::template __traits<_To>::_SimdMember; 105 106 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 107 operator()(_Arg __x) const noexcept 108 { return __converter_fallback<_Arg, _Ret, _To, simd_size_v<_To, _ATo>>(__x); } 109 }; 110 111 // }}} 112 // _SimdConverter scalar -> fixed_size<1> {{{1 113 template <typename _From, typename _To> 114 struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::fixed_size<1>, 115 void> 116 { 117 _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_To, simd_abi::scalar> 118 operator()(_From __x) const noexcept 119 { return {static_cast<_To>(__x)}; } 120 }; 121 122 // _SimdConverter fixed_size<1> -> scalar {{{1 123 template <typename _From, typename _To> 124 struct _SimdConverter<_From, simd_abi::fixed_size<1>, _To, simd_abi::scalar, 125 void> 126 { 127 _GLIBCXX_SIMD_INTRINSIC constexpr _To 128 operator()(_SimdTuple<_From, simd_abi::scalar> __x) const noexcept 129 { return {static_cast<_To>(__x.first)}; } 130 }; 131 132 // _SimdConverter fixed_size<_Np> -> fixed_size<_Np> {{{1 133 template <typename _From, typename _To, int _Np> 134 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, 135 simd_abi::fixed_size<_Np>, 136 enable_if_t<!is_same_v<_From, _To>>> 137 { 138 using _Ret = __fixed_size_storage_t<_To, _Np>; 139 using _Arg = __fixed_size_storage_t<_From, _Np>; 140 141 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 142 operator()(const _Arg& __x) const noexcept 143 { 144 if constexpr (is_same_v<_From, _To>) 145 return __x; 146 147 // fallback to sequential when sve is available 148 else if constexpr (__have_sve) 149 return __converter_fallback<_Arg, _Ret, _To, _Np>(__x); 150 151 // special case (optimize) int signedness casts 152 else if constexpr (sizeof(_From) == sizeof(_To) 153 && is_integral_v<_From> && is_integral_v<_To>) 154 return __bit_cast<_Ret>(__x); 155 156 // special case if all ABI tags in _Ret are scalar 157 else if constexpr (__is_scalar_abi<typename _Ret::_FirstAbi>()) 158 { 159 return __call_with_subscripts( 160 __x, make_index_sequence<_Np>(), 161 [](auto... __values) constexpr _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA -> _Ret { 162 return __make_simd_tuple<_To, decltype((void) __values, 163 simd_abi::scalar())...>( 164 static_cast<_To>(__values)...); 165 }); 166 } 167 168 // from one vector to one vector 169 else if constexpr (_Arg::_S_first_size == _Ret::_S_first_size) 170 { 171 _SimdConverter<_From, typename _Arg::_FirstAbi, _To, 172 typename _Ret::_FirstAbi> 173 __native_cvt; 174 if constexpr (_Arg::_S_tuple_size == 1) 175 return {__native_cvt(__x.first)}; 176 else 177 { 178 constexpr size_t _NRemain = _Np - _Arg::_S_first_size; 179 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To, 180 simd_abi::fixed_size<_NRemain>> 181 __remainder_cvt; 182 return {__native_cvt(__x.first), __remainder_cvt(__x.second)}; 183 } 184 } 185 186 // from one vector to multiple vectors 187 else if constexpr (_Arg::_S_first_size > _Ret::_S_first_size) 188 { 189 const auto __multiple_return_chunks 190 = __convert_all<__vector_type_t<_To, _Ret::_S_first_size>>( 191 __x.first); 192 constexpr auto __converted = __multiple_return_chunks.size() 193 * _Ret::_FirstAbi::template _S_size<_To>; 194 constexpr auto __remaining = _Np - __converted; 195 if constexpr (_Arg::_S_tuple_size == 1 && __remaining == 0) 196 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks); 197 else if constexpr (_Arg::_S_tuple_size == 1) 198 { // e.g. <int, 3> -> <double, 2, 1> or <short, 7> -> <double, 4, 2, 199 // 1> 200 using _RetRem 201 = __remove_cvref_t<decltype(__simd_tuple_pop_front<__converted>( 202 _Ret()))>; 203 const auto __return_chunks2 204 = __convert_all<__vector_type_t<_To, _RetRem::_S_first_size>, 0, 205 __converted>(__x.first); 206 constexpr auto __converted2 207 = __converted 208 + __return_chunks2.size() * _RetRem::_S_first_size; 209 if constexpr (__converted2 == _Np) 210 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks, 211 __return_chunks2); 212 else 213 { 214 using _RetRem2 = __remove_cvref_t< 215 decltype(__simd_tuple_pop_front<__return_chunks2.size() 216 * _RetRem::_S_first_size>( 217 _RetRem()))>; 218 const auto __return_chunks3 = __convert_all< 219 __vector_type_t<_To, _RetRem2::_S_first_size>, 0, 220 __converted2>(__x.first); 221 constexpr auto __converted3 222 = __converted2 223 + __return_chunks3.size() * _RetRem2::_S_first_size; 224 if constexpr (__converted3 == _Np) 225 return __to_simd_tuple<_To, _Np>(__multiple_return_chunks, 226 __return_chunks2, 227 __return_chunks3); 228 else 229 { 230 using _RetRem3 231 = __remove_cvref_t<decltype(__simd_tuple_pop_front< 232 __return_chunks3.size() 233 * _RetRem2::_S_first_size>( 234 _RetRem2()))>; 235 const auto __return_chunks4 = __convert_all< 236 __vector_type_t<_To, _RetRem3::_S_first_size>, 0, 237 __converted3>(__x.first); 238 constexpr auto __converted4 239 = __converted3 240 + __return_chunks4.size() * _RetRem3::_S_first_size; 241 if constexpr (__converted4 == _Np) 242 return __to_simd_tuple<_To, _Np>( 243 __multiple_return_chunks, __return_chunks2, 244 __return_chunks3, __return_chunks4); 245 else 246 __assert_unreachable<_To>(); 247 } 248 } 249 } 250 else 251 { 252 constexpr size_t _NRemain = _Np - _Arg::_S_first_size; 253 _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To, 254 simd_abi::fixed_size<_NRemain>> 255 __remainder_cvt; 256 return __simd_tuple_concat( 257 __to_simd_tuple<_To, _Arg::_S_first_size>( 258 __multiple_return_chunks), 259 __remainder_cvt(__x.second)); 260 } 261 } 262 263 // from multiple vectors to one vector 264 // _Arg::_S_first_size < _Ret::_S_first_size 265 // a) heterogeneous input at the end of the tuple (possible with partial 266 // native registers in _Ret) 267 else if constexpr (_Ret::_S_tuple_size == 1 268 && _Np % _Arg::_S_first_size != 0) 269 { 270 static_assert(_Ret::_FirstAbi::template _S_is_partial<_To>); 271 return _Ret{__generate_from_n_evaluations< 272 _Np, typename _VectorTraits<typename _Ret::_FirstType>::type>( 273 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 274 return static_cast<_To>(__x[__i]); 275 })}; 276 } 277 else 278 { 279 static_assert(_Arg::_S_tuple_size > 1); 280 constexpr auto __n 281 = __div_roundup(_Ret::_S_first_size, _Arg::_S_first_size); 282 return __call_with_n_evaluations<__n>( 283 [&__x](auto... __uncvted) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 284 // assuming _Arg Abi tags for all __i are _Arg::_FirstAbi 285 _SimdConverter<_From, typename _Arg::_FirstAbi, _To, 286 typename _Ret::_FirstAbi> 287 __native_cvt; 288 if constexpr (_Ret::_S_tuple_size == 1) 289 return _Ret{__native_cvt(__uncvted...)}; 290 else 291 return _Ret{ 292 __native_cvt(__uncvted...), 293 _SimdConverter< 294 _From, simd_abi::fixed_size<_Np - _Ret::_S_first_size>, _To, 295 simd_abi::fixed_size<_Np - _Ret::_S_first_size>>()( 296 __simd_tuple_pop_front<_Ret::_S_first_size>(__x))}; 297 }, [&__x](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 298 return __get_tuple_at<__i>(__x); 299 }); 300 } 301 } 302 }; 303 304 // _SimdConverter "native" -> fixed_size<_Np> {{{1 305 // i.e. 1 register to ? registers 306 template <typename _From, typename _Ap, typename _To, int _Np> 307 struct _SimdConverter<_From, _Ap, _To, simd_abi::fixed_size<_Np>, 308 enable_if_t<!__is_fixed_size_abi_v<_Ap>>> 309 { 310 static_assert( 311 _Np == simd_size_v<_From, _Ap>, 312 "_SimdConverter to fixed_size only works for equal element counts"); 313 314 using _Ret = __fixed_size_storage_t<_To, _Np>; 315 using _Arg = typename _SimdTraits<_From, _Ap>::_SimdMember; 316 317 _GLIBCXX_SIMD_INTRINSIC constexpr _Ret 318 operator()(_Arg __x) const noexcept 319 { 320 if constexpr (__have_sve) 321 return __converter_fallback<_Arg, _Ret, _To, _Np>(__x); 322 else if constexpr (_Ret::_S_tuple_size == 1) 323 return {__vector_convert<typename _Ret::_FirstType::_BuiltinType>(__x)}; 324 else 325 { 326 using _FixedNp = simd_abi::fixed_size<_Np>; 327 _SimdConverter<_From, _FixedNp, _To, _FixedNp> __fixed_cvt; 328 using _FromFixedStorage = __fixed_size_storage_t<_From, _Np>; 329 if constexpr (_FromFixedStorage::_S_tuple_size == 1) 330 return __fixed_cvt(_FromFixedStorage{__x}); 331 else if constexpr (_FromFixedStorage::_S_tuple_size == 2) 332 { 333 _FromFixedStorage __tmp; 334 static_assert(sizeof(__tmp) <= sizeof(__x)); 335 __builtin_memcpy(&__tmp.first, &__x, sizeof(__tmp.first)); 336 __builtin_memcpy(&__tmp.second.first, 337 reinterpret_cast<const char*>(&__x) 338 + sizeof(__tmp.first), 339 sizeof(__tmp.second.first)); 340 return __fixed_cvt(__tmp); 341 } 342 else 343 __assert_unreachable<_From>(); 344 } 345 } 346 }; 347 348 // _SimdConverter fixed_size<_Np> -> "native" {{{1 349 // i.e. ? register to 1 registers 350 template <typename _From, int _Np, typename _To, typename _Ap> 351 struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, _Ap, 352 enable_if_t<!__is_fixed_size_abi_v<_Ap>>> 353 { 354 static_assert( 355 _Np == simd_size_v<_To, _Ap>, 356 "_SimdConverter to fixed_size only works for equal element counts"); 357 358 using _Arg = __fixed_size_storage_t<_From, _Np>; 359 using _Ret = typename _SimdTraits<_To, _Ap>::_SimdMember; 360 361 _GLIBCXX_SIMD_INTRINSIC constexpr 362 _Ret 363 operator()(const _Arg& __x) const noexcept 364 { 365 if constexpr(__have_sve) 366 return __converter_fallback<_Arg, _Ret, _To, _Np>(__x); 367 else if constexpr (_Arg::_S_tuple_size == 1) 368 return __vector_convert<__vector_type_t<_To, _Np>>(__x.first); 369 else if constexpr (_Arg::_S_is_homogeneous) 370 return __call_with_n_evaluations<_Arg::_S_tuple_size>( 371 [](auto... __members) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 372 if constexpr ((is_convertible_v<decltype(__members), _To> && ...)) 373 return __vector_type_t<_To, _Np>{static_cast<_To>(__members)...}; 374 else 375 return __vector_convert<__vector_type_t<_To, _Np>>(__members...); 376 }, [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { 377 return __get_tuple_at<__i>(__x); 378 }); 379 else if constexpr (__fixed_size_storage_t<_To, _Np>::_S_tuple_size == 1) 380 { 381 _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, 382 simd_abi::fixed_size<_Np>> 383 __fixed_cvt; 384 return __fixed_cvt(__x).first; 385 } 386 else 387 { 388 const _SimdWrapper<_From, _Np> __xv 389 = __generate_from_n_evaluations<_Np, __vector_type_t<_From, _Np>>( 390 [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA { return __x[__i]; }); 391 return __vector_convert<__vector_type_t<_To, _Np>>(__xv); 392 } 393 } 394 }; 395 396 // }}}1 397 _GLIBCXX_SIMD_END_NAMESPACE 398 #endif // __cplusplus >= 201703L 399 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ 400 401 // vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
