libstdc++
simd_mask.h
1 // Implementation of <simd> -*- C++ -*-
2 
3 // Copyright The GNU Toolchain Authors.
4 //
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
9 // any later version.
10 
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
15 
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
19 
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
24 
25 #ifndef _GLIBCXX_SIMD_MASK_H
26 #define _GLIBCXX_SIMD_MASK_H 1
27 
28 #ifdef _GLIBCXX_SYSHDR
29 #pragma GCC system_header
30 #endif
31 
32 #if __cplusplus >= 202400L
33 
34 #include "simd_iterator.h"
35 #include "vec_ops.h"
36 #if _GLIBCXX_X86
37 #include "simd_x86.h"
38 #endif
39 
40 #include <bit>
41 #include <bitset>
42 
43 // psabi warnings are bogus because the ABI of the internal types never leaks into user code
44 #pragma GCC diagnostic push
45 #pragma GCC diagnostic ignored "-Wpsabi"
46 
47 namespace std _GLIBCXX_VISIBILITY(default)
48 {
49 _GLIBCXX_BEGIN_NAMESPACE_VERSION
50 namespace simd
51 {
52  template <unsigned _Np>
53  struct _SwapNeighbors
54  {
55  consteval unsigned
56  operator()(unsigned __i, unsigned __size) const
57  {
58  if (__size % (2 * _Np) != 0)
59  __builtin_abort(); // swap_neighbors<N> permutation requires a multiple of 2N elements
60  else if (std::has_single_bit(_Np))
61  return __i ^ _Np;
62  else if (__i % (2 * _Np) >= _Np)
63  return __i - _Np;
64  else
65  return __i + _Np;
66  }
67  };
68 
69  template <size_t _Np, size_t _Mp>
70  constexpr auto
71  __bitset_split(const bitset<_Mp>& __b)
72  {
73  constexpr auto __bits_per_word = __CHAR_BIT__ * __SIZEOF_LONG__;
74  if constexpr (_Np % __bits_per_word == 0)
75  {
76  struct _Tmp
77  {
78  bitset<_Np> _M_lo;
79  bitset<_Mp - _Np> _M_hi;
80  };
81  return __builtin_bit_cast(_Tmp, __b);
82  }
83  else
84  {
85  constexpr auto __bits_per_ullong = __CHAR_BIT__ * __SIZEOF_LONG_LONG__;
86  static_assert(_Mp <= __bits_per_ullong);
87  using _Lo = _Bitmask<_Np>;
88  using _Hi = _Bitmask<_Mp - _Np>;
89  struct _Tmp
90  {
91  _Lo _M_lo;
92  _Hi _M_hi;
93  };
94  return _Tmp {static_cast<_Lo>(__b.to_ullong()), static_cast<_Hi>(__b.to_ullong() >> _Np)};
95  }
96  }
97 
98  static_assert(__bitset_split<64>(bitset<128>(1))._M_lo == bitset<64>(1));
99  static_assert(__bitset_split<64>(bitset<128>(1))._M_hi == bitset<64>(0));
100 
101  // [simd.traits]
102  // --- rebind ---
103  template <typename _Tp, typename _Vp, _ArchTraits _Traits = {}>
104  struct rebind
105  {};
106 
107  /**
108  * Computes a member @c type `basic_vec<_Tp, Abi>`, where @c Abi is chosen such that the
109  * number of elements is equal to `_Vp::size()` and features of the ABI tag (such as the
110  * internal representation of masks, or storage order of complex components) are preserved.
111  */
112  template <__vectorizable _Tp, __simd_vec_type _Vp, _ArchTraits _Traits>
113  //requires requires { typename __deduce_abi_t<_Tp, _Vp::size()>; }
114  struct rebind<_Tp, _Vp, _Traits>
115  { using type = __similar_vec<_Tp, _Vp::size(), typename _Vp::abi_type>; };
116 
117  /**
118  * As above, except for @c basic_mask.
119  */
120  template <__vectorizable _Tp, __simd_mask_type _Mp, _ArchTraits _Traits>
121  //requires requires { typename __deduce_abi_t<_Tp, _Mp::size()>; }
122  struct rebind<_Tp, _Mp, _Traits>
123  { using type = __similar_mask<_Tp, _Mp::size(), typename _Mp::abi_type>; };
124 
125  template <typename _Tp, typename _Vp>
126  using rebind_t = typename rebind<_Tp, _Vp>::type;
127 
128  // --- resize ---
129  template <__simd_size_type _Np, typename _Vp, _ArchTraits _Traits = {}>
130  struct resize
131  {};
132 
133  template <__simd_size_type _Np, __simd_vec_type _Vp, _ArchTraits _Traits>
134  requires (_Np >= 1)
135  //requires requires { typename __deduce_abi_t<typename _Vp::value_type, _Np>; }
136  struct resize<_Np, _Vp, _Traits>
137  { using type = __similar_vec<typename _Vp::value_type, _Np, typename _Vp::abi_type>; };
138 
139  template <__simd_size_type _Np, __simd_mask_type _Mp, _ArchTraits _Traits>
140  requires (_Np >= 1)
141  //requires requires { typename __deduce_abi_t<typename _Mp::value_type, _Np>; }
142  struct resize<_Np, _Mp, _Traits>
143  {
144  using _A1 = decltype(__abi_rebind<__mask_element_size<_Mp>, _Np, typename _Mp::abi_type,
145  true>());
146 
147  static_assert(__abi_tag<_A1>);
148 
149  static_assert(_Mp::abi_type::_S_variant == _A1::_S_variant || __scalar_abi_tag<_A1>
150  || __scalar_abi_tag<typename _Mp::abi_type>);
151 
152  using type = basic_mask<__mask_element_size<_Mp>, _A1>;
153  };
154 
155  template <__simd_size_type _Np, typename _Vp>
156  using resize_t = typename resize<_Np, _Vp>::type;
157 
158  // [simd.syn]
159  inline constexpr __simd_size_type zero_element = numeric_limits<int>::min();
160 
161  inline constexpr __simd_size_type uninit_element = zero_element + 1;
162 
163  // [simd.permute.static]
164  template<__simd_size_type _Np = 0, __simd_vec_or_mask_type _Vp,
165  __index_permutation_function<_Vp> _IdxMap>
166  [[__gnu__::__always_inline__]]
167  constexpr resize_t<_Np == 0 ? _Vp::size() : _Np, _Vp>
168  permute(const _Vp& __v, _IdxMap&& __idxmap)
169  { return resize_t<_Np == 0 ? _Vp::size() : _Np, _Vp>::_S_static_permute(__v, __idxmap); }
170 
171  // [simd.permute.dynamic]
172  template<__simd_vec_or_mask_type _Vp, __simd_integral _Ip>
173  [[__gnu__::__always_inline__]]
174  constexpr resize_t<_Ip::size(), _Vp>
175  permute(const _Vp& __v, const _Ip& __indices)
176  { return __v[__indices]; }
177 
178  // [simd.creation] ----------------------------------------------------------
179  template<__simd_vec_type _Vp, typename _Ap>
180  [[__gnu__::__always_inline__]]
181  constexpr auto
182  chunk(const basic_vec<typename _Vp::value_type, _Ap>& __x) noexcept
183  { return __x.template _M_chunk<_Vp>(); }
184 
185  template<__simd_mask_type _Mp, typename _Ap>
186  [[__gnu__::__always_inline__]]
187  constexpr auto
188  chunk(const basic_mask<__mask_element_size<_Mp>, _Ap>& __x) noexcept
189  { return __x.template _M_chunk<_Mp>(); }
190 
191  template<__simd_size_type _Np, typename _Tp, typename _Ap>
192  [[__gnu__::__always_inline__]]
193  constexpr auto
194  chunk(const basic_vec<_Tp, _Ap>& __x) noexcept
195  -> decltype(chunk<resize_t<_Np, basic_vec<_Tp, _Ap>>>(__x))
196  { return chunk<resize_t<_Np, basic_vec<_Tp, _Ap>>>(__x); }
197 
198  template<__simd_size_type _Np, size_t _Bytes, typename _Ap>
199  [[__gnu__::__always_inline__]]
200  constexpr auto
201  chunk(const basic_mask<_Bytes, _Ap>& __x) noexcept
202  -> decltype(chunk<resize_t<_Np, basic_mask<_Bytes, _Ap>>>(__x))
203  { return chunk<resize_t<_Np, basic_mask<_Bytes, _Ap>>>(__x); }
204 
205  // LWG???? (reported 2025-11-25)
206  template<typename _Tp, typename _A0, typename... _Abis>
207  constexpr resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_vec<_Tp, _A0>>
208  cat(const basic_vec<_Tp, _A0>& __x0, const basic_vec<_Tp, _Abis>&... __xs) noexcept
209  {
210  return resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_vec<_Tp, _A0>>
211  ::_S_concat(__x0, __xs...);
212  }
213 
214  // LWG???? (reported 2025-11-25)
215  template<size_t _Bytes, typename _A0, typename... _Abis>
216  constexpr resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_mask<_Bytes, _A0>>
217  cat(const basic_mask<_Bytes, _A0>& __x0, const basic_mask<_Bytes, _Abis>&... __xs) noexcept
218  {
219  return resize_t<(_A0::_S_size + ... + _Abis::_S_size), basic_mask<_Bytes, _A0>>
220  ::_S_concat(__x0, __xs...);
221  }
222 
223  // implementation helper for chunk and cat
224  consteval int
225  __packs_to_skip_at_front(int __offset, initializer_list<int> __sizes)
226  {
227  int __i = 0;
228  int __n = 0;
229  for (int __s : __sizes)
230  {
231  __n += __s;
232  if (__n > __offset)
233  return __i;
234  ++__i;
235  }
236  __builtin_trap(); // called out of contract
237  }
238 
239  consteval int
240  __packs_to_skip_at_back(int __offset, int __max, initializer_list<int> __sizes)
241  {
242  int __i = 0;
243  int __n = -__offset;
244  for (int __s : __sizes)
245  {
246  ++__i;
247  __n += __s;
248  if (__n >= __max)
249  return int(__sizes.size()) - __i;
250  }
251  return 0;
252  }
253 
254  // in principle, this overload allows conversions to _Dst - and it wouldn't be wrong - but the
255  // general overload below is still a better candidate in overload resolution
256  template <typename _Dst>
257  [[__gnu__::__always_inline__]]
258  constexpr _Dst
259  __extract_simd_at(auto _Offset, const _Dst& __r, const auto&...)
260  requires(_Offset.value == 0)
261  { return __r; }
262 
263  template <typename _Dst, typename _V0>
264  [[__gnu__::__always_inline__]]
265  constexpr _Dst
266  __extract_simd_at(auto _Offset, const _V0&, const _Dst& __r, const auto&...)
267  requires(_Offset.value == _V0::size.value)
268  { return __r; }
269 
270  template <typename _Dst, typename... _Vs>
271  [[__gnu__::__always_inline__]]
272  constexpr _Dst
273  __extract_simd_at(auto _Offset, const _Vs&... __xs)
274  {
275  using _Adst = typename _Dst::abi_type;
276  if constexpr (_Adst::_S_nreg >= 2)
277  {
278  using _Dst0 = remove_cvref_t<decltype(declval<_Dst>()._M_get_low())>;
279  using _Dst1 = remove_cvref_t<decltype(declval<_Dst>()._M_get_high())>;
280  return _Dst::_S_init(__extract_simd_at<_Dst0>(_Offset, __xs...),
281  __extract_simd_at<_Dst1>(_Offset + _Dst0::size, __xs...));
282  }
283  else
284  {
285  using _Ret = remove_cvref_t<decltype(declval<_Dst>()._M_get())>;
286  constexpr bool __use_bitmask = __simd_mask_type<_Dst> && _Adst::_S_is_bitmask;
287  constexpr int __dst_full_size = __bit_ceil(unsigned(_Adst::_S_size));
288  constexpr int __nargs = sizeof...(__xs);
289  using _Afirst = typename _Vs...[0]::abi_type;
290  using _Alast = typename _Vs...[__nargs - 1]::abi_type;
291  const auto& __x0 = __xs...[0];
292  const auto& __xlast = __xs...[__nargs - 1];
293  constexpr int __ninputs = (_Vs::size.value + ...);
294  if constexpr (_Offset.value >= _Afirst::_S_size
295  || __ninputs - _Offset.value - _Alast::_S_size >= _Adst::_S_size)
296  { // can drop inputs at the front and/or back of the pack
297  constexpr int __skip_front = __packs_to_skip_at_front(_Offset.value,
298  {_Vs::size.value...});
299  constexpr int __skip_back = __packs_to_skip_at_back(_Offset.value, _Adst::_S_size,
300  {_Vs::size.value...});
301  static_assert(__skip_front > 0 || __skip_back > 0);
302  constexpr auto [...__skip] = _IotaArray<__skip_front>;
303  constexpr auto [...__is] = _IotaArray<__nargs - __skip_front - __skip_back>;
304  constexpr int __new_offset = _Offset.value - (0 + ... + _Vs...[__skip]::size.value);
305  return __extract_simd_at<_Dst>(cw<__new_offset>, __xs...[__is + __skip_front]...);
306  }
307  else if constexpr (_Adst::_S_size == 1)
308  { // trivial conversion to one value_type
309  return _Dst(__x0[_Offset.value]);
310  }
311  else if constexpr (_Afirst::_S_nreg >= 2 || _Alast::_S_nreg >= 2)
312  { // flatten first and/or last multi-register argument
313  constexpr bool __flatten_first = _Afirst::_S_nreg >= 2;
314  constexpr bool __flatten_last = __nargs > 1 && _Alast::_S_nreg >= 2;
315  constexpr auto [...__is] = _IotaArray<__nargs - __flatten_first - __flatten_last>;
316  if constexpr (__flatten_first && __flatten_last)
317  return __extract_simd_at<_Dst>(
318  _Offset, __x0._M_get_low(), __x0._M_get_high(), __xs...[__is + 1]...,
319  __xlast._M_get_low(), __xlast._M_get_high());
320  else if constexpr (__flatten_first)
321  return __extract_simd_at<_Dst>(
322  _Offset, __x0._M_get_low(), __x0._M_get_high(), __xs...[__is + 1]...);
323  else
324  return __extract_simd_at<_Dst>(
325  _Offset, __xs...[__is]..., __xlast._M_get_low(), __xlast._M_get_high());
326  }
327  else if constexpr (__simd_mask_type<_Dst>
328  && ((_Adst::_S_variant != _Vs::abi_type::_S_variant
329  && !__scalar_abi_tag<typename _Vs::abi_type>) || ...))
330  { // convert ABI tag if incompatible
331  return __extract_simd_at<_Dst>(
332  _Offset, static_cast<const resize_t<_Vs::size.value, _Dst>&>(__xs)...);
333  }
334 
335  // at this point __xs should be as small as possible; there may be some corner cases left
336 
337  else if constexpr (__nargs == 1)
338  { // simple and optimal
339  if constexpr (__use_bitmask)
340  return _Dst(_Ret(__x0._M_to_uint() >> _Offset.value));
341  else
342  return _VecOps<_Ret>::_S_extract(__x0._M_concat_data(false), _Offset);
343  }
344  else if constexpr (__use_bitmask)
345  { // fairly simple and optimal bit shifting solution
346  static_assert(_Afirst::_S_nreg == 1);
347  static_assert(_Offset.value < _Afirst::_S_size);
348  int __offset = -_Offset.value;
349  _Ret __r;
350  template for (const auto& __x : {__xs...})
351  {
352  if (__offset <= 0)
353  __r = _Ret(__x._M_to_uint() >> -__offset);
354  else if (__offset < _Adst::_S_size)
355  __r |= _Ret(_Ret(__x._M_to_uint()) << __offset);
356  __offset += __x.size.value;
357  }
358  return _Dst(__r);
359  }
360  else if constexpr (__nargs == 2 && _Offset == 0 && _Adst::_S_nreg == 1
361  && _Afirst::_S_size >= _Alast::_S_size
362  && __has_single_bit(unsigned(_Afirst::_S_size)))
363  { // simple __vec_concat
364  if constexpr (_Afirst::_S_size == 1)
365  // even simpler init from two values
366  return _Ret{__x0._M_concat_data()[0], __xlast._M_concat_data()[0]};
367  else
368  {
369  const auto __v0 = __x0._M_concat_data();
370  const auto __v1 = __vec_zero_pad_to<sizeof(__v0)>(__xlast._M_concat_data());
371  return __vec_concat(__v0, __v1);
372  }
373  }
374  else if constexpr (__nargs == 2 && _Adst::_S_nreg == 1 && _Offset == 0
375  && _Afirst::_S_nreg == 1 && _Alast::_S_size == 1)
376  { // optimize insertion of one element at the end
377  _Ret __r = __vec_zero_pad_to<sizeof(_Ret)>(__x0._M_get());
378  __vec_set(__r, _Afirst::_S_size, __xlast._M_concat_data()[0]);
379  return __r;
380  }
381  else if constexpr (__nargs == 2 && _Adst::_S_nreg == 1 && _Offset == 0
382  && _Afirst::_S_nreg == 1 && _Alast::_S_size == 2)
383  { // optimize insertion of two elements at the end
384  _Ret __r = __vec_zero_pad_to<sizeof(_Ret)>(__x0._M_concat_data());
385  const auto __x1 = __xlast._M_concat_data();
386  if constexpr (sizeof(__x1) <= sizeof(double) && (_Afirst::_S_size & 1) == 0)
387  { // can use a single insert instruction
388  using _Up = __conditional_t<
389  is_floating_point_v<__vec_value_type<_Ret>>,
390  __conditional_t<sizeof(__x1) == sizeof(double), double, float>,
391  __integer_from<sizeof(__x1)>>;
392  auto __r2 = __vec_bit_cast<_Up>(__r);
393  __vec_set(__r2, _Afirst::_S_size / 2, __vec_bit_cast<_Up>(__x1)[0]);
394  __r = reinterpret_cast<_Ret>(__r2);
395  }
396  else
397  {
398  __vec_set(__r, _Afirst::_S_size, __x1[0]);
399  __vec_set(__r, _Afirst::_S_size + 1, __x1[1]);
400  }
401  return __r;
402  }
403  else if constexpr (__nargs == 2 && _Afirst::_S_nreg == 1 && _Alast::_S_nreg == 1)
404  { // optimize concat of two input vectors (e.g. using palignr)
405  constexpr auto [...__is] = _IotaArray<__dst_full_size>;
406  constexpr int __v2_offset = __width_of<decltype(__x0._M_concat_data())>;
407  return __builtin_shufflevector(
408  __x0._M_concat_data(), __xlast._M_concat_data(), [](int __i) consteval {
409  if (__i < _Afirst::_S_size)
410  return __i;
411  __i -= _Afirst::_S_size;
412  if (__i < _Alast::_S_size)
413  return __i + __v2_offset;
414  else
415  return -1;
416  }(__is + _Offset.value)...);
417  }
418  else if (__is_const_known(__xs...) || __ninputs == _Adst::_S_size)
419  { // hard to optimize for the compiler, but necessary in constant expressions
420  return _VecOps<_Ret>::_S_extract(
421  __vec_concat_sized<__xs.size.value...>(__xs._M_concat_data(false)...),
422  _Offset);
423  }
424  else
425  { // fallback to concatenation in memory => load the result
426  alignas(_Ret) __vec_value_type<_Ret>
427  __tmp[std::max(__ninputs, _Offset.value + __dst_full_size)] = {};
428  int __offset = 0;
429  template for (const auto& __x : {__xs...})
430  {
431  if constexpr (__simd_mask_type<_Dst>)
432  (-__x)._M_store(__tmp + __offset);
433  else
434  __x._M_store(__tmp + __offset);
435  __offset += __x.size.value;
436  }
437  _Ret __r;
438  __builtin_memcpy(&__r, __tmp + _Offset.value, sizeof(_Ret));
439  return __r;
440  }
441  }
442  }
443 
444  // [simd.mask] --------------------------------------------------------------
445  template <size_t _Bytes, typename _Ap>
446  class basic_mask
447  {
448  public:
449  using value_type = bool;
450 
451  using abi_type = _Ap;
452 
453 #define _GLIBCXX_DELETE_SIMD "This specialization is disabled because of an invalid combination " \
454  "of template arguments to basic_mask."
455 
456  basic_mask() = delete(_GLIBCXX_DELETE_SIMD);
457 
458  ~basic_mask() = delete(_GLIBCXX_DELETE_SIMD);
459 
460  basic_mask(const basic_mask&) = delete(_GLIBCXX_DELETE_SIMD);
461 
462  basic_mask& operator=(const basic_mask&) = delete(_GLIBCXX_DELETE_SIMD);
463 
464 #undef _GLIBCXX_DELETE_SIMD
465  };
466 
467  template <size_t _Bytes, typename _Ap>
468  class _MaskBase
469  {
470  using _Mp = basic_mask<_Bytes, _Ap>;
471 
472  protected:
473  using _VecType = __simd_vec_from_mask_t<_Bytes, _Ap>;
474 
475  static_assert(destructible<_VecType> || _Bytes > sizeof(0ull));
476 
477  public:
478  using iterator = __iterator<_Mp>;
479 
480  using const_iterator = __iterator<const _Mp>;
481 
482  constexpr iterator
483  begin() noexcept
484  { return {static_cast<_Mp&>(*this), 0}; }
485 
486  constexpr const_iterator
487  begin() const noexcept
488  { return cbegin(); }
489 
490  constexpr const_iterator
491  cbegin() const noexcept
492  { return {static_cast<const _Mp&>(*this), 0}; }
493 
494  constexpr default_sentinel_t
495  end() const noexcept
496  { return {}; }
497 
498  constexpr default_sentinel_t
499  cend() const noexcept
500  { return {}; }
501 
502  static constexpr auto size = __simd_size_c<_Ap::_S_size>;
503 
504  _MaskBase() = default;
505 
506  // LWG issue from 2026-03-04 / P4042R0
507  template <size_t _UBytes, typename _UAbi>
508  requires (_Ap::_S_size != _UAbi::_S_size)
509  explicit
510  _MaskBase(const basic_mask<_UBytes, _UAbi>&) = delete("size mismatch");
511 
512  template <typename _Up, typename _UAbi>
513  explicit
514  _MaskBase(const basic_vec<_Up, _UAbi>&)
515  = delete("use operator! or a comparison to convert a vec into a mask");
516 
517  template <typename _Up, typename _UAbi>
518  requires (_Ap::_S_size != _UAbi::_S_size)
519  operator basic_vec<_Up, _UAbi>() const
520  = delete("size mismatch");
521  };
522 
523  template <size_t _Bytes, __abi_tag _Ap>
524  requires (_Ap::_S_nreg == 1)
525  class basic_mask<_Bytes, _Ap>
526  : public _MaskBase<_Bytes, _Ap>
527  {
528  using _Base = _MaskBase<_Bytes, _Ap>;
529 
530  using _VecType = _Base::_VecType;
531 
532  template <size_t, typename>
533  friend class basic_mask;
534 
535  template <typename, typename>
536  friend class basic_vec;
537 
538  static constexpr int _S_size = _Ap::_S_size;
539 
540  using _DataType = typename _Ap::template _MaskDataType<_Bytes>;
541 
542  static constexpr bool _S_has_bool_member = is_same_v<_DataType, bool>;
543 
544  static constexpr bool _S_is_scalar = _S_has_bool_member;
545 
546  static constexpr bool _S_use_bitmask = _Ap::_S_is_bitmask && !_S_is_scalar;
547 
548  static constexpr int _S_full_size = [] {
549  if constexpr (_S_is_scalar)
550  return _S_size;
551  else if constexpr (_S_use_bitmask && _S_size < __CHAR_BIT__)
552  return __CHAR_BIT__;
553  else
554  return __bit_ceil(unsigned(_S_size));
555  }();
556 
557  static constexpr bool _S_is_partial = _S_size != _S_full_size;
558 
559  static constexpr _DataType _S_implicit_mask = [] {
560  if constexpr (_S_is_scalar)
561  return true;
562  else if (!_S_is_partial)
563  return _DataType(~_DataType());
564  else if constexpr (_S_use_bitmask)
565  return _DataType((_DataType(1) << _S_size) - 1);
566  else
567  {
568  constexpr auto [...__is] = _IotaArray<_S_full_size>;
569  return _DataType{ (__is < _S_size ? -1 : 0)... };
570  }
571  }();
572 
573  // Actual padding bytes, not padding elements.
574  // => _S_padding_bytes is 0 even if _S_is_partial is true.
575  static constexpr size_t _S_padding_bytes = 0;
576 
577  _DataType _M_data;
578 
579  public:
580  using value_type = bool;
581 
582  using abi_type = _Ap;
583 
584  using iterator = _Base::iterator;
585 
586  using const_iterator = _Base::const_iterator;
587 
588  // internal but public API ----------------------------------------------
589  [[__gnu__::__always_inline__]]
590  static constexpr basic_mask
591  _S_init(_DataType __x)
592  {
593  basic_mask __r;
594  __r._M_data = __x;
595  return __r;
596  }
597 
598  [[__gnu__::__always_inline__]]
599  static constexpr basic_mask
600  _S_init(unsigned_integral auto __bits)
601  { return basic_mask(__bits); }
602 
603  [[__gnu__::__always_inline__]]
604  constexpr const _DataType&
605  _M_get() const
606  { return _M_data; }
607 
608  /** @internal
609  * Bit-cast the given object @p __x to basic_mask.
610  *
611  * This is necessary for _S_nreg > 1 where the last element can be bool or when the sizeof
612  * doesn't match because of different alignment requirements of the sub-masks.
613  */
614  template <size_t _UBytes, typename _UAbi>
615  [[__gnu__::__always_inline__]]
616  static constexpr basic_mask
617  _S_recursive_bit_cast(const basic_mask<_UBytes, _UAbi>& __x)
618  { return __builtin_bit_cast(basic_mask, __x._M_concat_data()); }
619 
620  [[__gnu__::__always_inline__]]
621  constexpr auto
622  _M_concat_data(bool __do_sanitize = _S_is_partial) const
623  {
624  if constexpr (_S_is_scalar)
625  return __vec_builtin_type<__integer_from<_Bytes>, 1>{__integer_from<_Bytes>(-_M_data)};
626  else
627  {
628  if constexpr (_S_is_partial)
629  if (__do_sanitize)
630  return _DataType(_M_data & _S_implicit_mask);
631  return _M_data;
632  }
633  }
634 
635  /** @internal
636  * Returns a mask where the first @p __n elements are true. All remaining elements are false.
637  *
638  * @pre @p __n > 0 && @p __n < _S_size
639  */
640  template <_ArchTraits _Traits = {}>
641  [[__gnu__::__always_inline__]]
642  static constexpr basic_mask
643  _S_partial_mask_of_n(int __n)
644  {
645  static_assert(!_S_is_scalar);
646  if constexpr (!_S_use_bitmask)
647  {
648  using _Ip = __integer_from<_Bytes>;
649  __glibcxx_simd_precondition(__n >= 0 && __n <= numeric_limits<_Ip>::max(),
650  "_S_partial_mask_of_n without _S_use_bitmask requires "
651  "positive __n that does not overflow.");
652  constexpr _DataType __0123
653  = __builtin_bit_cast(_DataType, _IotaArray<_Ip(_S_full_size)>);
654  return basic_mask(__0123 < _Ip(__n));
655  }
656  else
657  {
658  __glibcxx_simd_precondition(__n >= 0 && __n <= 255,
659  "The x86 BZHI instruction requires __n to "
660  "only use bits 0:7");
661 #if __has_builtin(__builtin_ia32_bzhi_si)
662  if constexpr (_S_size <= 32 && _Traits._M_have_bmi2())
663  return _S_init(_Bitmask<_S_size>(
664  __builtin_ia32_bzhi_si(~0u >> (32 - _S_size), unsigned(__n))));
665 #endif
666 #if __has_builtin(__builtin_ia32_bzhi_di)
667  else if constexpr (_S_size <= 64 && _Traits._M_have_bmi2())
668  return _S_init(__builtin_ia32_bzhi_di(~0ull >> (64 - _S_size), unsigned(__n)));
669 #endif
670  if constexpr (_S_size <= 32)
671  {
672  __glibcxx_simd_precondition(__n < 32, "invalid shift");
673  return _S_init(_Bitmask<_S_size>((1u << unsigned(__n)) - 1));
674  }
675  else if constexpr (_S_size <= 64)
676  {
677  __glibcxx_simd_precondition(__n < 64, "invalid shift");
678  return _S_init((1ull << unsigned(__n)) - 1);
679  }
680  else
681  static_assert(false);
682  }
683  }
684 
685  [[__gnu__::__always_inline__]]
686  constexpr basic_mask&
687  _M_and_neighbors()
688  {
689  if constexpr (_S_use_bitmask)
690  _M_data &= ((_M_data >> 1) & 0x5555'5555'5555'5555ull)
691  | ((_M_data << 1) & ~0x5555'5555'5555'5555ull);
692  else
693  _M_data &= _VecOps<_DataType>::_S_swap_neighbors(_M_data);
694  return *this;
695  }
696 
697  [[__gnu__::__always_inline__]]
698  constexpr basic_mask&
699  _M_or_neighbors()
700  {
701  if constexpr (_S_use_bitmask)
702  _M_data |= ((_M_data >> 1) & 0x5555'5555'5555'5555ull)
703  | ((_M_data << 1) & ~0x5555'5555'5555'5555ull);
704  else
705  _M_data |= _VecOps<_DataType>::_S_swap_neighbors(_M_data);
706  return *this;
707  }
708 
709  template <typename _Mp>
710  [[__gnu__::__always_inline__]]
711  constexpr auto _M_chunk() const noexcept
712  {
713  constexpr int __n = _S_size / _Mp::_S_size;
714  constexpr int __rem = _S_size % _Mp::_S_size;
715  constexpr auto [...__is] = _IotaArray<__n>;
716  if constexpr (__rem == 0)
717  return array<_Mp, __n>{__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>, *this)...};
718  else
719  {
720  using _Rest = resize_t<__rem, _Mp>;
721  return tuple(__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>, *this)...,
722  __extract_simd_at<_Rest>(cw<_Mp::_S_size * __n>, *this));
723  }
724  }
725 
726  [[__gnu__::__always_inline__]]
727  static constexpr const basic_mask&
728  _S_concat(const basic_mask& __x0) noexcept
729  { return __x0; }
730 
731  template <typename... _As>
732  requires (sizeof...(_As) > 1)
733  [[__gnu__::__always_inline__]]
734  static constexpr basic_mask
735  _S_concat(const basic_mask<_Bytes, _As>&... __xs) noexcept
736  {
737  static_assert(_S_size == (_As::_S_size + ...));
738  return __extract_simd_at<basic_mask>(cw<0>, __xs...);
739  }
740 
741  // [simd.mask.overview] default constructor -----------------------------
742  basic_mask() = default;
743 
744  // [simd.mask.overview] conversion extensions ---------------------------
745  [[__gnu__::__always_inline__]]
746  constexpr
747  basic_mask(_DataType __x) requires(!_S_is_scalar && !_S_use_bitmask)
748  : _M_data(__x)
749  {}
750 
751  [[__gnu__::__always_inline__]]
752  constexpr
753  operator _DataType() requires(!_S_is_scalar && !_S_use_bitmask)
754  { return _M_data; }
755 
756  // [simd.mask.ctor] broadcast constructor -------------------------------
757  [[__gnu__::__always_inline__]]
758  constexpr explicit
759  basic_mask(same_as<bool> auto __x) noexcept // LWG 4382.
760  : _M_data(__x ? _S_implicit_mask : _DataType())
761  {}
762 
763  // [simd.mask.ctor] conversion constructor ------------------------------
764  template <size_t _UBytes, typename _UAbi>
765  requires (_S_size == _UAbi::_S_size)
766  [[__gnu__::__always_inline__]]
767  constexpr explicit(__is_mask_conversion_explicit<_Ap, _UAbi>(_Bytes, _UBytes))
768  basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
769  : _M_data([&] [[__gnu__::__always_inline__]] {
770  using _UV = basic_mask<_UBytes, _UAbi>;
771  // bool to bool
772  if constexpr (_S_is_scalar)
773  return __x[0];
774 
775  // converting from an "array of bool"
776  else if constexpr (_UV::_S_is_scalar)
777  {
778  constexpr auto [...__is] = _IotaArray<_S_size>;
779  if constexpr (_S_use_bitmask)
780  return ((_DataType(__x[__is]) << __is) | ...);
781  else
782  return _DataType{__vec_value_type<_DataType>(-__x[__is])...};
783  }
784 
785  // vec-/bit-mask to bit-mask | bit-mask to vec-mask
786  else if constexpr (_S_use_bitmask || _UV::_S_use_bitmask)
787  return basic_mask(__x.to_bitset())._M_data;
788 
789  // vec-mask to vec-mask
790  else if constexpr (_Bytes == _UBytes)
791  return _S_recursive_bit_cast(__x)._M_data;
792 
793  else
794  {
795 #if _GLIBCXX_X86
796  // TODO: turn this into a __vec_mask_cast overload in simd_x86.h
797  if constexpr (_Bytes == 1 && _UBytes == 2)
798  if (!__is_const_known(__x))
799  {
800  if constexpr (_UAbi::_S_nreg == 1)
801  return __x86_cvt_vecmask<_DataType>(__x._M_data);
802  else if constexpr (_UAbi::_S_nreg == 2)
803  {
804  auto __lo = __x._M_data0._M_data;
805  auto __hi = __vec_zero_pad_to<sizeof(__lo)>(
806  __x._M_data1._M_concat_data());
807  return __x86_cvt_vecmask<_DataType>(__lo, __hi);
808  }
809  }
810 #endif
811  return __vec_mask_cast<_DataType>(__x._M_concat_data());
812  }
813  }())
814  {}
815 
816  using _Base::_MaskBase;
817 
818  // [simd.mask.ctor] generator constructor -------------------------------
819  template <__simd_generator_invokable<bool, _S_size> _Fp>
820  [[__gnu__::__always_inline__]]
821  constexpr explicit
822  basic_mask(_Fp&& __gen)
823  : _M_data([&] [[__gnu__::__always_inline__]] {
824  constexpr auto [...__is] = _IotaArray<_S_size>;
825  if constexpr (_S_is_scalar)
826  return __gen(__simd_size_c<0>);
827  else if constexpr (_S_use_bitmask)
828  return _DataType(((_DataType(__gen(__simd_size_c<__is>)) << __is)
829  | ...));
830  else
831  return _DataType{__vec_value_type<_DataType>(
832  __gen(__simd_size_c<__is>) ? -1 : 0)...};
833  }())
834  {}
835 
836  // [simd.mask.ctor] bitset constructor ----------------------------------
837  [[__gnu__::__always_inline__]]
838  constexpr
839  basic_mask(const same_as<bitset<_S_size>> auto& __b) noexcept // LWG 4382.
840  : basic_mask(static_cast<_Bitmask<_S_size>>(__b.to_ullong()))
841  {
842  // more than 64 elements in one register? not yet.
843  static_assert(_S_size <= numeric_limits<unsigned long long>::digits);
844  }
845 
846  // [simd.mask.ctor] uint constructor ------------------------------------
847  template <unsigned_integral _Tp>
848  requires (!same_as<_Tp, bool>) // LWG 4382.
849  [[__gnu__::__always_inline__]]
850  constexpr explicit
851  basic_mask(_Tp __val) noexcept
852  : _M_data([&] [[__gnu__::__always_inline__]] () {
853  if constexpr (_S_use_bitmask)
854  return __val;
855  else if constexpr (_S_is_scalar)
856  return bool(__val & 1);
857  else if (__is_const_known(__val))
858  {
859  constexpr auto [...__is] = _IotaArray<_S_size>;
860  return _DataType {__vec_value_type<_DataType>((__val & (1ull << __is)) == 0
861  ? 0 : -1)...};
862  }
863  else
864  {
865  using _Ip = typename _VecType::value_type;
866  _VecType __v0 = _Ip(__val);
867  constexpr int __bits_per_element = sizeof(_Ip) * __CHAR_BIT__;
868  constexpr _VecType __pow2 = _VecType(cw<1>)
869  << (__iota<_VecType> % cw<__bits_per_element>);
870  if constexpr (_S_size < __bits_per_element)
871  return ((__v0 & __pow2) > cw<0>)._M_concat_data();
872  else if constexpr (_S_size == __bits_per_element)
873  return ((__v0 & __pow2) != cw<0>)._M_concat_data();
874  else
875  {
876  static_assert(_Bytes == 1);
877  static_assert(sizeof(_Ip) == 1);
878  _Bitmask<_S_size> __bits = __val;
879  static_assert(sizeof(_VecType) % sizeof(__bits) == 0);
880  if constexpr (sizeof(_DataType) == 32)
881  {
882  __vec_builtin_type<_UInt<8>, 4> __v1 = {
883  0xffu & (__bits >> (0 * __CHAR_BIT__)),
884  0xffu & (__bits >> (1 * __CHAR_BIT__)),
885  0xffu & (__bits >> (2 * __CHAR_BIT__)),
886  0xffu & (__bits >> (3 * __CHAR_BIT__)),
887  };
888  __v1 *= 0x0101'0101'0101'0101ull;
889  __v0 = __builtin_bit_cast(_VecType, __v1);
890  return ((__v0 & __pow2) != cw<0>)._M_data;
891  }
892  else
893  {
894  using _V1 = vec<_Ip, sizeof(__bits)>;
895  _V1 __v1 = __builtin_bit_cast(_V1, __bits);
896  __v0 = _VecType::_S_static_permute(__v1, [](int __i) {
897  return __i / __CHAR_BIT__;
898  });
899  return ((__v0 & __pow2) != cw<0>)._M_data;
900  }
901  }
902  }
903  }())
904  {}
905 
906  //Effects: Initializes the first M elements to the corresponding bit values in val, where M is
907  //the smaller of size() and the number of bits in the value representation
908  //([basic.types.general]) of the type of val. If M is less than size(), the remaining elements
909  //are initialized to zero.
910 
911 
912  // [simd.mask.subscr] ---------------------------------------------------
913  [[__gnu__::__always_inline__]]
914  constexpr value_type
915  operator[](__simd_size_type __i) const
916  {
917  __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
918  if constexpr (_S_is_scalar)
919  return _M_data;
920  else if constexpr (_S_use_bitmask)
921  return bool((_M_data >> __i) & 1);
922  else
923  return _M_data[__i] & 1;
924  }
925 
926  // [simd.mask.unary] ----------------------------------------------------
927  [[__gnu__::__always_inline__]]
928  constexpr basic_mask
929  operator!() const noexcept
930  {
931  if constexpr (_S_is_scalar)
932  return _S_init(!_M_data);
933  else
934  return _S_init(~_M_data);
935  }
936 
937  [[__gnu__::__always_inline__]]
938  constexpr _VecType
939  operator+() const noexcept requires destructible<_VecType>
940  { return operator _VecType(); }
941 
942  constexpr _VecType
943  operator+() const noexcept = delete;
944 
945  [[__gnu__::__always_inline__]]
946  constexpr _VecType
947  operator-() const noexcept requires destructible<_VecType>
948  {
949  using _Ip = typename _VecType::value_type;
950  if constexpr (_S_is_scalar)
951  return _Ip(-int(_M_data));
952  else if constexpr (_S_use_bitmask)
953  return __select_impl(*this, _Ip(-1), _Ip());
954  else
955  {
956  static_assert(sizeof(_VecType) == sizeof(_M_data));
957  return __builtin_bit_cast(_VecType, _M_data);
958  }
959  }
960 
961  constexpr _VecType
962  operator-() const noexcept = delete;
963 
964  [[__gnu__::__always_inline__]]
965  constexpr _VecType
966  operator~() const noexcept requires destructible<_VecType>
967  {
968  using _Ip = typename _VecType::value_type;
969  if constexpr (_S_is_scalar)
970  return _Ip(~int(_M_data));
971  else if constexpr (_S_use_bitmask)
972  return __select_impl(*this, _Ip(-2), _Ip(-1));
973  else
974  {
975  static_assert(sizeof(_VecType) == sizeof(_M_data));
976  return __builtin_bit_cast(_VecType, _M_data) - _Ip(1);
977  }
978  }
979 
980  constexpr _VecType
981  operator~() const noexcept = delete;
982 
983  // [simd.mask.conv] -----------------------------------------------------
984  template <typename _Up, typename _UAbi>
985  requires (_UAbi::_S_size == _S_size)
986  [[__gnu__::__always_inline__]]
987  constexpr explicit(sizeof(_Up) != _Bytes)
988  operator basic_vec<_Up, _UAbi>() const noexcept
989  {
990  if constexpr (_S_is_scalar)
991  return _Up(_M_data);
992  else
993  {
994  using _UV = basic_vec<_Up, _UAbi>;
995  return __select_impl(static_cast<_UV::mask_type>(*this), _Up(1), _UV());
996  }
997  }
998 
999  using _Base::operator basic_vec;
1000 
1001  // [simd.mask.namedconv] ------------------------------------------------
1002  [[__gnu__::__always_inline__]]
1003  constexpr bitset<_S_size>
1004  to_bitset() const noexcept
1005  {
1006  // more than 64 elements in one register? not yet.
1007  static_assert(_S_size <= numeric_limits<unsigned long long>::digits);
1008  return to_ullong();
1009  }
1010 
1011  /** @internal
1012  * Return the mask as the smallest possible unsigned integer (up to 64 bits).
1013  *
1014  * @tparam _Offset Adjust the return type & value to start at bit @p _Offset.
1015  * @tparam _Use_2_for_1 Store the value of every second element into one bit of the result.
1016  * (precondition: each even/odd pair stores the same value)
1017  */
1018  template <int _Offset = 0, _ArchTraits _Traits = {}>
1019  [[__gnu__::__always_inline__]]
1020  constexpr _Bitmask<_S_size + _Offset>
1021  _M_to_uint() const
1022  {
1023  constexpr int __nbits = _S_size;
1024  static_assert(__nbits + _Offset <= numeric_limits<unsigned long long>::digits);
1025  // before shifting
1026  using _U0 = _Bitmask<__nbits>;
1027  // potentially wider type needed for shift by _Offset
1028  using _Ur = _Bitmask<__nbits + _Offset>;
1029  if constexpr (_S_is_scalar || _S_use_bitmask)
1030  {
1031  auto __bits = _M_data;
1032  if constexpr (_S_is_partial)
1033  __bits &= _S_implicit_mask;
1034  return _Ur(__bits) << _Offset;
1035  }
1036  else
1037  {
1038 #if _GLIBCXX_X86
1039  if (!__is_const_known(*this))
1040  {
1041  _U0 __uint;
1042  if constexpr (_Bytes != 2) // movmskb would duplicate each bit
1043  __uint = _U0(__x86_movmsk(_M_data));
1044  else if constexpr (_Bytes == 2 && _Traits._M_have_bmi2())
1045  __uint = __bit_extract_even<__nbits>(__x86_movmsk(_M_data));
1046  else if constexpr (_Bytes == 2)
1047  return __similar_mask<char, __nbits, _Ap>(*this).template _M_to_uint<_Offset>();
1048  else
1049  static_assert(false);
1050  // TODO: with AVX512 use __builtin_ia32_cvt[bwdq]2mask(128|256|512)
1051  // TODO: Ask for compiler builtin to do the best of the above. This should also
1052  // combine with a preceding vector-mask compare to produce a bit-mask compare (on
1053  // AVX512)
1054  if constexpr (_S_is_partial)
1055  __uint &= (_U0(1) << _S_size) - 1;
1056  return _Ur(__uint) << _Offset;
1057  }
1058 #endif
1059  using _IV = _VecType;
1060  static_assert(destructible<_IV>);
1061  const typename _IV::mask_type& __k = [&] [[__gnu__::__always_inline__]] () {
1062  if constexpr (is_same_v<typename _IV::mask_type, basic_mask>)
1063  return *this;
1064  else
1065  return typename _IV::mask_type(*this);
1066  }();
1067  constexpr int __n = _IV::size();
1068  if constexpr (_Bytes * __CHAR_BIT__ >= __n) // '1 << __iota' cannot overflow
1069  { // reduce(select(k, powers_of_2, 0))
1070  constexpr _IV __pow2 = _IV(cw<1>) << __iota<_IV>;
1071  return _Ur(_U0(__select_impl(__k, __pow2, _IV())
1072  ._M_reduce(bit_or<>()))) << _Offset;
1073  }
1074  else if constexpr (__n % __CHAR_BIT__ != 0)
1075  { // recurse after splitting in two
1076  constexpr int __n_lo = __n - __n % __CHAR_BIT__;
1077  const auto [__lo, __hi] = chunk<__n_lo>(__k);
1078  _Ur __bits = __hi.template _M_to_uint<_Offset + __n_lo>();
1079  return __bits | __lo.template _M_to_uint<_Offset>();
1080  }
1081  else
1082  { // limit powers_of_2 to 1, 2, 4, ..., 128
1083  constexpr _IV __pow2 = _IV(cw<1>) << (__iota<_IV> % _IV(cw<__CHAR_BIT__>));
1084  _IV __x = __select_impl(__k, __pow2, _IV());
1085  // partial reductions of 8 neighboring elements
1086  __x |= _IV::_S_static_permute(__x, _SwapNeighbors<4>());
1087  __x |= _IV::_S_static_permute(__x, _SwapNeighbors<2>());
1088  __x |= _IV::_S_static_permute(__x, _SwapNeighbors<1>());
1089  // permute partial reduction results to the front
1090  __x = _IV::_S_static_permute(__x, [](int __i) {
1091  return __i * 8 < __n ? __i * 8 : uninit_element;
1092  });
1093  // extract front as scalar unsigned
1094  _U0 __bits = __builtin_bit_cast(
1095  __similar_vec<_U0, __n * _Bytes / sizeof(_U0), _Ap>, __x)[0];
1096  // mask off unused bits
1097  if constexpr (!__has_single_bit(unsigned(__nbits)))
1098  __bits &= (_U0(1) << __nbits) - 1;
1099  return _Ur(__bits) << _Offset;
1100  }
1101  }
1102  }
1103 
1104  [[__gnu__::__always_inline__]]
1105  constexpr unsigned long long
1106  to_ullong() const
1107  { return _M_to_uint(); }
1108 
1109  // [simd.mask.binary] ---------------------------------------------------
1110  [[__gnu__::__always_inline__]]
1111  friend constexpr basic_mask
1112  operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
1113  { return _S_init(__x._M_data & __y._M_data); }
1114 
1115  [[__gnu__::__always_inline__]]
1116  friend constexpr basic_mask
1117  operator||(const basic_mask& __x, const basic_mask& __y) noexcept
1118  { return _S_init(__x._M_data | __y._M_data); }
1119 
1120  [[__gnu__::__always_inline__]]
1121  friend constexpr basic_mask
1122  operator&(const basic_mask& __x, const basic_mask& __y) noexcept
1123  { return _S_init(__x._M_data & __y._M_data); }
1124 
1125  [[__gnu__::__always_inline__]]
1126  friend constexpr basic_mask
1127  operator|(const basic_mask& __x, const basic_mask& __y) noexcept
1128  { return _S_init(__x._M_data | __y._M_data); }
1129 
1130  [[__gnu__::__always_inline__]]
1131  friend constexpr basic_mask
1132  operator^(const basic_mask& __x, const basic_mask& __y) noexcept
1133  { return _S_init(__x._M_data ^ __y._M_data); }
1134 
1135  // [simd.mask.cassign] --------------------------------------------------
1136  [[__gnu__::__always_inline__]]
1137  friend constexpr basic_mask&
1138  operator&=(basic_mask& __x, const basic_mask& __y) noexcept
1139  {
1140  __x._M_data &= __y._M_data;
1141  return __x;
1142  }
1143 
1144  [[__gnu__::__always_inline__]]
1145  friend constexpr basic_mask&
1146  operator|=(basic_mask& __x, const basic_mask& __y) noexcept
1147  {
1148  __x._M_data |= __y._M_data;
1149  return __x;
1150  }
1151 
1152  [[__gnu__::__always_inline__]]
1153  friend constexpr basic_mask&
1154  operator^=(basic_mask& __x, const basic_mask& __y) noexcept
1155  {
1156  __x._M_data ^= __y._M_data;
1157  return __x;
1158  }
1159 
1160  // [simd.mask.comparison] -----------------------------------------------
1161  [[__gnu__::__always_inline__]]
1162  friend constexpr basic_mask
1163  operator==(const basic_mask& __x, const basic_mask& __y) noexcept
1164  { return !(__x ^ __y); }
1165 
1166  [[__gnu__::__always_inline__]]
1167  friend constexpr basic_mask
1168  operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
1169  { return __x ^ __y; }
1170 
1171  [[__gnu__::__always_inline__]]
1172  friend constexpr basic_mask
1173  operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
1174  { return __x || !__y; }
1175 
1176  [[__gnu__::__always_inline__]]
1177  friend constexpr basic_mask
1178  operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
1179  { return !__x || __y; }
1180 
1181  [[__gnu__::__always_inline__]]
1182  friend constexpr basic_mask
1183  operator>(const basic_mask& __x, const basic_mask& __y) noexcept
1184  { return __x && !__y; }
1185 
1186  [[__gnu__::__always_inline__]]
1187  friend constexpr basic_mask
1188  operator<(const basic_mask& __x, const basic_mask& __y) noexcept
1189  { return !__x && __y; }
1190 
1191  // [simd.mask.cond] -----------------------------------------------------
1192  [[__gnu__::__always_inline__]]
1193  friend constexpr basic_mask
1194  __select_impl(const basic_mask& __k, const basic_mask& __t, const basic_mask& __f) noexcept
1195  {
1196  if constexpr (!_S_use_bitmask)
1197  {
1198 #if _GLIBCXX_X86
1199  // this works around bad code-gen when the compiler can't see that __k is a vector-mask.
1200  // This pattern, is recognized to match the x86 blend instructions, which only consider
1201  // the sign bit of the mask register. Also, without SSE4, if the compiler knows that __k
1202  // is a vector-mask, then the '< 0' is elided.
1203  return __k._M_data < 0 ? __t._M_data : __f._M_data;
1204 #endif
1205  return __k._M_data ? __t._M_data : __f._M_data;
1206  }
1207  else
1208  return (__k._M_data & __t._M_data) | (~__k._M_data & __f._M_data);
1209  }
1210 
1211  [[__gnu__::__always_inline__]]
1212  friend constexpr basic_mask
1213  __select_impl(const basic_mask& __k, same_as<bool> auto __t, same_as<bool> auto __f) noexcept
1214  {
1215  if (__t == __f)
1216  return basic_mask(__t);
1217  else
1218  return __t ? __k : !__k;
1219  }
1220 
1221  template <__vectorizable _T0, same_as<_T0> _T1>
1222  requires (sizeof(_T0) == _Bytes)
1223  [[__gnu__::__always_inline__]]
1224  friend constexpr vec<_T0, _S_size>
1225  __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f) noexcept
1226  {
1227  if constexpr (_S_is_scalar)
1228  return __k._M_data ? __t : __f;
1229  else
1230  {
1231  using _Vp = vec<_T0, _S_size>;
1232  using _Mp = typename _Vp::mask_type;
1233  return __select_impl(_Mp(__k), _Vp(__t), _Vp(__f));
1234  }
1235  }
1236 
1237  // [simd.mask.reductions] implementation --------------------------------
1238  [[__gnu__::__always_inline__]]
1239  constexpr bool
1240  _M_all_of() const noexcept
1241  {
1242  if constexpr (_S_is_scalar)
1243  return _M_data;
1244  else if constexpr (_S_use_bitmask)
1245  {
1246  if constexpr (_S_is_partial)
1247  // PR120925 (partial kortest pattern not recognized)
1248  return (_M_data & _S_implicit_mask) == _S_implicit_mask;
1249  else
1250  return _M_data == _S_implicit_mask;
1251  }
1252 #if _GLIBCXX_X86
1253  else if (!__is_const_known(_M_data))
1254  return __x86_vecmask_all<_S_size>(_M_data);
1255 #endif
1256  else
1257  return _VecOps<_DataType, _S_size>::_S_all_of(_M_data);
1258  }
1259 
1260  [[__gnu__::__always_inline__]]
1261  constexpr bool
1262  _M_any_of() const noexcept
1263  {
1264  if constexpr (_S_is_scalar)
1265  return _M_data;
1266  else if constexpr (_S_use_bitmask)
1267  {
1268  if constexpr (_S_is_partial)
1269  // PR120925 (partial kortest pattern not recognized)
1270  return (_M_data & _S_implicit_mask) != 0;
1271  else
1272  return _M_data != 0;
1273  }
1274 #if _GLIBCXX_X86
1275  else if (!__is_const_known(_M_data))
1276  return __x86_vecmask_any<_S_size>(_M_data);
1277 #endif
1278  else
1279  return _VecOps<_DataType, _S_size>::_S_any_of(_M_data);
1280  }
1281 
1282  [[__gnu__::__always_inline__]]
1283  constexpr bool
1284  _M_none_of() const noexcept
1285  {
1286  if constexpr (_S_is_scalar)
1287  return !_M_data;
1288  else if constexpr (_S_use_bitmask)
1289  {
1290  if constexpr (_S_is_partial)
1291  // PR120925 (partial kortest pattern not recognized)
1292  return (_M_data & _S_implicit_mask) == 0;
1293  else
1294  return _M_data == 0;
1295  }
1296 #if _GLIBCXX_X86
1297  else if (!__is_const_known(_M_data))
1298  return __x86_vecmask_none<_S_size>(_M_data);
1299 #endif
1300  else
1301  return _VecOps<_DataType, _S_size>::_S_none_of(_M_data);
1302  }
1303 
1304  [[__gnu__::__always_inline__]]
1305  constexpr __simd_size_type
1306  _M_reduce_count() const noexcept
1307  {
1308  if constexpr (_S_is_scalar)
1309  return int(_M_data);
1310  else if constexpr (_S_size <= numeric_limits<unsigned>::digits)
1311  return __builtin_popcount(_M_to_uint());
1312  else
1313  return __builtin_popcountll(to_ullong());
1314  }
1315 
1316  [[__gnu__::__always_inline__]]
1317  constexpr __simd_size_type
1318  _M_reduce_min_index() const
1319  {
1320  const auto __bits = _M_to_uint();
1321  __glibcxx_simd_precondition(__bits, "An empty mask does not have a min_index.");
1322  if constexpr (_S_size == 1)
1323  return 0;
1324  else
1325  return __countr_zero(__bits);
1326  }
1327 
1328  [[__gnu__::__always_inline__]]
1329  constexpr __simd_size_type
1330  _M_reduce_max_index() const
1331  {
1332  const auto __bits = _M_to_uint();
1333  __glibcxx_simd_precondition(__bits, "An empty mask does not have a max_index.");
1334  if constexpr (_S_size == 1)
1335  return 0;
1336  else
1337  return __highest_bit(__bits);
1338  }
1339 
1340  [[__gnu__::__always_inline__]]
1341  friend constexpr bool
1342  __is_const_known(const basic_mask& __x)
1343  { return __builtin_constant_p(__x._M_data); }
1344  };
1345 
1346  template <size_t _Bytes, __abi_tag _Ap>
1347  requires (_Ap::_S_nreg > 1)
1348  class basic_mask<_Bytes, _Ap>
1349  : public _MaskBase<_Bytes, _Ap>
1350  {
1351  using _Base = _MaskBase<_Bytes, _Ap>;
1352 
1353  using _VecType = _Base::_VecType;
1354 
1355  template <size_t, typename>
1356  friend class basic_mask;
1357 
1358  template <typename, typename>
1359  friend class basic_vec;
1360 
1361  static constexpr int _S_size = _Ap::_S_size;
1362 
1363  static constexpr int _N0 = __bit_ceil(unsigned(_S_size)) / 2;
1364 
1365  static constexpr int _N1 = _S_size - _N0;
1366 
1367  static constexpr int _Nreg0 = __bit_ceil(unsigned(_Ap::_S_nreg)) / 2;
1368 
1369  static constexpr int _Nreg1 = _Ap::_S_nreg - _Nreg0;
1370 
1371  // explicitly request _Nreg0 rather than use __abi_rebind. This way _Float16 can use half
1372  // of native registers (since they convert to full float32 registers).
1373  using _Abi0 = decltype(_Ap::template _S_resize<_N0, _Nreg0>());
1374 
1375  using _Abi1 = decltype(_Ap::template _S_resize<_N1, _Nreg1>());
1376 
1377  using _Mask0 = basic_mask<_Bytes, _Abi0>;
1378 
1379  // the implementation (and users) depend on elements being contiguous in memory
1380  static_assert(_Mask0::_S_padding_bytes == 0 && !_Mask0::_S_is_partial);
1381 
1382  using _Mask1 = basic_mask<_Bytes, _Abi1>;
1383 
1384  static constexpr bool _S_is_partial = _Mask1::_S_is_partial;
1385 
1386  // _Ap::_S_nreg determines how deep the recursion goes. E.g. basic_mask<4, _Abi<8, 4>> cannot
1387  // use basic_mask<4, _Abi<4, 1>> as _Mask0/1 types.
1388  static_assert(_Mask0::abi_type::_S_nreg + _Mask1::abi_type::_S_nreg == _Ap::_S_nreg);
1389 
1390  static constexpr bool _S_use_bitmask = _Mask0::_S_use_bitmask;
1391 
1392  static constexpr bool _S_is_scalar = _Mask0::_S_is_scalar;
1393 
1394  _Mask0 _M_data0;
1395 
1396  _Mask1 _M_data1;
1397 
1398  static constexpr bool _S_has_bool_member = _Mask1::_S_has_bool_member;
1399 
1400  // by construction _N0 >= _N1
1401  // => sizeof(_Mask0) >= sizeof(_Mask1)
1402  // and __alignof__(_Mask0) >= __alignof__(_Mask1)
1403  static constexpr size_t _S_padding_bytes
1404  = (__alignof__(_Mask0) == __alignof__(_Mask1)
1405  ? 0 : __alignof__(_Mask0) - (sizeof(_Mask1) % __alignof__(_Mask0)))
1406  + _Mask1::_S_padding_bytes;
1407 
1408  public:
1409  using value_type = bool;
1410 
1411  using abi_type = _Ap;
1412 
1413  using iterator = _Base::iterator;
1414 
1415  using const_iterator = _Base::const_iterator;
1416 
1417  [[__gnu__::__always_inline__]]
1418  static constexpr basic_mask
1419  _S_init(const _Mask0& __x, const _Mask1& __y)
1420  {
1421  basic_mask __r;
1422  __r._M_data0 = __x;
1423  __r._M_data1 = __y;
1424  return __r;
1425  }
1426 
1427  [[__gnu__::__always_inline__]]
1428  static constexpr basic_mask
1429  _S_init(unsigned_integral auto __bits)
1430  { return basic_mask(__bits); }
1431 
1432  template <typename _U0, typename _U1>
1433  [[__gnu__::__always_inline__]]
1434  static constexpr basic_mask
1435  _S_init(const __trivial_pair<_U0, _U1>& __bits)
1436  {
1437  if constexpr (is_unsigned_v<_U0>)
1438  {
1439  static_assert(is_unsigned_v<_U1>);
1440  return _S_init(_Mask0(__bits._M_first), _Mask1(__bits._M_second));
1441  }
1442  else if constexpr (is_unsigned_v<_U1>)
1443  return _S_init(_Mask0::_S_init(__bits._M_first), _Mask1(__bits._M_second));
1444  else
1445  return _S_init(_Mask0::_S_init(__bits._M_first), _Mask1::_S_init(__bits._M_second));
1446  }
1447 
1448  [[__gnu__::__always_inline__]]
1449  constexpr const _Mask0&
1450  _M_get_low() const
1451  { return _M_data0; }
1452 
1453  [[__gnu__::__always_inline__]]
1454  constexpr const _Mask1&
1455  _M_get_high() const
1456  { return _M_data1; }
1457 
1458  template <size_t _UBytes, typename _UAbi>
1459  [[__gnu__::__always_inline__]]
1460  static constexpr basic_mask
1461  _S_recursive_bit_cast(const basic_mask<_UBytes, _UAbi>& __x)
1462  {
1463  using _Mp = basic_mask<_UBytes, _UAbi>;
1464  if constexpr (_Mp::_S_has_bool_member || sizeof(basic_mask) > sizeof(__x)
1465  || _Mp::_S_padding_bytes != 0)
1466  return _S_init(__builtin_bit_cast(_Mask0, __x._M_data0),
1467  _Mask1::_S_recursive_bit_cast(__x._M_data1));
1468  else if constexpr (sizeof(basic_mask) == sizeof(__x))
1469  return __builtin_bit_cast(basic_mask, __x);
1470  else
1471  { // e.g. on IvyBridge (different alignment => different sizeof)
1472  struct _Tmp { alignas(_Mp) basic_mask _M_data; };
1473  return __builtin_bit_cast(_Tmp, __x)._M_data;
1474  }
1475  }
1476 
1477  [[__gnu__::__always_inline__]]
1478  constexpr auto
1479  _M_concat_data(bool __do_sanitize = _S_is_partial) const
1480  {
1481  if constexpr (_S_use_bitmask)
1482  {
1483  static_assert(_S_size <= numeric_limits<unsigned long long>::digits,
1484  "cannot concat more than 64 bits");
1485  using _Up = _Bitmask<_S_size>;
1486  return _Up(_M_data0._M_concat_data() | (_Up(_M_data1._M_concat_data(__do_sanitize)) << _N0));
1487  }
1488  else
1489  {
1490  auto __lo = _M_data0._M_concat_data();
1491  auto __hi = __vec_zero_pad_to<sizeof(__lo)>(_M_data1._M_concat_data(__do_sanitize));
1492  return __vec_concat(__lo, __hi);
1493  }
1494  }
1495 
1496  template <_ArchTraits _Traits = {}>
1497  [[__gnu__::__always_inline__]]
1498  static constexpr basic_mask
1499  _S_partial_mask_of_n(int __n)
1500  {
1501 #if __has_builtin(__builtin_ia32_bzhi_di)
1502  if constexpr (_S_use_bitmask && _S_size <= 64 && _Traits._M_have_bmi2())
1503  return basic_mask(__builtin_ia32_bzhi_di(~0ull >> (64 - _S_size), unsigned(__n)));
1504 #endif
1505  if constexpr (_N0 == 1)
1506  {
1507  static_assert(_S_size == 2); // => __n == 1
1508  return _S_init(_Mask0(true), _Mask1(false));
1509  }
1510  else if (__n < _N0)
1511  return _S_init(_Mask0::_S_partial_mask_of_n(__n), _Mask1(false));
1512  else if (__n == _N0 || _N1 == 1)
1513  return _S_init(_Mask0(true), _Mask1(false));
1514  else if constexpr (_N1 != 1)
1515  return _S_init(_Mask0(true), _Mask1::_S_partial_mask_of_n(__n - _N0));
1516  }
1517 
1518  [[__gnu__::__always_inline__]]
1519  constexpr basic_mask&
1520  _M_and_neighbors()
1521  {
1522  if constexpr (_S_size == 2)
1523  {
1524  static_assert(_S_is_scalar);
1525  _M_data0 = _M_data1 = _M_data0 && _M_data1;
1526  }
1527  else
1528  {
1529  _M_data0._M_and_neighbors();
1530  _M_data1._M_and_neighbors();
1531  }
1532  return *this;
1533  }
1534 
1535  [[__gnu__::__always_inline__]]
1536  constexpr basic_mask&
1537  _M_or_neighbors()
1538  {
1539  if constexpr (_S_size == 2)
1540  {
1541  static_assert(_S_is_scalar);
1542  _M_data0 = _M_data1 = _M_data0 || _M_data1;
1543  }
1544  else
1545  {
1546  _M_data0._M_or_neighbors();
1547  _M_data1._M_or_neighbors();
1548  }
1549  return *this;
1550  }
1551 
1552  template <typename _Mp>
1553  [[__gnu__::__always_inline__]]
1554  constexpr auto
1555  _M_chunk() const noexcept
1556  {
1557  constexpr int __n = _S_size / _Mp::_S_size;
1558  constexpr int __rem = _S_size % _Mp::_S_size;
1559  constexpr auto [...__is] = _IotaArray<__n>;
1560  if constexpr (__rem == 0)
1561  return array<_Mp, __n>{__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>,
1562  _M_data0, _M_data1)...};
1563  else
1564  {
1565  using _Rest = resize_t<__rem, _Mp>;
1566  return tuple(__extract_simd_at<_Mp>(cw<_Mp::_S_size * __is>, _M_data0, _M_data1)...,
1567  __extract_simd_at<_Rest>(cw<_Mp::_S_size * __n>, _M_data0, _M_data1));
1568  }
1569  }
1570 
1571  [[__gnu__::__always_inline__]]
1572  static constexpr basic_mask
1573  _S_concat(const basic_mask& __x0) noexcept
1574  { return __x0; }
1575 
1576  template <typename... _As>
1577  requires (sizeof...(_As) >= 2)
1578  [[__gnu__::__always_inline__]]
1579  static constexpr basic_mask
1580  _S_concat(const basic_mask<_Bytes, _As>&... __xs) noexcept
1581  {
1582  static_assert(_S_size == (_As::_S_size + ...));
1583  return _S_init(__extract_simd_at<_Mask0>(cw<0>, __xs...),
1584  __extract_simd_at<_Mask1>(cw<_N0>, __xs...));
1585  }
1586 
1587  // [simd.mask.overview] default constructor -----------------------------
1588  basic_mask() = default;
1589 
1590  // [simd.mask.overview] conversion extensions ---------------------------
1591  // TODO: any?
1592 
1593  // [simd.mask.ctor] broadcast constructor -------------------------------
1594  [[__gnu__::__always_inline__]]
1595  constexpr explicit
1596  basic_mask(same_as<bool> auto __x) noexcept // LWG 4382.
1597  : _M_data0(__x), _M_data1(__x)
1598  {}
1599 
1600  // [simd.mask.ctor] conversion constructor ------------------------------
1601  template <size_t _UBytes, typename _UAbi>
1602  requires (_S_size == _UAbi::_S_size)
1603  [[__gnu__::__always_inline__]]
1604  constexpr explicit(__is_mask_conversion_explicit<_Ap, _UAbi>(_Bytes, _UBytes))
1605  basic_mask(const basic_mask<_UBytes, _UAbi>& __x) noexcept
1606  : _M_data0([&] {
1607  if constexpr (_UAbi::_S_nreg > 1)
1608  {
1609  return __x._M_data0;
1610  }
1611  else if constexpr (_N0 == 1)
1612  return _Mask0(__x[0]);
1613  else
1614  return get<0>(chunk<_N0>(__x));
1615  }()),
1616  _M_data1([&] {
1617  if constexpr (_UAbi::_S_nreg > 1)
1618  {
1619  return __x._M_data1;
1620  }
1621  else if constexpr (_N1 == 1)
1622  return _Mask1(__x[_N0]);
1623  else
1624  return get<1>(chunk<_N0>(__x));
1625  }())
1626  {}
1627 
1628  using _Base::_MaskBase;
1629 
1630  // [simd.mask.ctor] generator constructor -------------------------------
1631  template <__simd_generator_invokable<bool, _S_size> _Fp>
1632  [[__gnu__::__always_inline__]]
1633  constexpr explicit
1634  basic_mask(_Fp&& __gen)
1635  : _M_data0(__gen), _M_data1([&] [[__gnu__::__always_inline__]] (auto __i) {
1636  return __gen(__simd_size_c<__i + _N0>);
1637  })
1638  {}
1639 
1640  // [simd.mask.ctor] bitset constructor ----------------------------------
1641  [[__gnu__::__always_inline__]]
1642  constexpr
1643  basic_mask(const same_as<bitset<_S_size>> auto& __b) noexcept // LWG 4382.
1644  : _M_data0(__bitset_split<_N0>(__b)._M_lo), _M_data1(__bitset_split<_N0>(__b)._M_hi)
1645  {}
1646 
1647  // [simd.mask.ctor] uint constructor ------------------------------------------
1648  template <unsigned_integral _Tp>
1649  requires (!same_as<_Tp, bool>) // LWG 4382.
1650  [[__gnu__::__always_inline__]]
1651  constexpr explicit
1652  basic_mask(_Tp __val) noexcept
1653  : _M_data0(static_cast<_Bitmask<_N0>>(__val)),
1654  _M_data1(sizeof(_Tp) * __CHAR_BIT__ > _N0
1655  ? static_cast<_Bitmask<_N1>>(__val >> _N0) : _Bitmask<_N1>())
1656  {}
1657 
1658  // [simd.mask.subscr] ---------------------------------------------------
1659  [[__gnu__::__always_inline__]]
1660  constexpr value_type
1661  operator[](__simd_size_type __i) const
1662  {
1663  __glibcxx_simd_precondition(__i >= 0 && __i < _S_size, "subscript is out of bounds");
1664  if (__is_const_known(__i))
1665  return __i < _N0 ? _M_data0[__i] : _M_data1[__i - _N0];
1666  else if constexpr (_M_data1._S_has_bool_member)
1667  // in some cases the last element can be 'bool' instead of bit-/vector-mask;
1668  // e.g. mask<short, 17> is {mask<short, 16>, mask<short, 1>}, where the latter uses
1669  // _Abi<1, 1>, which is stored as 'bool'
1670  return __i < _N0 ? _M_data0[__i] : _M_data1[__i - _N0];
1671  else if constexpr (abi_type::_S_is_bitmask)
1672  {
1673  using _AliasingByte [[__gnu__::__may_alias__]] = unsigned char;
1674  return bool((reinterpret_cast<const _AliasingByte*>(this)
1675  [__i / __CHAR_BIT__] >> (__i % __CHAR_BIT__)) & 1);
1676  }
1677  else
1678  {
1679  using _AliasingInt [[__gnu__::__may_alias__]] = __integer_from<_Bytes>;
1680  return reinterpret_cast<const _AliasingInt*>(this)[__i] != 0;
1681  }
1682  }
1683 
1684  // [simd.mask.unary] ----------------------------------------------------
1685  [[__gnu__::__always_inline__]]
1686  constexpr basic_mask
1687  operator!() const noexcept
1688  { return _S_init(!_M_data0, !_M_data1); }
1689 
1690  [[__gnu__::__always_inline__]]
1691  constexpr _VecType
1692  operator+() const noexcept requires destructible<_VecType>
1693  { return _VecType::_S_concat(+_M_data0, +_M_data1); }
1694 
1695  constexpr _VecType
1696  operator+() const noexcept = delete;
1697 
1698  [[__gnu__::__always_inline__]]
1699  constexpr _VecType
1700  operator-() const noexcept requires destructible<_VecType>
1701  { return _VecType::_S_concat(-_M_data0, -_M_data1); }
1702 
1703  constexpr _VecType
1704  operator-() const noexcept = delete;
1705 
1706  [[__gnu__::__always_inline__]]
1707  constexpr _VecType
1708  operator~() const noexcept requires destructible<_VecType>
1709  { return _VecType::_S_concat(~_M_data0, ~_M_data1); }
1710 
1711  constexpr _VecType
1712  operator~() const noexcept = delete;
1713 
1714  // [simd.mask.conv] -----------------------------------------------------
1715  template <typename _Up, typename _UAbi>
1716  requires (_UAbi::_S_size == _S_size)
1717  [[__gnu__::__always_inline__]]
1718  constexpr explicit(sizeof(_Up) != _Bytes)
1719  operator basic_vec<_Up, _UAbi>() const noexcept
1720  {
1721  using _Rp = basic_vec<_Up, _UAbi>;
1722  return _Rp::_S_init(static_cast<_Rp::_DataType0>(_M_data0),
1723  static_cast<_Rp::_DataType1>(_M_data1));
1724  }
1725 
1726  using _Base::operator basic_vec;
1727 
1728  // [simd.mask.namedconv] ------------------------------------------------
1729  [[__gnu__::__always_inline__]]
1730  constexpr bitset<_S_size>
1731  to_bitset() const noexcept
1732  {
1733  if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
1734  return to_ullong();
1735  else
1736  {
1737  static_assert(_N0 % numeric_limits<unsigned long long>::digits == 0);
1738  struct _Tmp
1739  {
1740  bitset<_N0> _M_lo;
1741  bitset<_N1> _M_hi;
1742  } __tmp = {_M_data0.to_bitset(), _M_data1.to_bitset()};
1743  return __builtin_bit_cast(bitset<_S_size>, __tmp);
1744  }
1745  }
1746 
1747  template <int _Offset = 0, _ArchTraits _Traits = {}>
1748  [[__gnu__::__always_inline__]]
1749  constexpr auto
1750  _M_to_uint() const
1751  {
1752  constexpr int _N0x = _N0;
1753  if constexpr (_N0x >= numeric_limits<unsigned long long>::digits)
1754  {
1755  static_assert(_Offset == 0);
1756  return __trivial_pair {
1757  _M_data0.template _M_to_uint<0>(),
1758  _M_data1.template _M_to_uint<0>()
1759  };
1760  }
1761  else
1762  {
1763 #if _GLIBCXX_X86
1764  if constexpr (_Bytes == 2 && !_Traits._M_have_bmi2() && _Ap::_S_nreg == 2
1765  && !_S_use_bitmask)
1766  return __similar_mask<char, _S_size, _Ap>(*this).template _M_to_uint<_Offset>();
1767 #endif
1768  auto __uint = _M_data1.template _M_to_uint<_N0x + _Offset>();
1769  __uint |= _M_data0.template _M_to_uint<_Offset>();
1770  return __uint;
1771  }
1772  }
1773 
1774  [[__gnu__::__always_inline__]]
1775  constexpr unsigned long long
1776  to_ullong() const
1777  {
1778  if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
1779  return _M_to_uint();
1780  else
1781  {
1782  __glibcxx_simd_precondition(_M_data1.to_ullong() == 0,
1783  "to_ullong called on mask with 'true' elements at indices"
1784  "higher than representable in a ullong");
1785  return _M_data0.to_ullong();
1786  }
1787  }
1788 
1789  // [simd.mask.binary]
1790  [[__gnu__::__always_inline__]]
1791  friend constexpr basic_mask
1792  operator&&(const basic_mask& __x, const basic_mask& __y) noexcept
1793  { return _S_init(__x._M_data0 && __y._M_data0, __x._M_data1 && __y._M_data1); }
1794 
1795  [[__gnu__::__always_inline__]]
1796  friend constexpr basic_mask
1797  operator||(const basic_mask& __x, const basic_mask& __y) noexcept
1798  { return _S_init(__x._M_data0 || __y._M_data0, __x._M_data1 || __y._M_data1); }
1799 
1800  [[__gnu__::__always_inline__]]
1801  friend constexpr basic_mask
1802  operator&(const basic_mask& __x, const basic_mask& __y) noexcept
1803  { return _S_init(__x._M_data0 & __y._M_data0, __x._M_data1 & __y._M_data1); }
1804 
1805  [[__gnu__::__always_inline__]]
1806  friend constexpr basic_mask
1807  operator|(const basic_mask& __x, const basic_mask& __y) noexcept
1808  { return _S_init(__x._M_data0 | __y._M_data0, __x._M_data1 | __y._M_data1); }
1809 
1810  [[__gnu__::__always_inline__]]
1811  friend constexpr basic_mask
1812  operator^(const basic_mask& __x, const basic_mask& __y) noexcept
1813  { return _S_init(__x._M_data0 ^ __y._M_data0, __x._M_data1 ^ __y._M_data1); }
1814 
1815  // [simd.mask.cassign]
1816  [[__gnu__::__always_inline__]]
1817  friend constexpr basic_mask&
1818  operator&=(basic_mask& __x, const basic_mask& __y) noexcept
1819  {
1820  __x._M_data0 &= __y._M_data0;
1821  __x._M_data1 &= __y._M_data1;
1822  return __x;
1823  }
1824 
1825  [[__gnu__::__always_inline__]]
1826  friend constexpr basic_mask&
1827  operator|=(basic_mask& __x, const basic_mask& __y) noexcept
1828  {
1829  __x._M_data0 |= __y._M_data0;
1830  __x._M_data1 |= __y._M_data1;
1831  return __x;
1832  }
1833 
1834  [[__gnu__::__always_inline__]]
1835  friend constexpr basic_mask&
1836  operator^=(basic_mask& __x, const basic_mask& __y) noexcept
1837  {
1838  __x._M_data0 ^= __y._M_data0;
1839  __x._M_data1 ^= __y._M_data1;
1840  return __x;
1841  }
1842 
1843  // [simd.mask.comparison] -----------------------------------------------
1844  [[__gnu__::__always_inline__]]
1845  friend constexpr basic_mask
1846  operator==(const basic_mask& __x, const basic_mask& __y) noexcept
1847  { return !(__x ^ __y); }
1848 
1849  [[__gnu__::__always_inline__]]
1850  friend constexpr basic_mask
1851  operator!=(const basic_mask& __x, const basic_mask& __y) noexcept
1852  { return __x ^ __y; }
1853 
1854  [[__gnu__::__always_inline__]]
1855  friend constexpr basic_mask
1856  operator>=(const basic_mask& __x, const basic_mask& __y) noexcept
1857  { return __x || !__y; }
1858 
1859  [[__gnu__::__always_inline__]]
1860  friend constexpr basic_mask
1861  operator<=(const basic_mask& __x, const basic_mask& __y) noexcept
1862  { return !__x || __y; }
1863 
1864  [[__gnu__::__always_inline__]]
1865  friend constexpr basic_mask
1866  operator>(const basic_mask& __x, const basic_mask& __y) noexcept
1867  { return __x && !__y; }
1868 
1869  [[__gnu__::__always_inline__]]
1870  friend constexpr basic_mask
1871  operator<(const basic_mask& __x, const basic_mask& __y) noexcept
1872  { return !__x && __y; }
1873 
1874  // [simd.mask.cond] -----------------------------------------------------
1875  [[__gnu__::__always_inline__]]
1876  friend constexpr basic_mask
1877  __select_impl(const basic_mask& __k, const basic_mask& __t, const basic_mask& __f) noexcept
1878  {
1879  return _S_init(__select_impl(__k._M_data0, __t._M_data0, __f._M_data0),
1880  __select_impl(__k._M_data1, __t._M_data1, __f._M_data1));
1881  }
1882 
1883  [[__gnu__::__always_inline__]]
1884  friend constexpr basic_mask
1885  __select_impl(const basic_mask& __k, same_as<bool> auto __t, same_as<bool> auto __f) noexcept
1886  {
1887  if (__t == __f)
1888  return basic_mask(__t);
1889  else
1890  return __t ? __k : !__k;
1891  }
1892 
1893  template <__vectorizable _T0, same_as<_T0> _T1>
1894  requires (sizeof(_T0) == _Bytes)
1895  [[__gnu__::__always_inline__]]
1896  friend constexpr vec<_T0, _S_size>
1897  __select_impl(const basic_mask& __k, const _T0& __t, const _T1& __f) noexcept
1898  {
1899  using _Vp = vec<_T0, _S_size>;
1900  if constexpr (!is_same_v<basic_mask, typename _Vp::mask_type>)
1901  return __select_impl(static_cast<_Vp::mask_type>(__k), __t, __f);
1902  else
1903  return _Vp::_S_init(__select_impl(__k._M_data0, __t, __f),
1904  __select_impl(__k._M_data1, __t, __f));
1905  }
1906 
1907  template <_ArchTraits _Traits = {}>
1908  [[__gnu__::__always_inline__]]
1909  constexpr bool
1910  _M_all_of() const
1911  {
1912  if constexpr (_N0 == _N1)
1913  return (_M_data0 && _M_data1)._M_all_of();
1914  else
1915  return _M_data0._M_all_of() && _M_data1._M_all_of();
1916  }
1917 
1918  template <_ArchTraits _Traits = {}>
1919  [[__gnu__::__always_inline__]]
1920  constexpr bool
1921  _M_any_of() const
1922  {
1923  if constexpr (_N0 == _N1)
1924  return (_M_data0 || _M_data1)._M_any_of();
1925  else
1926  return _M_data0._M_any_of() || _M_data1._M_any_of();
1927  }
1928 
1929  template <_ArchTraits _Traits = {}>
1930  [[__gnu__::__always_inline__]]
1931  constexpr bool
1932  _M_none_of() const
1933  {
1934  if constexpr (_N0 == _N1)
1935  return (_M_data0 || _M_data1)._M_none_of();
1936  else
1937  return _M_data0._M_none_of() && _M_data1._M_none_of();
1938  }
1939 
1940  [[__gnu__::__always_inline__]]
1941  constexpr __simd_size_type
1942  _M_reduce_min_index() const
1943  {
1944  if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
1945  {
1946  const auto __bits = _M_to_uint();
1947  __glibcxx_simd_precondition(__bits, "An empty mask does not have a min_index.");
1948  return __countr_zero(_M_to_uint());
1949  }
1950  else if (_M_data0._M_none_of())
1951  return _M_data1._M_reduce_min_index() + _N0;
1952  else
1953  return _M_data0._M_reduce_min_index();
1954  }
1955 
1956  [[__gnu__::__always_inline__]]
1957  constexpr __simd_size_type
1958  _M_reduce_max_index() const
1959  {
1960  if constexpr (_S_size <= numeric_limits<unsigned long long>::digits)
1961  {
1962  const auto __bits = _M_to_uint();
1963  __glibcxx_simd_precondition(__bits, "An empty mask does not have a max_index.");
1964  return __highest_bit(_M_to_uint());
1965  }
1966  else if (_M_data1._M_none_of())
1967  return _M_data0._M_reduce_max_index();
1968  else
1969  return _M_data1._M_reduce_max_index() + _N0;
1970  }
1971 
1972  [[__gnu__::__always_inline__]]
1973  friend constexpr bool
1974  __is_const_known(const basic_mask& __x)
1975  { return __is_const_known(__x._M_data0) && __is_const_known(__x._M_data1); }
1976  };
1977 } // namespace simd
1978 _GLIBCXX_END_NAMESPACE_VERSION
1979 } // namespace std
1980 
1981 #pragma GCC diagnostic pop
1982 #endif // C++26
1983 #endif // _GLIBCXX_SIMD_MASK_H
std::chrono::operator<=
constexpr bool operator<=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: chrono.h:859
std::chrono::operator>=
constexpr bool operator>=(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: chrono.h:873
std::chrono::operator<
constexpr bool operator<(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: chrono.h:826
std::chrono::operator>
constexpr bool operator>(const duration< _Rep1, _Period1 > &__lhs, const duration< _Rep2, _Period2 > &__rhs)
Definition: chrono.h:866
std::requires
requires requires
Definition: complex:1948
std::operator-
constexpr complex< _Tp > operator-(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x minus y.
Definition: complex:404
std::operator+
constexpr complex< _Tp > operator+(const complex< _Tp > &__x, const complex< _Tp > &__y)
Return new complex value x plus y.
Definition: complex:374
std::end
_Tp * end(valarray< _Tp > &__va) noexcept
Return an iterator pointing to one past the last element of the valarray.
Definition: valarray:1251
std::begin
_Tp * begin(valarray< _Tp > &__va) noexcept
Return an iterator pointing to the first element of the valarray.
Definition: valarray:1229
std::max
constexpr const _Tp & max(const _Tp &, const _Tp &)
This does what you think it does.
Definition: stl_algobase.h:256
std
ISO C++ entities toplevel namespace is std.
std::operator&
constexpr bitset< _Nb > operator&(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1618
std::same_as
concept same_as
[concept.same], concept same_as
Definition: concepts:65
std::cend
constexpr auto cend(const _Container &__cont) noexcept(noexcept(std::end(__cont))) -> decltype(std::end(__cont))
Return an iterator pointing to one past the last element of the const container.
Definition: range_access.h:144
std::operator|
constexpr bitset< _Nb > operator|(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1628
std::size
constexpr auto size(const _Container &__cont) noexcept(noexcept(__cont.size())) -> decltype(__cont.size())
Return the size of a container.
Definition: range_access.h:274
std::operator^
constexpr bitset< _Nb > operator^(const bitset< _Nb > &__x, const bitset< _Nb > &__y) noexcept
Global bitwise operations on bitsets.
Definition: bitset:1638
std::destructible
concept destructible
[concept.destructible], concept destructible
Definition: concepts:157
std::cbegin
constexpr auto cbegin(const _Container &__cont) noexcept(noexcept(std::begin(__cont))) -> decltype(std::begin(__cont))
Return an iterator pointing to the first element of the const container.
Definition: range_access.h:132
std::__numeric_limits_base::digits
static constexpr int digits
Definition: limits:218
std::numeric_limits::max
static constexpr _Tp max() noexcept
Definition: limits:328
std::numeric_limits::min
static constexpr _Tp min() noexcept
Definition: limits:324
Generated by doxygen 1.9.1