// Simd fixed_size ABI specific implementations -*- C++ -*-

// Copyright (C) 2020-2022 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/*
 * The fixed_size ABI gives the following guarantees:
 *  - simd objects are passed via the stack
 *  - memory layout of `simd<_Tp, _Np>` is equivalent to `array<_Tp, _Np>`
 *  - alignment of `simd<_Tp, _Np>` is `_Np * sizeof(_Tp)` if _Np is __a
 *    power-of-2 value, otherwise `std::__bit_ceil(_Np * sizeof(_Tp))` (Note:
 *    if the alignment were to exceed the system/compiler maximum, it is bounded
 *    to that maximum)
 *  - simd_mask objects are passed like bitset<_Np>
 *  - memory layout of `simd_mask<_Tp, _Np>` is equivalent to `bitset<_Np>`
 *  - alignment of `simd_mask<_Tp, _Np>` is equal to the alignment of
 *    `bitset<_Np>`
 */

#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_
#define _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_

#if __cplusplus >= 201703L

#include <array>

_GLIBCXX_SIMD_BEGIN_NAMESPACE

// __simd_tuple_element {{{
template <size_t _I, typename _Tp>
  struct __simd_tuple_element;

template <typename _Tp, typename _A0, typename... _As>
  struct __simd_tuple_element<0, _SimdTuple<_Tp, _A0, _As...>>
  { using type = simd<_Tp, _A0>; };

template <size_t _I, typename _Tp, typename _A0, typename... _As>
  struct __simd_tuple_element<_I, _SimdTuple<_Tp, _A0, _As...>>
  {
    using type =
      typename __simd_tuple_element<_I - 1, _SimdTuple<_Tp, _As...>>::type;
  };

template <size_t _I, typename _Tp>
  using __simd_tuple_element_t = typename __simd_tuple_element<_I, _Tp>::type;

// }}}
// __simd_tuple_concat {{{

template <typename _Tp, typename... _A0s, typename... _A1s>
  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, _A0s..., _A1s...>
  __simd_tuple_concat(const _SimdTuple<_Tp, _A0s...>& __left,
		      const _SimdTuple<_Tp, _A1s...>& __right)
  {
    if constexpr (sizeof...(_A0s) == 0)
      return __right;
    else if constexpr (sizeof...(_A1s) == 0)
      return __left;
    else
      return {__left.first, __simd_tuple_concat(__left.second, __right)};
  }

template <typename _Tp, typename _A10, typename... _A1s>
  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_Tp, simd_abi::scalar, _A10,
					       _A1s...>
  __simd_tuple_concat(const _Tp& __left,
		      const _SimdTuple<_Tp, _A10, _A1s...>& __right)
  { return {__left, __right}; }

// }}}
// __simd_tuple_pop_front {{{
// Returns the next _SimdTuple in __x that has _Np elements less.
// Precondition: _Np must match the number of elements in __first (recursively)
template <size_t _Np, typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto)
  __simd_tuple_pop_front(_Tp&& __x)
  {
    if constexpr (_Np == 0)
      return static_cast<_Tp&&>(__x);
    else
      {
	using _Up = __remove_cvref_t<_Tp>;
	static_assert(_Np >= _Up::_S_first_size);
	return __simd_tuple_pop_front<_Np - _Up::_S_first_size>(__x.second);
      }
  }

// }}}
// __get_simd_at<_Np> {{{1
struct __as_simd {};

struct __as_simd_tuple {};

template <typename _Tp, typename _A0, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr simd<_Tp, _A0>
  __simd_tuple_get_impl(__as_simd, const _SimdTuple<_Tp, _A0, _Abis...>& __t,
			_SizeConstant<0>)
  { return {__private_init, __t.first}; }

template <typename _Tp, typename _A0, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr const auto&
  __simd_tuple_get_impl(__as_simd_tuple,
			const _SimdTuple<_Tp, _A0, _Abis...>& __t,
			_SizeConstant<0>)
  { return __t.first; }

template <typename _Tp, typename _A0, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
  __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _A0, _Abis...>& __t,
			_SizeConstant<0>)
  { return __t.first; }

template <typename _R, size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto
  __simd_tuple_get_impl(_R, const _SimdTuple<_Tp, _Abis...>& __t,
			_SizeConstant<_Np>)
  { return __simd_tuple_get_impl(_R(), __t.second, _SizeConstant<_Np - 1>()); }

template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
  __simd_tuple_get_impl(__as_simd_tuple, _SimdTuple<_Tp, _Abis...>& __t,
			_SizeConstant<_Np>)
  {
    return __simd_tuple_get_impl(__as_simd_tuple(), __t.second,
				 _SizeConstant<_Np - 1>());
  }

template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto
  __get_simd_at(const _SimdTuple<_Tp, _Abis...>& __t)
  { return __simd_tuple_get_impl(__as_simd(), __t, _SizeConstant<_Np>()); }

// }}}
// __get_tuple_at<_Np> {{{
template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto
  __get_tuple_at(const _SimdTuple<_Tp, _Abis...>& __t)
  {
    return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>());
  }

template <size_t _Np, typename _Tp, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC constexpr auto&
  __get_tuple_at(_SimdTuple<_Tp, _Abis...>& __t)
  {
    return __simd_tuple_get_impl(__as_simd_tuple(), __t, _SizeConstant<_Np>());
  }

// __tuple_element_meta {{{1
template <typename _Tp, typename _Abi, size_t _Offset>
  struct __tuple_element_meta : public _Abi::_SimdImpl
  {
    static_assert(is_same_v<typename _Abi::_SimdImpl::abi_type,
			    _Abi>); // this fails e.g. when _SimdImpl is an
				    // alias for _SimdImplBuiltin<_DifferentAbi>
    using value_type = _Tp;
    using abi_type = _Abi;
    using _Traits = _SimdTraits<_Tp, _Abi>;
    using _MaskImpl = typename _Abi::_MaskImpl;
    using _MaskMember = typename _Traits::_MaskMember;
    using simd_type = simd<_Tp, _Abi>;
    static constexpr size_t _S_offset = _Offset;
    static constexpr size_t _S_size() { return simd_size<_Tp, _Abi>::value; }
    static constexpr _MaskImpl _S_mask_impl = {};

    template <size_t _Np, bool _Sanitized>
      _GLIBCXX_SIMD_INTRINSIC static auto
      _S_submask(_BitMask<_Np, _Sanitized> __bits)
      { return __bits.template _M_extract<_Offset, _S_size()>(); }

    template <size_t _Np, bool _Sanitized>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember
      _S_make_mask(_BitMask<_Np, _Sanitized> __bits)
      {
	return _MaskImpl::template _S_convert<_Tp>(
	  __bits.template _M_extract<_Offset, _S_size()>()._M_sanitized());
      }

    _GLIBCXX_SIMD_INTRINSIC static _ULLong
    _S_mask_to_shifted_ullong(_MaskMember __k)
    { return _MaskImpl::_S_to_bits(__k).to_ullong() << _Offset; }
  };

template <size_t _Offset, typename _Tp, typename _Abi, typename... _As>
  _GLIBCXX_SIMD_INTRINSIC
  __tuple_element_meta<_Tp, _Abi, _Offset>
  __make_meta(const _SimdTuple<_Tp, _Abi, _As...>&)
  { return {}; }

// }}}1
// _WithOffset wrapper class {{{
template <size_t _Offset, typename _Base>
  struct _WithOffset : public _Base
  {
    static inline constexpr size_t _S_offset = _Offset;

    _GLIBCXX_SIMD_INTRINSIC char* _M_as_charptr()
    {
      return reinterpret_cast<char*>(this)
	     + _S_offset * sizeof(typename _Base::value_type);
    }

    _GLIBCXX_SIMD_INTRINSIC const char* _M_as_charptr() const
    {
      return reinterpret_cast<const char*>(this)
	     + _S_offset * sizeof(typename _Base::value_type);
    }
  };

// make _WithOffset<_WithOffset> ill-formed to use:
template <size_t _O0, size_t _O1, typename _Base>
  struct _WithOffset<_O0, _WithOffset<_O1, _Base>> {};

template <size_t _Offset, typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC
  decltype(auto)
  __add_offset(_Tp& __base)
  { return static_cast<_WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(__base); }

template <size_t _Offset, typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC
  decltype(auto)
  __add_offset(const _Tp& __base)
  {
    return static_cast<const _WithOffset<_Offset, __remove_cvref_t<_Tp>>&>(
      __base);
  }

template <size_t _Offset, size_t _ExistingOffset, typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC
  decltype(auto)
  __add_offset(_WithOffset<_ExistingOffset, _Tp>& __base)
  {
    return static_cast<_WithOffset<_Offset + _ExistingOffset, _Tp>&>(
      static_cast<_Tp&>(__base));
  }

template <size_t _Offset, size_t _ExistingOffset, typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC
  decltype(auto)
  __add_offset(const _WithOffset<_ExistingOffset, _Tp>& __base)
  {
    return static_cast<const _WithOffset<_Offset + _ExistingOffset, _Tp>&>(
      static_cast<const _Tp&>(__base));
  }

template <typename _Tp>
  constexpr inline size_t __offset = 0;

template <size_t _Offset, typename _Tp>
  constexpr inline size_t __offset<_WithOffset<_Offset, _Tp>>
    = _WithOffset<_Offset, _Tp>::_S_offset;

template <typename _Tp>
  constexpr inline size_t __offset<const _Tp> = __offset<_Tp>;

template <typename _Tp>
  constexpr inline size_t __offset<_Tp&> = __offset<_Tp>;

template <typename _Tp>
  constexpr inline size_t __offset<_Tp&&> = __offset<_Tp>;

// }}}
// _SimdTuple specializations {{{1
// empty {{{2
template <typename _Tp>
  struct _SimdTuple<_Tp>
  {
    using value_type = _Tp;
    static constexpr size_t _S_tuple_size = 0;
    static constexpr size_t _S_size() { return 0; }
  };

// _SimdTupleData {{{2
template <typename _FirstType, typename _SecondType>
  struct _SimdTupleData
  {
    _FirstType first;
    _SecondType second;

    _GLIBCXX_SIMD_INTRINSIC
    constexpr bool _M_is_constprop() const
    {
      if constexpr (is_class_v<_FirstType>)
	return first._M_is_constprop() && second._M_is_constprop();
      else
	return __builtin_constant_p(first) && second._M_is_constprop();
    }
  };

template <typename _FirstType, typename _Tp>
  struct _SimdTupleData<_FirstType, _SimdTuple<_Tp>>
  {
    _FirstType first;
    static constexpr _SimdTuple<_Tp> second = {};

    _GLIBCXX_SIMD_INTRINSIC
    constexpr bool _M_is_constprop() const
    {
      if constexpr (is_class_v<_FirstType>)
	return first._M_is_constprop();
      else
	return __builtin_constant_p(first);
    }
  };

// 1 or more {{{2
template <typename _Tp, typename _Abi0, typename... _Abis>
  struct _SimdTuple<_Tp, _Abi0, _Abis...>
    : _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
		     _SimdTuple<_Tp, _Abis...>>
  {
    static_assert(!__is_fixed_size_abi_v<_Abi0>);
    using value_type = _Tp;
    using _FirstType = typename _SimdTraits<_Tp, _Abi0>::_SimdMember;
    using _FirstAbi = _Abi0;
    using _SecondType = _SimdTuple<_Tp, _Abis...>;
    static constexpr size_t _S_tuple_size = sizeof...(_Abis) + 1;

    static constexpr size_t _S_size()
    { return simd_size_v<_Tp, _Abi0> + _SecondType::_S_size(); }

    static constexpr size_t _S_first_size = simd_size_v<_Tp, _Abi0>;
    static constexpr bool _S_is_homogeneous = (is_same_v<_Abi0, _Abis> && ...);

    using _Base = _SimdTupleData<typename _SimdTraits<_Tp, _Abi0>::_SimdMember,
				 _SimdTuple<_Tp, _Abis...>>;
    using _Base::first;
    using _Base::second;

    _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple() = default;
    _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(const _SimdTuple&) = default;
    _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple& operator=(const _SimdTuple&)
      = default;

    template <typename _Up>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(_Up&& __x)
      : _Base{static_cast<_Up&&>(__x)} {}

    template <typename _Up, typename _Up2>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(_Up&& __x, _Up2&& __y)
      : _Base{static_cast<_Up&&>(__x), static_cast<_Up2&&>(__y)} {}

    template <typename _Up>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple(_Up&& __x, _SimdTuple<_Tp>)
      : _Base{static_cast<_Up&&>(__x)} {}

    _GLIBCXX_SIMD_INTRINSIC char* _M_as_charptr()
    { return reinterpret_cast<char*>(this); }

    _GLIBCXX_SIMD_INTRINSIC const char* _M_as_charptr() const
    { return reinterpret_cast<const char*>(this); }

    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC constexpr auto& _M_at()
      {
	if constexpr (_Np == 0)
	  return first;
	else
	  return second.template _M_at<_Np - 1>();
      }

    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC constexpr const auto& _M_at() const
      {
	if constexpr (_Np == 0)
	  return first;
	else
	  return second.template _M_at<_Np - 1>();
      }

    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC constexpr auto _M_simd_at() const
      {
	if constexpr (_Np == 0)
	  return simd<_Tp, _Abi0>(__private_init, first);
	else
	  return second.template _M_simd_at<_Np - 1>();
      }

    template <size_t _Offset = 0, typename _Fp>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdTuple
      _S_generate(_Fp&& __gen, _SizeConstant<_Offset> = {})
      {
	auto&& __first = __gen(__tuple_element_meta<_Tp, _Abi0, _Offset>());
	if constexpr (_S_tuple_size == 1)
	  return {__first};
	else
	  return {__first,
		  _SecondType::_S_generate(
		    static_cast<_Fp&&>(__gen),
		    _SizeConstant<_Offset + simd_size_v<_Tp, _Abi0>>())};
      }

    template <size_t _Offset = 0, typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC _SimdTuple
      _M_apply_wrapped(_Fp&& __fun, const _More&... __more) const
      {
	auto&& __first
	  = __fun(__make_meta<_Offset>(*this), first, __more.first...);
	if constexpr (_S_tuple_size == 1)
	  return {__first};
	else
	  return {
	    __first,
	    second.template _M_apply_wrapped<_Offset + simd_size_v<_Tp, _Abi0>>(
	      static_cast<_Fp&&>(__fun), __more.second...)};
      }

    template <typename _Tup>
      _GLIBCXX_SIMD_INTRINSIC constexpr decltype(auto)
      _M_extract_argument(_Tup&& __tup) const
      {
	using _TupT = typename __remove_cvref_t<_Tup>::value_type;
	if constexpr (is_same_v<_SimdTuple, __remove_cvref_t<_Tup>>)
	  return __tup.first;
	else if (__builtin_is_constant_evaluated())
	  return __fixed_size_storage_t<_TupT, _S_first_size>::_S_generate([&](
	    auto __meta) constexpr {
	    return __meta._S_generator(
	      [&](auto __i) constexpr { return __tup[__i]; },
	      static_cast<_TupT*>(nullptr));
	  });
	else
	  return [&]() {
	    __fixed_size_storage_t<_TupT, _S_first_size> __r;
	    __builtin_memcpy(__r._M_as_charptr(), __tup._M_as_charptr(),
			     sizeof(__r));
	    return __r;
	  }();
      }

    template <typename _Tup>
      _GLIBCXX_SIMD_INTRINSIC constexpr auto&
      _M_skip_argument(_Tup&& __tup) const
      {
	static_assert(_S_tuple_size > 1);
	using _Up = __remove_cvref_t<_Tup>;
	constexpr size_t __off = __offset<_Up>;
	if constexpr (_S_first_size == _Up::_S_first_size && __off == 0)
	  return __tup.second;
	else if constexpr (_S_first_size > _Up::_S_first_size
			   && _S_first_size % _Up::_S_first_size == 0
			   && __off == 0)
	  return __simd_tuple_pop_front<_S_first_size>(__tup);
	else if constexpr (_S_first_size + __off < _Up::_S_first_size)
	  return __add_offset<_S_first_size>(__tup);
	else if constexpr (_S_first_size + __off == _Up::_S_first_size)
	  return __tup.second;
	else
	  __assert_unreachable<_Tup>();
      }

    template <size_t _Offset, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC constexpr void
      _M_assign_front(const _SimdTuple<_Tp, _Abi0, _More...>& __x) &
      {
	static_assert(_Offset == 0);
	first = __x.first;
	if constexpr (sizeof...(_More) > 0)
	  {
	    static_assert(sizeof...(_Abis) >= sizeof...(_More));
	    second.template _M_assign_front<0>(__x.second);
	  }
      }

    template <size_t _Offset>
      _GLIBCXX_SIMD_INTRINSIC constexpr void
      _M_assign_front(const _FirstType& __x) &
      {
	static_assert(_Offset == 0);
	first = __x;
      }

    template <size_t _Offset, typename... _As>
      _GLIBCXX_SIMD_INTRINSIC constexpr void
      _M_assign_front(const _SimdTuple<_Tp, _As...>& __x) &
      {
	__builtin_memcpy(_M_as_charptr() + _Offset * sizeof(value_type),
			 __x._M_as_charptr(),
			 sizeof(_Tp) * _SimdTuple<_Tp, _As...>::_S_size());
      }

    /*
     * Iterate over the first objects in this _SimdTuple and call __fun for each
     * of them. If additional arguments are passed via __more, chunk them into
     * _SimdTuple or __vector_type_t objects of the same number of values.
     */
    template <typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple
      _M_apply_per_chunk(_Fp&& __fun, _More&&... __more) const
      {
	if constexpr ((...
		       || conjunction_v<
			 is_lvalue_reference<_More>,
			 negation<is_const<remove_reference_t<_More>>>>) )
	  {
	    // need to write back at least one of __more after calling __fun
	    auto&& __first = [&](auto... __args) constexpr
	    {
	      auto __r = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
			       __args...);
	      [[maybe_unused]] auto&& __ignore_me = {(
		[](auto&& __dst, const auto& __src) {
		  if constexpr (is_assignable_v<decltype(__dst),
						decltype(__dst)>)
		    {
		      __dst.template _M_assign_front<__offset<decltype(__dst)>>(
			__src);
		    }
		}(static_cast<_More&&>(__more), __args),
		0)...};
	      return __r;
	    }
	    (_M_extract_argument(__more)...);
	    if constexpr (_S_tuple_size == 1)
	      return {__first};
	    else
	      return {__first,
		      second._M_apply_per_chunk(static_cast<_Fp&&>(__fun),
						_M_skip_argument(__more)...)};
	  }
	else
	  {
	    auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
				   _M_extract_argument(__more)...);
	    if constexpr (_S_tuple_size == 1)
	      return {__first};
	    else
	      return {__first,
		      second._M_apply_per_chunk(static_cast<_Fp&&>(__fun),
						_M_skip_argument(__more)...)};
	  }
      }

    template <typename _R = _Tp, typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC auto _M_apply_r(_Fp&& __fun,
					      const _More&... __more) const
      {
	auto&& __first = __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), first,
			       __more.first...);
	if constexpr (_S_tuple_size == 1)
	  return __first;
	else
	  return __simd_tuple_concat<_R>(
	    __first, second.template _M_apply_r<_R>(static_cast<_Fp&&>(__fun),
						    __more.second...));
      }

    template <typename _Fp, typename... _More>
      _GLIBCXX_SIMD_INTRINSIC constexpr friend _SanitizedBitMask<_S_size()>
      _M_test(const _Fp& __fun, const _SimdTuple& __x, const _More&... __more)
      {
	const _SanitizedBitMask<_S_first_size> __first
	  = _Abi0::_MaskImpl::_S_to_bits(
	    __fun(__tuple_element_meta<_Tp, _Abi0, 0>(), __x.first,
		  __more.first...));
	if constexpr (_S_tuple_size == 1)
	  return __first;
	else
	  return _M_test(__fun, __x.second, __more.second...)
	    ._M_prepend(__first);
      }

    template <typename _Up, _Up _I>
      _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
      operator[](integral_constant<_Up, _I>) const noexcept
      {
	if constexpr (_I < simd_size_v<_Tp, _Abi0>)
	  return _M_subscript_read(_I);
	else
	  return second[integral_constant<_Up, _I - simd_size_v<_Tp, _Abi0>>()];
      }

    _GLIBCXX_SIMD_INTRINSIC
    _Tp operator[](size_t __i) const noexcept
    {
      if constexpr (_S_tuple_size == 1)
	return _M_subscript_read(__i);
      else
	{
#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
	  return reinterpret_cast<const __may_alias<_Tp>*>(this)[__i];
#else
	  if constexpr (__is_scalar_abi<_Abi0>())
	    {
	      const _Tp* ptr = &first;
	      return ptr[__i];
	    }
	  else
	    return __i < simd_size_v<_Tp, _Abi0>
		     ? _M_subscript_read(__i)
		     : second[__i - simd_size_v<_Tp, _Abi0>];
#endif
	}
    }

    _GLIBCXX_SIMD_INTRINSIC
    void _M_set(size_t __i, _Tp __val) noexcept
    {
      if constexpr (_S_tuple_size == 1)
	return _M_subscript_write(__i, __val);
      else
	{
#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
	  reinterpret_cast<__may_alias<_Tp>*>(this)[__i] = __val;
#else
	  if (__i < simd_size_v<_Tp, _Abi0>)
	    _M_subscript_write(__i, __val);
	  else
	    second._M_set(__i - simd_size_v<_Tp, _Abi0>, __val);
#endif
	}
    }

  private:
    // _M_subscript_read/_write {{{
    _GLIBCXX_SIMD_INTRINSIC
    _Tp _M_subscript_read([[maybe_unused]] size_t __i) const noexcept
    {
      if constexpr (__is_vectorizable_v<_FirstType>)
	return first;
      else
	return first[__i];
    }

    _GLIBCXX_SIMD_INTRINSIC
    void _M_subscript_write([[maybe_unused]] size_t __i, _Tp __y) noexcept
    {
      if constexpr (__is_vectorizable_v<_FirstType>)
	first = __y;
      else
	first._M_set(__i, __y);
    }

    // }}}
  };

// __make_simd_tuple {{{1
template <typename _Tp, typename _A0>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0>
  __make_simd_tuple(simd<_Tp, _A0> __x0)
  { return {__data(__x0)}; }

template <typename _Tp, typename _A0, typename... _As>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0, _As...>
  __make_simd_tuple(const simd<_Tp, _A0>& __x0, const simd<_Tp, _As>&... __xs)
  { return {__data(__x0), __make_simd_tuple(__xs...)}; }

template <typename _Tp, typename _A0>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0>
  __make_simd_tuple(const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0)
  { return {__arg0}; }

template <typename _Tp, typename _A0, typename _A1, typename... _Abis>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp, _A0, _A1, _Abis...>
  __make_simd_tuple(
    const typename _SimdTraits<_Tp, _A0>::_SimdMember& __arg0,
    const typename _SimdTraits<_Tp, _A1>::_SimdMember& __arg1,
    const typename _SimdTraits<_Tp, _Abis>::_SimdMember&... __args)
  { return {__arg0, __make_simd_tuple<_Tp, _A1, _Abis...>(__arg1, __args...)}; }

// __to_simd_tuple {{{1
template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX>
  _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np>
  __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX);

template <typename _Tp, size_t _Np,
	  size_t _Offset = 0, // skip this many elements in __from0
	  typename _R = __fixed_size_storage_t<_Tp, _Np>, typename _V0,
	  typename _V0VT = _VectorTraits<_V0>, typename... _VX>
  _GLIBCXX_SIMD_INTRINSIC _R constexpr __to_simd_tuple(const _V0 __from0,
						       const _VX... __fromX)
  {
    static_assert(is_same_v<typename _V0VT::value_type, _Tp>);
    static_assert(_Offset < _V0VT::_S_full_size);
    using _R0 = __vector_type_t<_Tp, _R::_S_first_size>;
    if constexpr (_R::_S_tuple_size == 1)
      {
	if constexpr (_Np == 1)
	  return _R{__from0[_Offset]};
	else if constexpr (_Offset == 0 && _V0VT::_S_full_size >= _Np)
	  return _R{__intrin_bitcast<_R0>(__from0)};
	else if constexpr (_Offset * 2 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size / 2 >= _Np)
	  return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0))};
	else if constexpr (_Offset * 4 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size / 4 >= _Np)
	  return _R{__intrin_bitcast<_R0>(__extract_part<1, 4>(__from0))};
	else
	  __assert_unreachable<_Tp>();
      }
    else
      {
	if constexpr (1 == _R::_S_first_size)
	  { // extract one scalar and recurse
	    if constexpr (_Offset + 1 < _V0VT::_S_full_size)
	      return _R{__from0[_Offset],
			__to_simd_tuple<_Tp, _Np - 1, _Offset + 1>(__from0,
								   __fromX...)};
	    else
	      return _R{__from0[_Offset],
			__to_simd_tuple<_Tp, _Np - 1, 0>(__fromX...)};
	  }

	// place __from0 into _R::first and recurse for __fromX -> _R::second
	else if constexpr (_V0VT::_S_full_size == _R::_S_first_size
			   && _Offset == 0)
	  return _R{__from0,
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size>(__fromX...)};

	// place lower part of __from0 into _R::first and recurse with _Offset
	else if constexpr (_V0VT::_S_full_size > _R::_S_first_size
			   && _Offset == 0)
	  return _R{__intrin_bitcast<_R0>(__from0),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
				    _R::_S_first_size>(__from0, __fromX...)};

	// place lower part of second quarter of __from0 into _R::first and
	// recurse with _Offset
	else if constexpr (_Offset * 4 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
	  return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
				    _Offset + _R::_S_first_size>(__from0,
								 __fromX...)};

	// place lower half of high half of __from0 into _R::first and recurse
	// with _Offset
	else if constexpr (_Offset * 2 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size >= 4 * _R::_S_first_size)
	  return _R{__intrin_bitcast<_R0>(__extract_part<2, 4>(__from0)),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size,
				    _Offset + _R::_S_first_size>(__from0,
								 __fromX...)};

	// place high half of __from0 into _R::first and recurse with __fromX
	else if constexpr (_Offset * 2 == _V0VT::_S_full_size
			   && _V0VT::_S_full_size / 2 >= _R::_S_first_size)
	  return _R{__intrin_bitcast<_R0>(__extract_part<1, 2>(__from0)),
		    __to_simd_tuple<_Tp, _Np - _R::_S_first_size, 0>(
		      __fromX...)};

	// ill-formed if some unforseen pattern is needed
	else
	  __assert_unreachable<_Tp>();
      }
  }

template <typename _Tp, size_t _Np, typename _V, size_t _NV, typename... _VX>
  _GLIBCXX_SIMD_INTRINSIC constexpr __fixed_size_storage_t<_Tp, _Np>
  __to_simd_tuple(const array<_V, _NV>& __from, const _VX... __fromX)
  {
    if constexpr (is_same_v<_Tp, _V>)
      {
	static_assert(
	  sizeof...(_VX) == 0,
	  "An array of scalars must be the last argument to __to_simd_tuple");
	return __call_with_subscripts(
	  __from,
	  make_index_sequence<_NV>(), [&](const auto... __args) constexpr {
	    return __simd_tuple_concat(
	      _SimdTuple<_Tp, simd_abi::scalar>{__args}..., _SimdTuple<_Tp>());
	  });
      }
    else
      return __call_with_subscripts(
	__from,
	make_index_sequence<_NV>(), [&](const auto... __args) constexpr {
	  return __to_simd_tuple<_Tp, _Np>(__args..., __fromX...);
	});
  }

template <size_t, typename _Tp>
  using __to_tuple_helper = _Tp;

template <typename _Tp, typename _A0, size_t _NOut, size_t _Np,
	  size_t... _Indexes>
  _GLIBCXX_SIMD_INTRINSIC __fixed_size_storage_t<_Tp, _NOut>
  __to_simd_tuple_impl(index_sequence<_Indexes...>,
      const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
  {
    return __make_simd_tuple<_Tp, __to_tuple_helper<_Indexes, _A0>...>(
      __args[_Indexes]...);
  }

template <typename _Tp, typename _A0, size_t _NOut, size_t _Np,
	  typename _R = __fixed_size_storage_t<_Tp, _NOut>>
  _GLIBCXX_SIMD_INTRINSIC _R
  __to_simd_tuple_sized(
    const array<__vector_type_t<_Tp, simd_size_v<_Tp, _A0>>, _Np>& __args)
  {
    static_assert(_Np * simd_size_v<_Tp, _A0> >= _NOut);
    return __to_simd_tuple_impl<_Tp, _A0, _NOut>(
      make_index_sequence<_R::_S_tuple_size>(), __args);
  }

// __optimize_simd_tuple {{{1
template <typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC _SimdTuple<_Tp>
  __optimize_simd_tuple(const _SimdTuple<_Tp>)
  { return {}; }

template <typename _Tp, typename _Ap>
  _GLIBCXX_SIMD_INTRINSIC const _SimdTuple<_Tp, _Ap>&
  __optimize_simd_tuple(const _SimdTuple<_Tp, _Ap>& __x)
  { return __x; }

template <typename _Tp, typename _A0, typename _A1, typename... _Abis,
	  typename _R = __fixed_size_storage_t<
	    _Tp, _SimdTuple<_Tp, _A0, _A1, _Abis...>::_S_size()>>
  _GLIBCXX_SIMD_INTRINSIC _R
  __optimize_simd_tuple(const _SimdTuple<_Tp, _A0, _A1, _Abis...>& __x)
  {
    using _Tup = _SimdTuple<_Tp, _A0, _A1, _Abis...>;
    if constexpr (is_same_v<_R, _Tup>)
      return __x;
    else if constexpr (is_same_v<typename _R::_FirstType,
				 typename _Tup::_FirstType>)
      return {__x.first, __optimize_simd_tuple(__x.second)};
    else if constexpr (__is_scalar_abi<_A0>()
		       || _A0::template _S_is_partial<_Tp>)
      return {__generate_from_n_evaluations<_R::_S_first_size,
					    typename _R::_FirstType>(
		[&](auto __i) { return __x[__i]; }),
	      __optimize_simd_tuple(
		__simd_tuple_pop_front<_R::_S_first_size>(__x))};
    else if constexpr (is_same_v<_A0, _A1>
	&& _R::_S_first_size == simd_size_v<_Tp, _A0> + simd_size_v<_Tp, _A1>)
      return {__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
	      __optimize_simd_tuple(__x.second.second)};
    else if constexpr (sizeof...(_Abis) >= 2
	&& _R::_S_first_size == (4 * simd_size_v<_Tp, _A0>)
	&& simd_size_v<_Tp, _A0> == __simd_tuple_element_t<
	    (sizeof...(_Abis) >= 2 ? 3 : 0), _Tup>::size())
      return {
	__concat(__concat(__x.template _M_at<0>(), __x.template _M_at<1>()),
		 __concat(__x.template _M_at<2>(), __x.template _M_at<3>())),
	__optimize_simd_tuple(__x.second.second.second.second)};
    else
      {
	static_assert(sizeof(_R) == sizeof(__x));
	_R __r;
	__builtin_memcpy(__r._M_as_charptr(), __x._M_as_charptr(),
			 sizeof(_Tp) * _R::_S_size());
	return __r;
      }
  }

// __for_each(const _SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__t), __t.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
						     static_cast<_Fp&&>(__fun));
  }

// __for_each(_SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0>& __t, _Fp&& __fun)
  { static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__t), __t.first); }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __t, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__t), __t.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__t.second,
						     static_cast<_Fp&&>(__fun));
  }

// __for_each(_SimdTuple &, const _SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b,
	     _Fp&& __fun)
  {
    static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first);
  }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(_SimdTuple<_Tp, _A0, _A1, _As...>& __a,
	     const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
						     static_cast<_Fp&&>(__fun));
  }

// __for_each(const _SimdTuple &, const _SimdTuple &, Fun) {{{1
template <size_t _Offset = 0, typename _Tp, typename _A0, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0>& __a, const _SimdTuple<_Tp, _A0>& __b,
	     _Fp&& __fun)
  {
    static_cast<_Fp&&>(__fun)(__make_meta<_Offset>(__a), __a.first, __b.first);
  }

template <size_t _Offset = 0, typename _Tp, typename _A0, typename _A1,
	  typename... _As, typename _Fp>
  _GLIBCXX_SIMD_INTRINSIC constexpr void
  __for_each(const _SimdTuple<_Tp, _A0, _A1, _As...>& __a,
	     const _SimdTuple<_Tp, _A0, _A1, _As...>& __b, _Fp&& __fun)
  {
    __fun(__make_meta<_Offset>(__a), __a.first, __b.first);
    __for_each<_Offset + simd_size<_Tp, _A0>::value>(__a.second, __b.second,
						     static_cast<_Fp&&>(__fun));
  }

// }}}1
// __extract_part(_SimdTuple) {{{
template <int _Index, int _Total, int _Combine, typename _Tp, typename _A0,
	  typename... _As>
  _GLIBCXX_SIMD_INTRINSIC auto // __vector_type_t or _SimdTuple
  __extract_part(const _SimdTuple<_Tp, _A0, _As...>& __x)
  {
    // worst cases:
    // (a) 4, 4, 4 => 3, 3, 3, 3 (_Total = 4)
    // (b) 2, 2, 2 => 3, 3       (_Total = 2)
    // (c) 4, 2 => 2, 2, 2       (_Total = 3)
    using _Tuple = _SimdTuple<_Tp, _A0, _As...>;
    static_assert(_Index + _Combine <= _Total && _Index >= 0 && _Total >= 1);
    constexpr size_t _Np = _Tuple::_S_size();
    static_assert(_Np >= _Total && _Np % _Total == 0);
    constexpr size_t __values_per_part = _Np / _Total;
    [[maybe_unused]] constexpr size_t __values_to_skip
      = _Index * __values_per_part;
    constexpr size_t __return_size = __values_per_part * _Combine;
    using _RetAbi = simd_abi::deduce_t<_Tp, __return_size>;

    // handle (optimize) the simple cases
    if constexpr (_Index == 0 && _Tuple::_S_first_size == __return_size)
      return __x.first._M_data;
    else if constexpr (_Index == 0 && _Total == _Combine)
      return __x;
    else if constexpr (_Index == 0 && _Tuple::_S_first_size >= __return_size)
      return __intrin_bitcast<__vector_type_t<_Tp, __return_size>>(
	__as_vector(__x.first));

    // recurse to skip unused data members at the beginning of _SimdTuple
    else if constexpr (__values_to_skip >= _Tuple::_S_first_size)
      { // recurse
	if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
	  {
	    constexpr int __parts_in_first
	      = _Tuple::_S_first_size / __values_per_part;
	    return __extract_part<_Index - __parts_in_first,
				  _Total - __parts_in_first, _Combine>(
	      __x.second);
	  }
	else
	  return __extract_part<__values_to_skip - _Tuple::_S_first_size,
				_Np - _Tuple::_S_first_size, __return_size>(
	    __x.second);
      }

    // extract from multiple _SimdTuple data members
    else if constexpr (__return_size > _Tuple::_S_first_size - __values_to_skip)
      {
#ifdef _GLIBCXX_SIMD_USE_ALIASING_LOADS
	const __may_alias<_Tp>* const element_ptr
	  = reinterpret_cast<const __may_alias<_Tp>*>(&__x) + __values_to_skip;
	return __as_vector(simd<_Tp, _RetAbi>(element_ptr, element_aligned));
#else
	[[maybe_unused]] constexpr size_t __offset = __values_to_skip;
	return __as_vector(simd<_Tp, _RetAbi>([&](auto __i) constexpr {
	  constexpr _SizeConstant<__i + __offset> __k;
	  return __x[__k];
	}));
#endif
      }

    // all of the return values are in __x.first
    else if constexpr (_Tuple::_S_first_size % __values_per_part == 0)
      return __extract_part<_Index, _Tuple::_S_first_size / __values_per_part,
			    _Combine>(__x.first);
    else
      return __extract_part<__values_to_skip, _Tuple::_S_first_size,
			    _Combine * __values_per_part>(__x.first);
  }

// }}}
// __fixed_size_storage_t<_Tp, _Np>{{{
template <typename _Tp, int _Np, typename _Tuple,
	  typename _Next = simd<_Tp, _AllNativeAbis::_BestAbi<_Tp, _Np>>,
	  int _Remain = _Np - int(_Next::size())>
  struct __fixed_size_storage_builder;

template <typename _Tp, int _Np>
  struct __fixed_size_storage
  : public __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp>> {};

template <typename _Tp, int _Np, typename... _As, typename _Next>
  struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
				      0>
  { using type = _SimdTuple<_Tp, _As..., typename _Next::abi_type>; };

template <typename _Tp, int _Np, typename... _As, typename _Next, int _Remain>
  struct __fixed_size_storage_builder<_Tp, _Np, _SimdTuple<_Tp, _As...>, _Next,
				      _Remain>
  {
    using type = typename __fixed_size_storage_builder<
      _Tp, _Remain, _SimdTuple<_Tp, _As..., typename _Next::abi_type>>::type;
  };

// }}}
// __autocvt_to_simd {{{
template <typename _Tp, bool = is_arithmetic_v<__remove_cvref_t<_Tp>>>
  struct __autocvt_to_simd
  {
    _Tp _M_data;
    using _TT = __remove_cvref_t<_Tp>;

    _GLIBCXX_SIMD_INTRINSIC
    operator _TT()
    { return _M_data; }

    _GLIBCXX_SIMD_INTRINSIC
    operator _TT&()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return _M_data;
    }

    _GLIBCXX_SIMD_INTRINSIC
    operator _TT*()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return &_M_data;
    }

    _GLIBCXX_SIMD_INTRINSIC
    constexpr __autocvt_to_simd(_Tp dd) : _M_data(dd) {}

    template <typename _Abi>
      _GLIBCXX_SIMD_INTRINSIC
      operator simd<typename _TT::value_type, _Abi>()
      { return {__private_init, _M_data}; }

    template <typename _Abi>
      _GLIBCXX_SIMD_INTRINSIC
      operator simd<typename _TT::value_type, _Abi>&()
      {
	return *reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(
	  &_M_data);
      }

    template <typename _Abi>
      _GLIBCXX_SIMD_INTRINSIC
      operator simd<typename _TT::value_type, _Abi>*()
      {
	return reinterpret_cast<simd<typename _TT::value_type, _Abi>*>(
	  &_M_data);
      }
  };

template <typename _Tp>
  __autocvt_to_simd(_Tp &&) -> __autocvt_to_simd<_Tp>;

template <typename _Tp>
  struct __autocvt_to_simd<_Tp, true>
  {
    using _TT = __remove_cvref_t<_Tp>;
    _Tp _M_data;
    fixed_size_simd<_TT, 1> _M_fd;

    _GLIBCXX_SIMD_INTRINSIC
    constexpr __autocvt_to_simd(_Tp dd) : _M_data(dd), _M_fd(_M_data) {}

    _GLIBCXX_SIMD_INTRINSIC
    ~__autocvt_to_simd()
    { _M_data = __data(_M_fd).first; }

    _GLIBCXX_SIMD_INTRINSIC
    operator fixed_size_simd<_TT, 1>()
    { return _M_fd; }

    _GLIBCXX_SIMD_INTRINSIC
    operator fixed_size_simd<_TT, 1> &()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return _M_fd;
    }

    _GLIBCXX_SIMD_INTRINSIC
    operator fixed_size_simd<_TT, 1> *()
    {
      static_assert(is_lvalue_reference<_Tp>::value, "");
      static_assert(!is_const<_Tp>::value, "");
      return &_M_fd;
    }
  };

// }}}

struct _CommonImplFixedSize;
template <int _Np, typename = __detail::__odr_helper> struct _SimdImplFixedSize;
template <int _Np, typename = __detail::__odr_helper> struct _MaskImplFixedSize;
// simd_abi::_Fixed {{{
template <int _Np>
  struct simd_abi::_Fixed
  {
    template <typename _Tp> static constexpr size_t _S_size = _Np;
    template <typename _Tp> static constexpr size_t _S_full_size = _Np;
    // validity traits {{{
    struct _IsValidAbiTag : public __bool_constant<(_Np > 0)> {};

    template <typename _Tp>
      struct _IsValidSizeFor
      : __bool_constant<(_Np <= simd_abi::max_fixed_size<_Tp>)> {};

    template <typename _Tp>
      struct _IsValid : conjunction<_IsValidAbiTag, __is_vectorizable<_Tp>,
				    _IsValidSizeFor<_Tp>> {};

    template <typename _Tp>
      static constexpr bool _S_is_valid_v = _IsValid<_Tp>::value;

    // }}}
    // _S_masked {{{
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
    _S_masked(_BitMask<_Np> __x)
    { return __x._M_sanitized(); }

    _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
    _S_masked(_SanitizedBitMask<_Np> __x)
    { return __x; }

    // }}}
    // _*Impl {{{
    using _CommonImpl = _CommonImplFixedSize;
    using _SimdImpl = _SimdImplFixedSize<_Np>;
    using _MaskImpl = _MaskImplFixedSize<_Np>;

    // }}}
    // __traits {{{
    template <typename _Tp, bool = _S_is_valid_v<_Tp>>
      struct __traits : _InvalidTraits {};

    template <typename _Tp>
      struct __traits<_Tp, true>
      {
	using _IsValid = true_type;
	using _SimdImpl = _SimdImplFixedSize<_Np>;
	using _MaskImpl = _MaskImplFixedSize<_Np>;

	// simd and simd_mask member types {{{
	using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;
	using _MaskMember = _SanitizedBitMask<_Np>;

	static constexpr size_t _S_simd_align
	  = std::__bit_ceil(_Np * sizeof(_Tp));

	static constexpr size_t _S_mask_align = alignof(_MaskMember);

	// }}}
	// _SimdBase / base class for simd, providing extra conversions {{{
	struct _SimdBase
	{
	  // The following ensures, function arguments are passed via the stack.
	  // This is important for ABI compatibility across TU boundaries
	  _GLIBCXX_SIMD_ALWAYS_INLINE
	  _SimdBase(const _SimdBase&) {}
	  _SimdBase() = default;

	  _GLIBCXX_SIMD_ALWAYS_INLINE
	  explicit operator const _SimdMember &() const
	  { return static_cast<const simd<_Tp, _Fixed>*>(this)->_M_data; }

	  _GLIBCXX_SIMD_ALWAYS_INLINE
	  explicit operator array<_Tp, _Np>() const
	  {
	    array<_Tp, _Np> __r;
	    // _SimdMember can be larger because of higher alignment
	    static_assert(sizeof(__r) <= sizeof(_SimdMember), "");
	    __builtin_memcpy(__r.data(), &static_cast<const _SimdMember&>(*this),
			     sizeof(__r));
	    return __r;
	  }
	};

	// }}}
	// _MaskBase {{{
	// empty. The bitset interface suffices
	struct _MaskBase {};

	// }}}
	// _SimdCastType {{{
	struct _SimdCastType
	{
	  _GLIBCXX_SIMD_ALWAYS_INLINE
	  _SimdCastType(const array<_Tp, _Np>&);
	  _GLIBCXX_SIMD_ALWAYS_INLINE
	  _SimdCastType(const _SimdMember& dd) : _M_data(dd) {}
	  _GLIBCXX_SIMD_ALWAYS_INLINE
	  explicit operator const _SimdMember &() const { return _M_data; }

	private:
	  const _SimdMember& _M_data;
	};

	// }}}
	// _MaskCastType {{{
	class _MaskCastType
	{
	  _MaskCastType() = delete;
	};
	// }}}
      };
    // }}}
  };

// }}}
// _CommonImplFixedSize {{{
struct _CommonImplFixedSize
{
  // _S_store {{{
  template <typename _Tp, typename... _As>
    _GLIBCXX_SIMD_INTRINSIC static void
    _S_store(const _SimdTuple<_Tp, _As...>& __x, void* __addr)
    {
      constexpr size_t _Np = _SimdTuple<_Tp, _As...>::_S_size();
      __builtin_memcpy(__addr, &__x, _Np * sizeof(_Tp));
    }

  // }}}
};

// }}}
// _SimdImplFixedSize {{{1
// fixed_size should not inherit from _SimdMathFallback in order for
// specializations in the used _SimdTuple Abis to get used
template <int _Np, typename>
  struct _SimdImplFixedSize
  {
    // member types {{{2
    using _MaskMember = _SanitizedBitMask<_Np>;

    template <typename _Tp>
      using _SimdMember = __fixed_size_storage_t<_Tp, _Np>;

    template <typename _Tp>
      static constexpr size_t _S_tuple_size = _SimdMember<_Tp>::_S_tuple_size;

    template <typename _Tp>
      using _Simd = simd<_Tp, simd_abi::fixed_size<_Np>>;

    template <typename _Tp>
      using _TypeTag = _Tp*;

    // broadcast {{{2
    template <typename _Tp>
      static constexpr inline _SimdMember<_Tp> _S_broadcast(_Tp __x) noexcept
      {
	return _SimdMember<_Tp>::_S_generate([&](auto __meta) constexpr {
	  return __meta._S_broadcast(__x);
	});
      }

    // _S_generator {{{2
    template <typename _Fp, typename _Tp>
      static constexpr inline _SimdMember<_Tp> _S_generator(_Fp&& __gen,
							    _TypeTag<_Tp>)
      {
	return _SimdMember<_Tp>::_S_generate([&__gen](auto __meta) constexpr {
	  return __meta._S_generator(
	    [&](auto __i) constexpr {
	      return __i < _Np ? __gen(_SizeConstant<__meta._S_offset + __i>())
			       : 0;
	    },
	    _TypeTag<_Tp>());
	});
      }

    // _S_load {{{2
    template <typename _Tp, typename _Up>
      static inline _SimdMember<_Tp> _S_load(const _Up* __mem,
					     _TypeTag<_Tp>) noexcept
      {
	return _SimdMember<_Tp>::_S_generate([&](auto __meta) {
	  return __meta._S_load(&__mem[__meta._S_offset], _TypeTag<_Tp>());
	});
      }

    // _S_masked_load {{{2
    template <typename _Tp, typename... _As, typename _Up>
      static inline _SimdTuple<_Tp, _As...>
      _S_masked_load(const _SimdTuple<_Tp, _As...>& __old,
		     const _MaskMember __bits, const _Up* __mem) noexcept
      {
	auto __merge = __old;
	__for_each(__merge, [&](auto __meta, auto& __native) {
	  if (__meta._S_submask(__bits).any())
#pragma GCC diagnostic push
	  // __mem + __mem._S_offset could be UB ([expr.add]/4.3, but it punts
	  // the responsibility for avoiding UB to the caller of the masked load
	  // via the mask. Consequently, the compiler may assume this branch is
	  // unreachable, if the pointer arithmetic is UB.
#pragma GCC diagnostic ignored "-Warray-bounds"
	    __native
	      = __meta._S_masked_load(__native, __meta._S_make_mask(__bits),
				      __mem + __meta._S_offset);
#pragma GCC diagnostic pop
	});
	return __merge;
      }

    // _S_store {{{2
    template <typename _Tp, typename _Up>
      static inline void _S_store(const _SimdMember<_Tp>& __v, _Up* __mem,
				  _TypeTag<_Tp>) noexcept
      {
	__for_each(__v, [&](auto __meta, auto __native) {
	  __meta._S_store(__native, &__mem[__meta._S_offset], _TypeTag<_Tp>());
	});
      }

    // _S_masked_store {{{2
    template <typename _Tp, typename... _As, typename _Up>
      static inline void _S_masked_store(const _SimdTuple<_Tp, _As...>& __v,
					 _Up* __mem,
					 const _MaskMember __bits) noexcept
      {
	__for_each(__v, [&](auto __meta, auto __native) {
	  if (__meta._S_submask(__bits).any())
#pragma GCC diagnostic push
	  // __mem + __mem._S_offset could be UB ([expr.add]/4.3, but it punts
	  // the responsibility for avoiding UB to the caller of the masked
	  // store via the mask. Consequently, the compiler may assume this
	  // branch is unreachable, if the pointer arithmetic is UB.
#pragma GCC diagnostic ignored "-Warray-bounds"
	    __meta._S_masked_store(__native, __mem + __meta._S_offset,
				   __meta._S_make_mask(__bits));
#pragma GCC diagnostic pop
	});
      }

    // negation {{{2
    template <typename _Tp, typename... _As>
      static inline _MaskMember
      _S_negate(const _SimdTuple<_Tp, _As...>& __x) noexcept
      {
	_MaskMember __bits = 0;
	__for_each(
	  __x, [&__bits](auto __meta, auto __native) constexpr {
	    __bits
	      |= __meta._S_mask_to_shifted_ullong(__meta._S_negate(__native));
	  });
	return __bits;
      }

    // reductions {{{2
    template <typename _Tp, typename _BinaryOperation>
      static constexpr inline _Tp _S_reduce(const _Simd<_Tp>& __x,
					    const _BinaryOperation& __binary_op)
      {
	using _Tup = _SimdMember<_Tp>;
	const _Tup& __tup = __data(__x);
	if constexpr (_Tup::_S_tuple_size == 1)
	  return _Tup::_FirstAbi::_SimdImpl::_S_reduce(
	    __tup.template _M_simd_at<0>(), __binary_op);
	else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 2
			   && _Tup::_SecondType::_S_size() == 1)
	  {
	    return __binary_op(simd<_Tp, simd_abi::scalar>(
				 reduce(__tup.template _M_simd_at<0>(),
					__binary_op)),
			       __tup.template _M_simd_at<1>())[0];
	  }
	else if constexpr (_Tup::_S_tuple_size == 2 && _Tup::_S_size() > 4
			   && _Tup::_SecondType::_S_size() == 2)
	  {
	    return __binary_op(
	      simd<_Tp, simd_abi::scalar>(
		reduce(__tup.template _M_simd_at<0>(), __binary_op)),
	      simd<_Tp, simd_abi::scalar>(
		reduce(__tup.template _M_simd_at<1>(), __binary_op)))[0];
	  }
	else
	  {
	    const auto& __x2 = __call_with_n_evaluations<
	      __div_roundup(_Tup::_S_tuple_size, 2)>(
	      [](auto __first_simd, auto... __remaining) {
		if constexpr (sizeof...(__remaining) == 0)
		  return __first_simd;
		else
		  {
		    using _Tup2
		      = _SimdTuple<_Tp,
				   typename decltype(__first_simd)::abi_type,
				   typename decltype(__remaining)::abi_type...>;
		    return fixed_size_simd<_Tp, _Tup2::_S_size()>(
		      __private_init,
		      __make_simd_tuple(__first_simd, __remaining...));
		  }
	      },
	      [&](auto __i) {
		auto __left = __tup.template _M_simd_at<2 * __i>();
		if constexpr (2 * __i + 1 == _Tup::_S_tuple_size)
		  return __left;
		else
		  {
		    auto __right = __tup.template _M_simd_at<2 * __i + 1>();
		    using _LT = decltype(__left);
		    using _RT = decltype(__right);
		    if constexpr (_LT::size() == _RT::size())
		      return __binary_op(__left, __right);
		    else
		      {
			_GLIBCXX_SIMD_USE_CONSTEXPR_API
			typename _LT::mask_type __k(
			  __private_init,
			  [](auto __j) constexpr { return __j < _RT::size(); });
			_LT __ext_right = __left;
			where(__k, __ext_right)
			  = __proposed::resizing_simd_cast<_LT>(__right);
			where(__k, __left) = __binary_op(__left, __ext_right);
			return __left;
		      }
		  }
	      });
	    return reduce(__x2, __binary_op);
	  }
      }

    // _S_min, _S_max {{{2
    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_min(const _SimdTuple<_Tp, _As...>& __a,
	     const _SimdTuple<_Tp, _As...>& __b)
      {
	return __a._M_apply_per_chunk(
	  [](auto __impl, auto __aa, auto __bb) constexpr {
	    return __impl._S_min(__aa, __bb);
	  },
	  __b);
      }

    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_max(const _SimdTuple<_Tp, _As...>& __a,
	     const _SimdTuple<_Tp, _As...>& __b)
      {
	return __a._M_apply_per_chunk(
	  [](auto __impl, auto __aa, auto __bb) constexpr {
	    return __impl._S_max(__aa, __bb);
	  },
	  __b);
      }

    // _S_complement {{{2
    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_complement(const _SimdTuple<_Tp, _As...>& __x) noexcept
      {
	return __x._M_apply_per_chunk([](auto __impl, auto __xx) constexpr {
	  return __impl._S_complement(__xx);
	});
      }

    // _S_unary_minus {{{2
    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_unary_minus(const _SimdTuple<_Tp, _As...>& __x) noexcept
      {
	return __x._M_apply_per_chunk([](auto __impl, auto __xx) constexpr {
	  return __impl._S_unary_minus(__xx);
	});
      }

    // arithmetic operators {{{2

#define _GLIBCXX_SIMD_FIXED_OP(name_, op_)                                     \
    template <typename _Tp, typename... _As>                                   \
      static inline constexpr _SimdTuple<_Tp, _As...> name_(                   \
	const _SimdTuple<_Tp, _As...>& __x, const _SimdTuple<_Tp, _As...>& __y)\
      {                                                                        \
	return __x._M_apply_per_chunk(                                         \
	  [](auto __impl, auto __xx, auto __yy) constexpr {                    \
	    return __impl.name_(__xx, __yy);                                   \
	  },                                                                   \
	  __y);                                                                \
      }

    _GLIBCXX_SIMD_FIXED_OP(_S_plus, +)
    _GLIBCXX_SIMD_FIXED_OP(_S_minus, -)
    _GLIBCXX_SIMD_FIXED_OP(_S_multiplies, *)
    _GLIBCXX_SIMD_FIXED_OP(_S_divides, /)
    _GLIBCXX_SIMD_FIXED_OP(_S_modulus, %)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_and, &)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_or, |)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_xor, ^)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_left, <<)
    _GLIBCXX_SIMD_FIXED_OP(_S_bit_shift_right, >>)
#undef _GLIBCXX_SIMD_FIXED_OP

    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_bit_shift_left(const _SimdTuple<_Tp, _As...>& __x, int __y)
      {
	return __x._M_apply_per_chunk([__y](auto __impl, auto __xx) constexpr {
	  return __impl._S_bit_shift_left(__xx, __y);
	});
      }

    template <typename _Tp, typename... _As>
      static inline constexpr _SimdTuple<_Tp, _As...>
      _S_bit_shift_right(const _SimdTuple<_Tp, _As...>& __x, int __y)
      {
	return __x._M_apply_per_chunk([__y](auto __impl, auto __xx) constexpr {
	  return __impl._S_bit_shift_right(__xx, __y);
	});
      }

  // math {{{2
#define _GLIBCXX_SIMD_APPLY_ON_TUPLE(_RetTp, __name)                           \
    template <typename _Tp, typename... _As, typename... _More>                \
      static inline __fixed_size_storage_t<_RetTp, _Np>                        \
	_S_##__name(const _SimdTuple<_Tp, _As...>& __x,                        \
		    const _More&... __more)                                    \
      {                                                                        \
	if constexpr (sizeof...(_More) == 0)                                   \
	  {                                                                    \
	    if constexpr (is_same_v<_Tp, _RetTp>)                              \
	      return __x._M_apply_per_chunk(                                   \
		[](auto __impl, auto __xx) constexpr {                         \
		  using _V = typename decltype(__impl)::simd_type;             \
		  return __data(__name(_V(__private_init, __xx)));             \
		});                                                            \
	    else                                                               \
	      return __optimize_simd_tuple(                                    \
		__x.template _M_apply_r<_RetTp>([](auto __impl, auto __xx) {   \
		  return __impl._S_##__name(__xx);                             \
		}));                                                           \
	  }                                                                    \
	else if constexpr (                                                    \
	  is_same_v<                                                           \
	    _Tp,                                                               \
	    _RetTp> && (... && is_same_v<_SimdTuple<_Tp, _As...>, _More>) )    \
	  return __x._M_apply_per_chunk(                                       \
	    [](auto __impl, auto __xx, auto... __pack) constexpr {             \
	      using _V = typename decltype(__impl)::simd_type;                 \
	      return __data(__name(_V(__private_init, __xx),                   \
				   _V(__private_init, __pack)...));            \
	    },                                                                 \
	    __more...);                                                        \
	else if constexpr (is_same_v<_Tp, _RetTp>)                             \
	  return __x._M_apply_per_chunk(                                       \
	    [](auto __impl, auto __xx, auto... __pack) constexpr {             \
	      using _V = typename decltype(__impl)::simd_type;                 \
	      return __data(__name(_V(__private_init, __xx),                   \
				   __autocvt_to_simd(__pack)...));             \
	    },                                                                 \
	    __more...);                                                        \
	else                                                                   \
	  __assert_unreachable<_Tp>();                                         \
      }

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acos)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asin)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atan2)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cos)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sin)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tan)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, acosh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, asinh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, atanh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cosh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sinh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tanh)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, exp2)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, expm1)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, ilogb)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log10)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log1p)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, log2)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, logb)
    // modf implemented in simd_math.h
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp,
				 scalbn) // double scalbn(double x, int exp);
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, scalbln)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, cbrt)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, abs)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fabs)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, pow)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, sqrt)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erf)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, erfc)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, lgamma)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, tgamma)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, trunc)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ceil)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, floor)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nearbyint)

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, rint)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lrint)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llrint)

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, round)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long, lround)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(long long, llround)

    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, ldexp)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmod)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, remainder)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, copysign)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, nextafter)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fdim)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmax)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fmin)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(_Tp, fma)
    _GLIBCXX_SIMD_APPLY_ON_TUPLE(int, fpclassify)
#undef _GLIBCXX_SIMD_APPLY_ON_TUPLE

    template <typename _Tp, typename... _Abis>
      static _SimdTuple<_Tp, _Abis...> _S_remquo(
	const _SimdTuple<_Tp, _Abis...>& __x,
	const _SimdTuple<_Tp, _Abis...>& __y,
	__fixed_size_storage_t<int, _SimdTuple<_Tp, _Abis...>::_S_size()>* __z)
      {
	return __x._M_apply_per_chunk(
	  [](auto __impl, const auto __xx, const auto __yy, auto& __zz) {
	    return __impl._S_remquo(__xx, __yy, &__zz);
	  },
	  __y, *__z);
      }

    template <typename _Tp, typename... _As>
      static inline _SimdTuple<_Tp, _As...>
      _S_frexp(const _SimdTuple<_Tp, _As...>& __x,
	       __fixed_size_storage_t<int, _Np>& __exp) noexcept
      {
	return __x._M_apply_per_chunk(
	  [](auto __impl, const auto& __a, auto& __b) {
	    return __data(
	      frexp(typename decltype(__impl)::simd_type(__private_init, __a),
		    __autocvt_to_simd(__b)));
	  },
	  __exp);
      }

#define _GLIBCXX_SIMD_TEST_ON_TUPLE_(name_)                                    \
    template <typename _Tp, typename... _As>                                   \
      static inline _MaskMember                                                \
	_S_##name_(const _SimdTuple<_Tp, _As...>& __x) noexcept                \
      {                                                                        \
	return _M_test([](auto __impl,                                         \
			  auto __xx) { return __impl._S_##name_(__xx); },      \
		       __x);                                                   \
      }

    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isinf)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isfinite)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnan)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(isnormal)
    _GLIBCXX_SIMD_TEST_ON_TUPLE_(signbit)
#undef _GLIBCXX_SIMD_TEST_ON_TUPLE_

    // _S_increment & _S_decrement{{{2
    template <typename... _Ts>
      _GLIBCXX_SIMD_INTRINSIC static constexpr void
      _S_increment(_SimdTuple<_Ts...>& __x)
      {
	__for_each(
	  __x, [](auto __meta, auto& native) constexpr {
	    __meta._S_increment(native);
	  });
      }

    template <typename... _Ts>
      _GLIBCXX_SIMD_INTRINSIC static constexpr void
      _S_decrement(_SimdTuple<_Ts...>& __x)
      {
	__for_each(
	  __x, [](auto __meta, auto& native) constexpr {
	    __meta._S_decrement(native);
	  });
      }

    // compares {{{2
#define _GLIBCXX_SIMD_CMP_OPERATIONS(__cmp)                                    \
    template <typename _Tp, typename... _As>                                   \
      _GLIBCXX_SIMD_INTRINSIC constexpr static _MaskMember                     \
      __cmp(const _SimdTuple<_Tp, _As...>& __x,                                \
	    const _SimdTuple<_Tp, _As...>& __y)                                \
      {                                                                        \
	return _M_test(                                                        \
	  [](auto __impl, auto __xx, auto __yy) constexpr {                    \
	    return __impl.__cmp(__xx, __yy);                                   \
	  },                                                                   \
	  __x, __y);                                                           \
      }

    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_equal_to)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_not_equal_to)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_less_equal)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isless)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessequal)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreater)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isgreaterequal)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_islessgreater)
    _GLIBCXX_SIMD_CMP_OPERATIONS(_S_isunordered)
#undef _GLIBCXX_SIMD_CMP_OPERATIONS

    // smart_reference access {{{2
    template <typename _Tp, typename... _As, typename _Up>
      _GLIBCXX_SIMD_INTRINSIC static void _S_set(_SimdTuple<_Tp, _As...>& __v,
						 int __i, _Up&& __x) noexcept
      { __v._M_set(__i, static_cast<_Up&&>(__x)); }

    // _S_masked_assign {{{2
    template <typename _Tp, typename... _As>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
		       const __type_identity_t<_SimdTuple<_Tp, _As...>>& __rhs)
      {
	__for_each(
	  __lhs, __rhs,
	  [&](auto __meta, auto& __native_lhs, auto __native_rhs) constexpr {
	    __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
				    __native_rhs);
	  });
      }

    // Optimization for the case where the RHS is a scalar. No need to broadcast
    // the scalar to a simd first.
    template <typename _Tp, typename... _As>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_assign(const _MaskMember __bits, _SimdTuple<_Tp, _As...>& __lhs,
		       const __type_identity_t<_Tp> __rhs)
      {
	__for_each(
	  __lhs, [&](auto __meta, auto& __native_lhs) constexpr {
	    __meta._S_masked_assign(__meta._S_make_mask(__bits), __native_lhs,
				    __rhs);
	  });
      }

    // _S_masked_cassign {{{2
    template <typename _Op, typename _Tp, typename... _As>
      static inline void _S_masked_cassign(const _MaskMember __bits,
					   _SimdTuple<_Tp, _As...>& __lhs,
					   const _SimdTuple<_Tp, _As...>& __rhs,
					   _Op __op)
      {
	__for_each(
	  __lhs, __rhs,
	  [&](auto __meta, auto& __native_lhs, auto __native_rhs) constexpr {
	    __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
					      __native_lhs, __native_rhs, __op);
	  });
      }

    // Optimization for the case where the RHS is a scalar. No need to broadcast
    // the scalar to a simd first.
    template <typename _Op, typename _Tp, typename... _As>
      static inline void _S_masked_cassign(const _MaskMember __bits,
					   _SimdTuple<_Tp, _As...>& __lhs,
					   const _Tp& __rhs, _Op __op)
      {
	__for_each(
	  __lhs, [&](auto __meta, auto& __native_lhs) constexpr {
	    __meta.template _S_masked_cassign(__meta._S_make_mask(__bits),
					      __native_lhs, __rhs, __op);
	  });
      }

    // _S_masked_unary {{{2
    template <template <typename> class _Op, typename _Tp, typename... _As>
      static inline _SimdTuple<_Tp, _As...>
      _S_masked_unary(const _MaskMember __bits, const _SimdTuple<_Tp, _As...>& __v)
      {
	return __v._M_apply_wrapped([&__bits](auto __meta,
					      auto __native) constexpr {
	  return __meta.template _S_masked_unary<_Op>(__meta._S_make_mask(
							__bits),
						      __native);
	});
      }

    // }}}2
  };

// _MaskImplFixedSize {{{1
template <int _Np, typename>
  struct _MaskImplFixedSize
  {
    static_assert(
      sizeof(_ULLong) * __CHAR_BIT__ >= _Np,
      "The fixed_size implementation relies on one _ULLong being able to store "
      "all boolean elements."); // required in load & store

    // member types {{{
    using _Abi = simd_abi::fixed_size<_Np>;

    using _MaskMember = _SanitizedBitMask<_Np>;

    template <typename _Tp>
      using _FirstAbi = typename __fixed_size_storage_t<_Tp, _Np>::_FirstAbi;

    template <typename _Tp>
      using _TypeTag = _Tp*;

    // }}}
    // _S_broadcast {{{
    template <typename>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
      _S_broadcast(bool __x)
      { return __x ? ~_MaskMember() : _MaskMember(); }

    // }}}
    // _S_load {{{
    template <typename>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
      _S_load(const bool* __mem)
      {
	using _Ip = __int_for_sizeof_t<bool>;
	// the following load uses element_aligned and relies on __mem already
	// carrying alignment information from when this load function was
	// called.
	const simd<_Ip, _Abi> __bools(reinterpret_cast<const __may_alias<_Ip>*>(
					__mem),
				      element_aligned);
	return __data(__bools != 0);
      }

    // }}}
    // _S_to_bits {{{
    template <bool _Sanitized>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
      _S_to_bits(_BitMask<_Np, _Sanitized> __x)
      {
	if constexpr (_Sanitized)
	  return __x;
	else
	  return __x._M_sanitized();
      }

    // }}}
    // _S_convert {{{
    template <typename _Tp, typename _Up, typename _UAbi>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
      _S_convert(simd_mask<_Up, _UAbi> __x)
      {
	return _UAbi::_MaskImpl::_S_to_bits(__data(__x))
	  .template _M_extract<0, _Np>();
      }

    // }}}
    // _S_from_bitmask {{{2
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember
      _S_from_bitmask(_MaskMember __bits, _TypeTag<_Tp>) noexcept
      { return __bits; }

    // _S_load {{{2
    static inline _MaskMember _S_load(const bool* __mem) noexcept
    {
      // TODO: _UChar is not necessarily the best type to use here. For smaller
      // _Np _UShort, _UInt, _ULLong, float, and double can be more efficient.
      _ULLong __r = 0;
      using _Vs = __fixed_size_storage_t<_UChar, _Np>;
      __for_each(_Vs{}, [&](auto __meta, auto) {
	__r |= __meta._S_mask_to_shifted_ullong(
	  __meta._S_mask_impl._S_load(&__mem[__meta._S_offset],
				      _SizeConstant<__meta._S_size()>()));
      });
      return __r;
    }

    // _S_masked_load {{{2
    static inline _MaskMember _S_masked_load(_MaskMember __merge,
					     _MaskMember __mask,
					     const bool* __mem) noexcept
    {
      _BitOps::_S_bit_iteration(__mask.to_ullong(), [&](auto __i) {
	__merge.set(__i, __mem[__i]);
      });
      return __merge;
    }

    // _S_store {{{2
    static inline void _S_store(const _MaskMember __bitmask,
				bool* __mem) noexcept
    {
      if constexpr (_Np == 1)
	__mem[0] = __bitmask[0];
      else
	_FirstAbi<_UChar>::_CommonImpl::_S_store_bool_array(__bitmask, __mem);
    }

    // _S_masked_store {{{2
    static inline void _S_masked_store(const _MaskMember __v, bool* __mem,
				       const _MaskMember __k) noexcept
    {
      _BitOps::_S_bit_iteration(__k, [&](auto __i) { __mem[__i] = __v[__i]; });
    }

    // logical and bitwise operators {{{2
    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_logical_and(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x & __y; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_logical_or(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x | __y; }

    _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember
    _S_bit_not(const _MaskMember& __x) noexcept
    { return ~__x; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_bit_and(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x & __y; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_bit_or(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x | __y; }

    _GLIBCXX_SIMD_INTRINSIC static _MaskMember
    _S_bit_xor(const _MaskMember& __x, const _MaskMember& __y) noexcept
    { return __x ^ __y; }

    // smart_reference access {{{2
    _GLIBCXX_SIMD_INTRINSIC static void _S_set(_MaskMember& __k, int __i,
					       bool __x) noexcept
    { __k.set(__i, __x); }

    // _S_masked_assign {{{2
    _GLIBCXX_SIMD_INTRINSIC static void
    _S_masked_assign(const _MaskMember __k, _MaskMember& __lhs,
		     const _MaskMember __rhs)
    { __lhs = (__lhs & ~__k) | (__rhs & __k); }

    // Optimization for the case where the RHS is a scalar.
    _GLIBCXX_SIMD_INTRINSIC static void _S_masked_assign(const _MaskMember __k,
							 _MaskMember& __lhs,
							 const bool __rhs)
    {
      if (__rhs)
	__lhs |= __k;
      else
	__lhs &= ~__k;
    }

    // }}}2
    // _S_all_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_all_of(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).all(); }

    // }}}
    // _S_any_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_any_of(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).any(); }

    // }}}
    // _S_none_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_none_of(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).none(); }

    // }}}
    // _S_some_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool
      _S_some_of([[maybe_unused]] simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (_Np == 1)
	  return false;
	else
	  return __data(__k).any() && !__data(__k).all();
      }

    // }}}
    // _S_popcount {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int _S_popcount(simd_mask<_Tp, _Abi> __k)
      { return __data(__k).count(); }

    // }}}
    // _S_find_first_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_first_set(simd_mask<_Tp, _Abi> __k)
      { return std::__countr_zero(__data(__k).to_ullong()); }

    // }}}
    // _S_find_last_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_last_set(simd_mask<_Tp, _Abi> __k)
      { return std::__bit_width(__data(__k).to_ullong()) - 1; }

    // }}}
  };
// }}}1

_GLIBCXX_SIMD_END_NAMESPACE
#endif // __cplusplus >= 201703L
#endif // _GLIBCXX_EXPERIMENTAL_SIMD_FIXED_SIZE_H_

// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  z     <             z      =            7z     =            @z     =            Gz     =            Iz     =            Mz     =            Qz     =            z     =            z      =            z     $=            z     (=            z     ,=            {     0=            {     4=            {     8=            {     <=            {     @=            {     D=            |     H=            [|     L=            \|     P=            ]|     T=            b|     X=            z|     \=            |     `=            |     d=            |     h=            |     l=            |     p=            |     t=            |     x=            |     |=            T~     =            X~     =            Y~     =            [~     =            ]~     =            _~     =            a~     =            f~     =            ~     =            ~     =            ~     =            ~     =            ~     =                 =                 =                 =                  =            $     =            %     =            &     =            +     =            [     =            \     =            ]     =            b     =            p     =            v     =            w     =                 =                 =                 =                 =                  >                 >                 >                 >                 >                 >                 >                 >                  >                  $>            '     (>            )     ,>            *     0>            +     4>            /     8>            M     <>            N     @>            O     D>            Q     H>            S     L>            X     P>            Y     T>            _     X>            `     \>            a     `>            c     d>            e     h>            j     l>            |     p>                 t>                 x>                 |>                 >                 >                 >                 >            #     >            '     >            *     >            +     >            6     >            7     >                 >                 >                 >                 >                 >                 >                 >            Ƃ     >                 >                 >                  >                 >                 >                 >                 >                 >                 >                 >                 >                 >                 >                 >                  ?                 ?                 ?                 ?            Y     ?            \     ?            ^     ?            `     ?            b      ?            d     $?            i     (?            {     ,?                 0?                 4?                 8?                 <?                 @?                 D?                 H?            J     L?            K     P?            P     T?            U     X?            `     \?            g     `?            i     d?            j     h?            m     l?            w     p?            1     t?            2     x?            3     |?            5     ?            7     ?            <     ?                 ?                  ?            ?     ?            @     ?            G     ?            H     ?            I     ?                 ?                 ?                 ?                 ?                  ?                 ?                 ?                 ?                 ?            e     ?            h     ?            m     ?                 ?                  ?            /     ?            0     ?            6     ?            7     ?            z     ?            {     ?                 ?                 ?                  @                 @            g      @            s      @                  @                  @                  @                 @                  @                 $@            _     (@            `     ,@            e     0@            j     4@            p     8@            v     <@                 @@                 D@                 H@                 L@                  P@                 T@                 X@                 \@                 `@                 d@                 h@                 l@                 p@                 t@                 x@                 |@                 @            v     @            {     @                 @                 @                 @                 @                 @            ǎ     @            Ύ     @            Ў     @            Ҏ     @            ӎ     @            Ԏ     @            ߎ     @            ߏ     @                 @                 @                 @                 @                 @                 @                 @                 @            ǐ     @            ɐ     @            ѐ     @            Ӑ     @            Ԑ     @            ؐ     @            ܐ     @                 @                  A                 A                 A                 A                 A                 A                 A                 A                  A                 $A            $     (A            &     ,A            (     0A            )     4A            4     8A            ;     <A                 @A                 DA                 HA                 LA                 PA                 TA                 XA                 \A                 `A                 dA                 hA                 lA                 pA                  tA                 xA                 |A                 A            0     A            1     A            2     A            4     A            6     A            8     A            :     A            ?     A            ѕ     A                 A                 A                 A                 A                 A                 A                 A                 A                 A                 A                  A            "     A            $     A            )     A                 A                 A                 A                 A                 A                 A                 A                 A                  B                 B                 B            ×     B            ŗ     B            Ǘ     B            ̗     B            ї     B                  B                 $B                 (B                 ,B                 0B                 4B                 8B                 <B                 @B                 DB                 HB                 LB                 PB            $     TB            x     XB                 \B                 `B                 dB                 hB                 lB                 pB                 tB                 xB                 |B                 B                 B                  B                 B                 B            	     B            Қ     B            Ӛ     B            ՚     B            ښ     B                 B                 B                 B                 B                 B                 B                 B                 B            ̛     B            ͛     B            ϛ     B            ԛ     B                 B                 B                 B                 B                 B                 B                 B                 B                 B                 B                  C                 C            Ü     C            Ŝ     C            ǜ     C            ̜     C                 C                 C                  C                 $C                 (C                 ,C                 0C                 4C                 8C            v     <C            z     @C            {     DC            }     HC                 LC                 PC                 TC                 XC                 \C                 `C                 dC                 hC                 lC                 pC                 tC                 xC                 |C            3     C            4     C            5     C            7     C            9     C            ;     C            =     C            B     C            I     C            P     C            W     C            X     C            Y     C            ˡ     C            ̡     C            Ρ     C            ӡ     C                 C                 C                 C                 C                 C            	     C                 C                 C                 C                 C                 C                 C                 C                 C                 C                  D                 D                 D                 D            Ǣ     D            ɢ     D            ˢ     D            ̢     D            ͢      D                 $D                 (D                 ,D                 0D                 4D            
     8D                 <D                 @D                 DD                 HD                 LD                 PD                 TD                 XD                 \D            g     `D            h     dD            i     hD            k     lD            m     pD            o     tD            q     xD            v     |D                 D                 D                 D                 D                 D                 D                 D            ¤     D            Ƥ     D                 D                 D                 D                 D                 D                 D                 D                 D            ֥     D                 D                 D                 D                 D                 D            v     D            w     D            y     D            {     D                 D                 D                 D                 D                 D                  E                 E                 E                 E            	     E                 E                 E                 E                  E            9     $E            :     (E            ?     ,E            O     0E            P     4E            U     8E            `     <E            g     @E            i     DE            k     HE            m     LE            q     PE            t     TE            {     XE            v     \E            w     `E            x     dE            z     hE            |     lE            ~     pE                 tE                 xE            7     |E            @     E            F     E            G     E                 E            ª     E            Ǫ     E            ת     E            ت     E            ݪ     E                 E                 E                 E                 E            =     E            >     E            ?     E            D     E            ߫     E                 E                 E                 E            \     E            ]     E            b     E            r     E            s     E            x     E                 E                 E                 E                 E                 E                  F                 F                 F            խ     F            ֭     F            ׭     F            ٭     F            ۭ     F            ݭ      F            ߭     $F                 (F                 ,F                 0F                 4F                 8F                 <F                 @F                 DF                 HF                 LF                 PF                 TF                 XF                 \F                 `F                 dF                 hF            
     lF                 pF                 tF                 xF                 |F                 F                 F                 F                 F                 F                 F                 F                 F                 F            '     F            *     F            +     F            -     F            2     F                 F                 F                 F                 F                 F                 F            Q     F            T     F            U     F            W     F            \     F                 F                 F            	     F                  F            %     F                 F                 F                  G                 G                 G                 G                 G            Ʋ     G            в     G            ײ     G            ٲ      G            ۲     $G            ܲ     (G                 ,G                 0G                 4G                 8G                 <G                 @G                 DG                 HG                 LG            Ǵ     PG            ɴ     TG            ˴     XG            ʹ     \G            δ     `G            ϴ     dG            Ӵ     hG                 lG                 pG                 tG                 xG                 |G                 G            !     G            &     G                 G                 G                 G                  G            "     G            $     G            &     G            +     G                 G                 G                 G                 G                 G            R     G            S     G            U     G            Z     G            `     G            g     G            h     G            k     G                 G                 G                 G                 G                 G                 G                 G                 G                  H            
     H                 H                 H                  H            '     H            )     H            .     H            0      H            1     $H            2     (H            6     ,H            '     0H            +     4H            ,     8H            .     <H            0     @H            2     DH            4     HH            9     LH                 PH                  TH                  XH                  \H                  `H                  dH            n                                                                                        (                   0                    8                    H                    P                   X                    `                    p          	                         3                l                                        m                                        l           $             [      (                     0                  4          U          <                   @          U          H             $     L          U          T             %     X          U          `             &     d          U          l             '     p          U          x             (     |          U                       i*               U                       +               U                       :               g                       &>               g                       @               g                       AC               g                       Y               g                       \               g                                                                                                   ߓ                                     ŕ                                                                         Ŷ     $                   ,                 0                                 5P                   S                                      T                   [                                       U      $             H[      (                   0             Y      4             \      8            
       @             js     D             s     H             b       P             s     T             s     X             *       0                    8                   @             `9      H             7      `             `8                  @      P                             ;                     {                        Z\                C                       m\                                                       /                                           +                   8
                   	      8                   @             +      H             8
      P             H	      8         {          P         ;           .symtab .strtab .shstrtab .note.gnu.build-id .note.Linux .rela.text .rela.text.unlikely .rela.init.text .rela.exit.text .altinstr_replacement .rela__ksymtab __kcrctab .rela.altinstructions .rela.rodata __ksymtab_strings .rodata.str1.8 .rela__mcount_loc .rela.smp_locks .rodata.str1.1 .modinfo .rela.retpoline_sites .rela.return_sites .orc_unwind .rela.orc_unwind_ip .rela__param .rodata.cst2 __versions .rela__bug_table .rela__jump_table .rela.data .rela.exit.data .rela.init.data .data.once .rela.static_call_sites .rela__dyndbg .rela.gnu.linkonce.this_module .bss .comment .note.GNU-stack .BTF .gnu_debuglink                                                                                          @       $                              .                     d       <                              ?                                                        :      @                    d     <                    J                     ?     n                             E      @               r           <                    ^                          a                              Y      @                          <                    n                          $                              i      @                    x       <   	                 y                     2     &                                                   X                                         @               (     H       <                                         d                                                        h     l                                    @               p           <                                              `-                                    @                     (       <                          2               @                                        2               X                                                      6                                        @               H     XA      <                                                                                  @                    `      <                         2                                                                                                      +                    g                                  &     @                           <                    A                    !                                  <     @                    .      <                    O                    +)     l                             `                    Ǖ     hH                             [     @               $     p     <                     t                    0     x                              o     @                           <   "                 |                                                                               @+                                                   
     8                                  @               (           <   &                                     8     `                                   @                          <   (                                          l                                   @                           <   *                                     
                                        @               `            <   ,                                     
                                        @               x            <   .                                      
                                                       /
                                        @                           <   1                 
                    H
     p                                   @                            <   3                                     
                   @                    @                    0       <   5                 2                    @     h                               7     0               @                                 @                                                         P                          2                            U                     G                                                         G     e      =                   	                      x     `                                                        d                             0	*H
01
0	`He0	*H
1a0]080 10UDebian Secure Boot CA2(oe:B&C0	`He0
	*H
  
#gnuK@P7}ή@4\W/ӯ_u(40ҖMK2nw,\ 2K
O<{xERP_<:cSOyPjj*9:8M2v}14@
'{FwLë{N=
xu{w'%~U:^q+j]n[lPXDTg\h/M%w\ʨ[-         ~Module signature appended~
 070701000A045F000041ED000000000000000000000006688C084800000000000000CA0000000200000000000000000000004300000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/atheros    070701000A0460000041ED000000000000000000000002688C084800000000000000CA0000000200000000000000000000004700000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/atheros/alx    070701000A0461000081A4000000000000000000000001682F6DA70001A4C3000000CA0000000200000000000000000000004E00000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/atheros/alx/alx.ko ELF          >                             @     @ 2 1          GNU F9)Xz*--        Linux                Linux   6.1.0-37-amd64            HvH%        HG1҉     HG  HG HGB   t7ATUHS  ~Lg1ۉ1L    9  []A\            x  ~fATIUS1I0  Ht9H        H      H      H    IǄ0      HA9$x  []A\        fD      AT   UHSHG   1HG  HG HB   tE  Le1ۅ~,   L    H}ރ        9  []A\    1    []A\    ff.     @     L	  9	  u)EtDL    DL        f         HeH%(   HD$11fD$9	  uTL	  t6HL$Lǉ    uD$HT$eH+%(   uH    HT$L    ϸ             ATLT  UHSHL    H	      H
  LHEH8
  HEH
  HE@H@  H8  HH  HP  HEHH(
  HH
  H0
  H
  H
  H
  H
  HE H
  HUpH0
  HUPH(
  HU`H
  HUhH
  HU0HP  HH  HX  Hp  HU(HP  H   HX  H   HH  H   H
  HUH 
  HE []A\    ff.     f    HG8tHH	          f         V   AWAVAUIATUSN	tz\  HL	     Mu   1HLL	      ]  H   sB   tAE	     LL    1[]A\A]A^A_        IEILHH	  IDIDLH)H)H뫃\  I=     j   H    HH            AAE	  ADADNADfAD=D      AUIATUSH   eH%(   HD$1H$    с      L  H  I9tW   It$(       H        	HH	M$$I9u$I	    I	  T$  A   с   A	  %Ɓ   Eƅt
   I	  A	  H  HD$eH+%(   uH[]A\A]        ff.     f    fD      HcSHRHƋSuH;Ht       H    [    HsE1       C    ff.     @     AUIATE1USp     K0  H   H  H   HC H   f{( 7  1HC H{Hƃ/S(9|HRHC H1H    S(H{1H    C.HC*    H<HH     H       IE9p  ;I0  H   L  M   IT$8HtvfA|$@ t\1IT$8H@HH; t7SHsE1   I|$    H;    H    C    HC    fA;l$@rAD$D    1fAD$B[]A\A]    1f.         UHSQp  ~01H0  HtH  Ht	Hx     H9p  H0  HtH  Ht	Hx8    H   Ht"H}H  E1(  [H   ]    []        AWAVAUIATUSH W|  9r1Hְ  H  {,Hs*OH$)f9r
S(L)A   I   rfz ~t
BE9   HHkHE     HE    A   `<`t^D[*HkfD\$LINt AF    f;C(b  HHfC*AfE AFfEAFE   I   A   Hfx   A   A+   uQA   	ЀELC.H<$H    HHH  H         L    H 1[]A\A]A^A_    LI   EepE+etL{IcHHT$HL$    HT$HL$Y  Hо   %  HE  H   H+5    HE1A   HHLHH5        H=  S*H{ HRHDbS*LC HRIHBHEfDe A   I   x   E1   HfS*AVE1A   HkUIcH{HHHcH8P<Hp0I    H   S*HK AHRHDzS*Hs HRHHBHEfD} A   I   PA9  K*Qf;S(U11S=     u/IoP    HuI/L    HH    H    l$E1f9k*Hs H{C(9C*AD9uAE~t@ )e  fA   {  A   D  UUA   I   I+   	ЉEA   I   T  	ЉEk1M   C*HS H@HL(C*f;C(u1fC*C.HAUpHHH  m              9O  H<$C0S*H	  fK*C,f9sIS()f9C.HHHH  H    H      ps(D)11 
  L    <I   vA   E1H1fDR
I   BB    f1  A   fDU   ?A   E1A   E1HL$
1ҹ   fDHHpHx    fAD$AEpUE H    H      D$       9   tH5        ATUSx  H  ~XE1H=      
      HH    H  H;@   HH        J0  ID9x  p     1H=    8   
  L0      H    -    I$  HKfh.fP0-    fP2  fP(HHH   HP    HA	$  	  9p  qH=    P   
  H0      H    H  H0  HKHP1fPH  fP@HHH   HP1          []A\    ff.     f    U   SH  H󋶨  H}      HE   H    u   []    H       []    ff.     @     USHH  HHuH}    H}     H}    H߾       H߉    H    H    H} []    ff.          AWAVAUIATIUSHeH%(   HD$1    Ņt)HD$eH+%(     H[]A\A]A^A_    M$   HL      LH    Ņ    HL       L    H    L    Ņ    L    L    A$                    HHL  `  1LLp  H	  H	  L	  @L    ǀ|      ǀ      L`f  I$H      H	  H    Hǃ       1LHǃ          ǃ    l  AE~  H	  BH<    	  1fz>  H	  	  LX  H    ǃ	    L   f	  -  ǂ   "      H	  ǀ$  ($  H    H   	  f	  H    	      ǃ     f	  HU    dH	  ǃ	  ǃ	  o       L    L    L    L      Ņ    H	     H	  LH   H        	  1H߹   H	  	  EfB    	  L<  EH	  f@  Hǃ	      ǃ	     Hǃ	      Hǃ	              L    H  HH   H  H0  H8  H  Hǃ(      H   HǃH      H8  H@      H    Ņ        HL    fwfA$F  sH    LH        M   L    L    L    L    z@i QzH E    L    Ņu)	  	  L    Ņ                AWAVAUATUHSHH?H    H  H   H9tH   H} HG8tH[]A\A]A^A_        Lm e       I  E1HH$eD%        A  t]DHHI  L   L    D   H    H      ǃ   LA    E;  rH<$1       H=        x  )H0  H    9x  H[]A\A]A^A_            H5    H0               AUE1ATUHSHcp      H  HH4BHGH  (  H       H   H    p     E1E1J0  
  H  C(H<@H    HC H    H   IcHHSH  HCC(A    ID9p  H0  
  H  C@H<@H    HC8H    H   IcK@HHSH  HC H   DHcЍHHs(H  HS0    1[]A\A]    E1k         U  SHuP         u	1[]      HK    H5    H                 H5    H             []    ff.         ATLf   ULSH1    HC   #  H߁!uHC   11L    [   ]A\    f         ATL|  USHL    HC      #  H߉u  u"HC1҉   L       []A\    H0  H    u  HK%    H    f         HFH   x
#  u1    $@     HFH   	f         AWAVAUATUSH(H0  L  AGBD`fD$ A;      DD1HRIW8fD$"1L,I}    t$$IfE9gDU  IW(D$ I>HT$$HD$A  p@    HH!  H   H%  H=  uH@   @H   InA  HH4$H   HL$HD$    H4$HL$   H     HG  H   H+    HH|$E1A   HHH5        H     I] H|$A  IEfDd$ AUHAD$A;      DAfD$"HRIW8L,I}  uAfE9gDf|$" tL$ fAOBIFf  D$"H([]A\A]A^A_    =     HH|$Ht
H    H       HuH       HH    H    H    fHH    f         AWAE1AVAUATUHSH0H  L  HD$M  AD$.I$AT$2A\$,HH	  HH  fD
fD9   H$A   E1H|$LEEt$IHs HRHHtDjpAH{Ճ_f;k(u1fA9E~uI܉H$Hl$fA\$,DD$ALEtHH   DLt$    D$ H$HL$DD$      9xH     H     H  E1H  LI	  HD$E  Hl$(E1HD|$'LL$D$UC       L    EDM@1҃f9DfEDEFf9DAfEFfA   D9$$$  DUFH]ILӋC	  %CMD9c  T  HE8HIH}E1   L,IuAU    AE    M} IE     s  @f?L    Hu L    fA   HD$H	     t*C   u!C
wA    A   H}LA    H|$ 
  yA)D9$$D|$'Hl$(D$fE^  E9tAEu<DH    HL$HAB   tf  Hy1    HD$H@ EH0D[]A\A]A^A_    LL$    Hl$(H5    D|$'I0      D$sLt$I|  H    A  HIF  A  H$  H    eH	  HR8AET$,EL$*AT$(AJD)fE9rAL
D)f9HD$    D$HDD$HL$H$    DD$HL$H$H|$ 
  D$ff.     f    SHx       1H0  H  Ht3E1E1fDQ,H  fDY*H  qI.	    HK1H  HtL1E1E1fqDH  fDABH  fDIFH  q HKh  H  H0HCP  H9x  QHCD    HC  HC@    HCx    HC`    HCd     HC4    H[    ff.     @     USHHkH    H    H;S0HC  HC   H!H5        H  []            AWAVLpAUATLgHUSHLHx    H    HD       L{L    SHHS  L    A9t7tDC@    H    H          H    AtG[L]A\A]A^A_    L,C@    H    tzH5    HS         LpA  t$1H<HI      A;  rދC~1H|H    9kHp    VLLnH    H    HpH     1H    H    L6    AUATUHHǀ	  SH	  B   u:H	  H   t
0[]A\A]0x
#   u[]A\A]    H1E1Q  ID9  ~J     L    I}      IE   H    tH    ff.     @     AUATLg(UHPSHLLX    H ƃLǃ    11L    HHHsL[]A\A]    D      AU1ATUSLoHL    Hމ    HCB   t5x  ~+1L0  uLH    L    9x  LHk    H      H      H<C@HCD    11H    HH
HH[]A\A]g        USH_xHH@8t1    H;H      H    HH        1[]        UH	  SHX  H    HH    1[]    f.         AW   I   AVAUATA   USH(    H9O؅DOE1El$DD        E  Ex  Ap  AǇt     @  LwÅtI    H([]A\A]A^A_    LÅx  MgL    L    I?AW0IG  IOAG&B  fHL$D$$ H  IG  T$$      H|$1M'    11MMH          IW11B   tBAp        ! tH  !   C H  !    HDIw  IOH   1IG  LAp  I?    At  I?    IG   LO@S  H5    I          aLLCI?    (IGL$$   M7I1Lt$    MM1ɉ1H        AI0  HD$E   LNLI    IE1             *  H      LAǇ     AǇx     Ip      L    IGB  H        E  ID    L$M1ɉ1H        HD$&  EE;x  HD$EeL0I  I  I  H<$HtsHH.HT$nH<$H        eI    IG1҉   H|$1M'    1MMǹ   H        2    HtJHH<$H    HT$    HT$H<$H        I  1҉HHH     ;  rqI1McA       L    EIO(  DI    L    M9u          ff.         ATUSH_xL      L    H{    HH@8u1L        []A\       HŅuH;    @     UH	  SHX  H    H1IH߉    []    fD      AU  ATUSH   f	  -9L      HC8u
[1]A\A]    LX  H	  LL	      H ƃ	  Lǃ	      11L    HHHsL    1[]A\A]    f.     @     P      ff.     @     fP      ff.     @     2   D    fD      AUATL	  UHH@SHHeH%(   HD$1H|$D$        }t\ME=  u
   d@Ƅt	@   @t   
tt{   #H	  l$@>< 0ua@	  LX  L    	  L    L    HD$eH+%(   u3H[]A\A]    t	   u˽   딽           ATLX  UHSHL    1	  t	  E	  LE	  E[]A\        AVAUATUHSH	  H@><	  EAAÀA`  @   DD    fF		  F	  t%	  @tDta A `  DELX  L    	  LC	  C    H{0D    H{@D    1[]A\A]A^    @DE뤐    t    H    HzH    HHH    H8  H)H)ց@  H    ff.     @     ATLT  UHLSH    H	      H 
  H{H 
  HHH  H  H)H)ށÐ  HL[]A\    ff.     @     AUATL	  USHHVN@tFtLX  L    	  @@t	  1uI@L    uF	  1t@L    @	  L    1H[]A\A]    !tɨtLD$    D$fD      1ATMUHfA    S8    E@tiHu	ʉH  
  @HE  x   Ƨ      t:HEH      uHE  fA$1[]A\    	   []A\           %      @E   @E   @E   @E   @ E    E @  E    ff.          U   HES8    EA	@tZHE	ʉH  D  @HE  x   Ƨ      t*HEH      u1[]    D	   붸[]    D      AULoDATAUSHL    11DŹ   H,uEĹ   11H[L]A\A]    D      AULoDATIUSHL    11DŹ   HuM   11Ht[L]A\A]    fD      AULoDATIUHLS    MH11/L    []A\A]             AULoDATAUHLS    EH11L    []A\A]            AVLwDAUIATILUS    ML   L    []A\A]A^    ff.          AVLwDAUAATILUS    EL   fL    []A\A]A^    ff.     f    ATIUSHGHH   HO  fȉFfu	E  d   XA     6  HCH   uHK  d   XA        HCH   uHCH   HK  fAD$fA$A$u	   HCH      d   XA        HCH   !uHs  d   XA     tQHCH   uHCH   HS  fAD$fA$A$u	t[1]A\    []A\    ff.     @     FHOȉ  HWf           AW1ҹAVAUATUSHH@HAAAA<HGDG≈   HG     HG   HGH   Hw%  HGH   Hw߉       C(HKC(  2   Ƨ      f4  HCH   AuHC     E  HCH    HsA   Ƨ  1    Ƨ      2  HCH   u    2  HCD   Ƨ  Au    Et&HCA   D   A  tHCD  HCH   HK  HCH   A
  %Hs     
  Hs  HCH   Hs  Hs  O     HCH   Hs  HCH   %HKAC  O     S(HC  HCH$   
   HS$  1[]A\A]A^A_    HCH   AƩ  2HC  %Hs  HK  []A\A]A^A_    ff.          ATUHSHeH%(   HD$11fD$HG  ZHGڀ  Ƨ        HE  P   Ƨ      uHﺝ  )   ZH]D  H   DH    A   b  H      H    HEH@   HM@  H    E1<   H      <H    L     HF,     H,  6   H1  H=   HHT$T$=   H  eH    A  '  H      H    H    E1#  H      vH    }t uHH    11HA      IH    HD$eH+%(   
  H[]A\    H         HLD$   H    Dd$H         HfA    EEH    H=   HT$T$=   Hf .H         HLD$   *H    Dd$H    H     fA @   EE0H        ff.          U   SHHH?eH%(   HD$1HT$1oHfD$    T$@      HCH   1HC  HCH   Hs  HCH   HK%  HCH    @   %     t
Hs   HHHHCH   %G @wH
pAHEHC @  Ƨ      HD$eH+%(   uYH[]    H;   fT$    HJHH
         DP    ff.     f    HwH  H     Hp  Hx   H(ʃ߀x<u ʉѾ       x8  D	ʃP(H@          G(@EEHWG(      f         HHD@HHGA  G 
@AwLpA@HEǉǁ  @@EǉƁ  @HQEƉ      @     AWAVAAUATLgDUHLS    E1   11H	L    HED  A@  AAAD@DEDDEDDED DED@DEEDAA  HE E1@>t%   AA    DDL    E11H   BLA    E  L    E11H	   LA    EU  L    111HA   LA    E%  L    A?   11   HL    A	E1HUD  [D]A\A]A^A_    DA   fA f
`EDDCEfA  fA trunL    E111HLA    EfL    A?   11   HL    0D,    #fA f
@DE!Aff.     f    AUATUSHeH%(   HD$H@HAA<a  HHoD{8Hp           HLD$   H    #   HDl$HT$S8D$?  tgduufA wfEugf,va2     HjH         HLD$   fH    Dl$A@E^fAtwfEuf(w 2     H	H         HLD$   H    Dl$A  H    HE        H    A   HD$eH+%(   N  H[]A\A]             HLD$   xH    Dl$H    H߹     A     EH    As{t i     H   HHT$AHD$eH+%(      T$  ʸ   HH[]A\A]{t C8dt@=  HT$   HHD$eH+%(   u/T$  P  HD$eH+%(   u         D      AUATUSHH@eH%(   HD$1AA@t[A1fD$HCH   tHCH   %   u71HT$eH+%(   u_H[]A\A]    wD$    ALkDL    11   LD$HL    f|$?uD9    f    AVAUATIUHoDSHL/HeH%(   HD$11fD$    11LLD$    H    t'HD$eH+%(      H[]A\A]A^    H    11LLD$   H    uD$uAL$8AT$<H    11LLD$   HA    Eu;T$    f% f= @t)  f= tf    
   
Dd       D      UHoDSHHHeH%(   HD$1    H11LD$   H    HD$eH+%(   u
H[]        @     HGH   HW%߉      ff.         USHHHwPHGȉ  GHwf  ?   HG  vHG   t  iS4  HC  HCH    Hs
     CHsf   N  HC  S$HC  SHC  CH{H    ~c(     Hs  H;    =     HC    U HC<  HCH$   %  =    vw-  H{
     HHK@><    HCH   C Hs	
|   dHC8  []    -   뇸  H;       *ff.          ҍ   HGHG     ff.          AT1UHoDSHHHeH%(   HD$11fT$fD$    11HLCp   HA    Et#1HT$eH+%(      H[]A\    H    11HLCr   HA    EuH    H߹      LD$   }HA    ExH    H߹      LD$   EHA    E@D$T$	ЉCL   &        HOH   HwHWx  HOH     HwH     H   H  HHH     HpH     HxH     HHH      HpH   $  HxH   (  HHH   ,  HpH   0  HxH   4  HHH   8  HpH   <  HxH   @  HHH   D  HpH   H  HxH  L  HHH  P  HpH  T  HxH   X  HHH(  \  HpH0  `  HxH8  d  HHH@  h  HpHH  l  HxHP  p  HHHX  t  HpH`  x  HxHh  |  HHHp    HpHx    HxH    HHH    HpH    HxH    HHH    HpH    HxH    HHH    HpH    HxH    HHH    HpH    HxH    HHH    HpH    HxH    H           H    H    H            AUL   H    ATUSHH  HLLeH      H    H    tH    L    !H߻       L    L    t   H    []A\A]    H;H        H        H    L    L    H	      H        H    L    H0  H    H        H    L        H    L    1   H|$
D$
fD$      D$
1H߹   HT$
D$
    ]  ƃ\  Hv5=     u,   HH$H    H            H$H0  HH    H    L    H    L    H    L        H    L        H    L    H   H        H    L    ^    AVAUATAH    UHH   SH_xL3L      L    IF8tCLLs    H    C@LCD    11L    H    H       At
H       L    []A\A]A^    H} H            H} H        H?H            H?H                ATH   H    USH_xL#H      H    ID$8tH    L    [H]A\    HpH            =  tId@ƄuI@uK
uLtStO1H    HpH            t   @t   Ʉt   뾄t   H    I?H    $    $    I?H        IE1             Å    A   A          I   H            H                                                                                                                                                                                                                                                                                                       alx_probe                       i                           i                           i                           i                           i                           i                            i                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           @      |alx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     rx_packets                      rx_bcast_packets                rx_mcast_packets                rx_pause_packets                rx_ctrl_packets                 rx_fcs_errors                   rx_length_errors                rx_bytes                        rx_runt_packets                 rx_fragments                    rx_64B_or_less_packets          rx_65B_to_127B_packets          rx_128B_to_255B_packets         rx_256B_to_511B_packets         rx_512B_to_1023B_packets        rx_1024B_to_1518B_packets       rx_1519B_to_mtu_packets         rx_oversize_packets             rx_rxf_ov_drop_packets          rx_rrd_ov_drop_packets          rx_align_errors                 rx_bcast_bytes                  rx_mcast_bytes                  rx_address_errors               tx_packets                      tx_bcast_packets                tx_mcast_packets                tx_pause_packets                tx_exc_defer_packets            tx_ctrl_packets                 tx_defer_packets                tx_bytes                        tx_64B_or_less_packets          tx_65B_to_127B_packets          tx_128B_to_255B_packets         tx_256B_to_511B_packets         tx_512B_to_1023B_packets        tx_1024B_to_1518B_packets       tx_1519B_to_mtu_packets         tx_single_collision             tx_multiple_collisions          tx_late_collision               tx_abort_collision              tx_underrun                     tx_trd_eop                      tx_length_errors                tx_trunc_packets                tx_bcast_bytes                  tx_mcast_bytes                  tx_update                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           alx pci error slot reset
 include/linux/dma-mapping.h DMA to 64-BIT addresses
 cannot map device registers
 unrecognized chip, aborting
 &alx->mtx MAC Reset failed, error = %d
 failed to identify PHY
 register netdevice failed
 pci error detected
 alert interrupt: 0x%x
 pci error resume
 Unknown speed NIC Up: %s
 Link Down
 %s-TxRx-%u %s-tx-%u %s-rx-%u %s-unused IRQ registration failed!
 10 Mbps Half 10 Mbps Full 100 Mbps Half 100 Mbps Full 1 Gbps Full        Failed to re-enable PCI device after reset
     field "hw->mac_addr" at drivers/net/ethernet/atheros/alx/main.c:611     memcpy: detected field-spanning write (size %zu) of single %s (size %zu)
       drivers/net/ethernet/atheros/alx/main.c %s %s: rejecting DMA map of vmalloc memory
     include/linux/dynamic_queue_limits.h    error allocating internal structures
   No usable DMA config, aborting
 pci_request_mem_regions failed
 Can't find power management capability, aborting
       net device private data init failed
    failed to configure PHY speed/duplex (err=%d)
  Invalid permanent address programmed, using random one
 field "hw->perm_addr" at drivers/net/ethernet/atheros/alx/main.c:1834   Qualcomm Atheros AR816x/AR817x Ethernet [%pM]
  Allocation of tx buffer failed!
        Allocation of rx buffer failed!
        fatal interrupt 0x%x, resetting
        Enabling MSI-X interrupts failed!
      drivers/net/ethernet/atheros/alx/ethtool.c      invalid PHY speed/duplex: 0x%x
                                 license=GPL description=Qualcomm Atheros(R) AR816x/AR817x PCI-E Ethernet Network Driver author=Qualcomm Corporation author=Johannes Berg <johannes@sipsolutions.net> alias=pci:v00001969d000010A0sv*sd*bc*sc*i* alias=pci:v00001969d000010A1sv*sd*bc*sc*i* alias=pci:v00001969d00001090sv*sd*bc*sc*i* alias=pci:v00001969d0000E0B1sv*sd*bc*sc*i* alias=pci:v00001969d0000E0A1sv*sd*bc*sc*i* alias=pci:v00001969d0000E091sv*sd*bc*sc*i* alias=pci:v00001969d00001091sv*sd*bc*sc*i* depends=mdio retpoline=Y intree=Y name=alx vermagic=6.1.0-37-amd64 SMP preempt mod_unload modversions                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         (    0  8  0  (                     8                       (  8  (                 8                                                       (                 (                                                     (    0  8  X  8  0  (                     X                                                                                                      (    0  8  P  8  0  (                     P                         (    0  8  @  8  0  (                     @  8  0  (                                                    (                 (                                                                                                                                      (    0  8  `  8  0  (                     `                         (    0  8  h  8  0  (                     h                                                               (    0  8  0  (                     8                       (                 (                 (        // x86 specific conversion optimizations -*- C++ -*-

// Copyright (C) 2020-2022 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H
#define _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H

#if __cplusplus >= 201703L

// work around PR85827
// 1-arg __convert_x86 {{{1
template <typename _To, typename _V, typename _Traits>
  _GLIBCXX_SIMD_INTRINSIC _To
  __convert_x86(_V __v)
  {
    static_assert(__is_vector_type_v<_V>);
    using _Tp = typename _Traits::value_type;
    constexpr size_t _Np = _Traits::_S_full_size;
    [[maybe_unused]] const auto __intrin = __to_intrin(__v);
    using _Up = typename _VectorTraits<_To>::value_type;
    constexpr size_t _M = _VectorTraits<_To>::_S_full_size;

    // [xyz]_to_[xyz] {{{2
    [[maybe_unused]] constexpr bool __x_to_x
      = sizeof(__v) <= 16 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __x_to_y
      = sizeof(__v) <= 16 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __x_to_z
      = sizeof(__v) <= 16 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __y_to_x
      = sizeof(__v) == 32 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __y_to_y
      = sizeof(__v) == 32 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __y_to_z
      = sizeof(__v) == 32 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __z_to_x
      = sizeof(__v) == 64 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __z_to_y
      = sizeof(__v) == 64 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __z_to_z
      = sizeof(__v) == 64 && sizeof(_To) == 64;

    // iX_to_iX {{{2
    [[maybe_unused]] constexpr bool __i_to_i
      = is_integral_v<_Up> && is_integral_v<_Tp>;
    [[maybe_unused]] constexpr bool __i8_to_i16
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i8_to_i32
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __i8_to_i64
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i16_to_i8
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i16_to_i32
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __i16_to_i64
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i32_to_i8
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i32_to_i16
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i32_to_i64
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i64_to_i8
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i64_to_i16
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i64_to_i32
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 4;

    // [fsu]X_to_[fsu]X {{{2
    // ibw = integral && byte or word, i.e. char and short with any signedness
    [[maybe_unused]] constexpr bool __s64_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s32_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s16_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s8_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u64_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u32_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u16_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u8_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s64_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __s32_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u64_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u32_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __f32_to_s64
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_s32
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_u64
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_u32
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f64_to_s64
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_s32
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_u64
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_u32
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __ibw_to_f32
      = is_integral_v<_Tp> && sizeof(_Tp) <= 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __ibw_to_f64
      = is_integral_v<_Tp> && sizeof(_Tp) <= 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __f32_to_ibw
      = is_integral_v<_Up> && sizeof(_Up) <= 2
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f64_to_ibw
      = is_integral_v<_Up> && sizeof(_Up) <= 2
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f32_to_f64
      = is_floating_point_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __f64_to_f32
      = is_floating_point_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;

    if constexpr (__i_to_i && __y_to_x && !__have_avx2) //{{{2
      return __convert_x86<_To>(__lo128(__v), __hi128(__v));
    else if constexpr (__i_to_i && __x_to_y && !__have_avx2) //{{{2
      return __concat(__convert_x86<__vector_type_t<_Up, _M / 2>>(__v),
		      __convert_x86<__vector_type_t<_Up, _M / 2>>(
			__extract_part<1, _Np / _M * 2>(__v)));
    else if constexpr (__i_to_i) //{{{2
      {
	static_assert(__x_to_x || __have_avx2,
		      "integral conversions with ymm registers require AVX2");
	static_assert(__have_avx512bw
			|| ((sizeof(_Tp) >= 4 || sizeof(__v) < 64)
			    && (sizeof(_Up) >= 4 || sizeof(_To) < 64)),
		      "8/16-bit integers in zmm registers require AVX512BW");
	static_assert((sizeof(__v) < 64 && sizeof(_To) < 64) || __have_avx512f,
		      "integral conversions with ymm registers require AVX2");
      }
    if constexpr (is_floating_point_v<_Tp> == is_floating_point_v<_Up> && //{{{2
		  sizeof(_Tp) == sizeof(_Up))
      {
	// conversion uses simple bit reinterpretation (or no conversion at all)
	if constexpr (_Np >= _M)
	  return __intrin_bitcast<_To>(__v);
	else
	  return __zero_extend(__vector_bitcast<_Up>(__v));
      }
    else if constexpr (_Np < _M && sizeof(_To) > 16) //{{{2
      // zero extend (eg. xmm -> ymm)
      return __zero_extend(
	__convert_x86<__vector_type_t<
	  _Up, (16 / sizeof(_Up) > _Np) ? 16 / sizeof(_Up) : _Np>>(__v));
    else if constexpr (_Np > _M && sizeof(__v) > 16) //{{{2
      // partial input (eg. ymm -> xmm)
      return __convert_x86<_To>(__extract_part<0, _Np / _M>(__v));
    else if constexpr (__i64_to_i32) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi64_epi32(__intrin));
	else if constexpr (__x_to_x)
	  return __auto_bitcast(
	    _mm_shuffle_ps(__vector_bitcast<float>(__v), __m128(), 8));
	else if constexpr (__y_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi64_epi32(__intrin));
	else if constexpr (__y_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi64_epi32(__auto_bitcast(__v))));
	else if constexpr (__y_to_x)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm256_permute4x64_epi64(_mm256_shuffle_epi32(__intrin, 8),
					     0 + 4 * 2)));
	else if constexpr (__z_to_y)
	  return __intrin_bitcast<_To>(_mm512_cvtepi64_epi32(__intrin));
      }
    else if constexpr (__i64_to_i16) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi64_epi16(__intrin));
	else if constexpr (__x_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi64_epi16(__auto_bitcast(__v))));
	else if constexpr (__x_to_x && __have_ssse3)
	  {
	    return __intrin_bitcast<_To>(
	      _mm_shuffle_epi8(__intrin,
			       _mm_setr_epi8(0, 1, 8, 9, -0x80, -0x80, -0x80,
					     -0x80, -0x80, -0x80, -0x80, -0x80,
					     -0x80, -0x80, -0x80, -0x80)));
	    // fallback without SSSE3
	  }
	else if constexpr (__y_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi64_epi16(__intrin));
	else if constexpr (__y_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi64_epi16(__auto_bitcast(__v))));
	else if constexpr (__y_to_x)
	  {
	    const auto __a = _mm256_shuffle_epi8(
	      __intrin,
	      _mm256_setr_epi8(0, 1, 8, 9, -0x80, -0x80, -0x80, -0x80, -0x80,
			       -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
			       -0x80, -0x80, -0x80, -0x80, 0, 1, 8, 9, -0x80,
			       -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
			       -0x80));
	    return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
	  }
	else if constexpr (__z_to_x)
	  return __intrin_bitcast<_To>(_mm512_cvtepi64_epi16(__intrin));
      }
    else if constexpr (__i64_to_i8) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi64_epi8(__intrin));
	else if constexpr (__x_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi64_epi8(__zero_extend(__intrin))));
	else if constexpr (__y_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi64_epi8(__intrin));
	else if constexpr (__y_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    _mm512_cvtepi64_epi8(__zero_extend(__intrin)));
	else if constexpr (__z_to_x)
	  return __intrin_bitcast<_To>(_mm512_cvtepi64_epi8(__intrin));
      }
    else if constexpr (__i32_to_i64) //{{{2
      {
	if constexpr (__have_sse4_1 && __x_to_x)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm_cvtepi32_epi64(__intrin)
					 : _mm_cvtepu32_epi64(__intrin));
	else if constexpr (__x_to_x)
	  {
	    return __intrin_bitcast<_To>(
	      _mm_unpacklo_epi32(__intrin, is_signed_v<_Tp>
					     ? _mm_srai_epi32(__intrin, 31)
					     : __m128i()));
	  }
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm256_cvtepi32_epi64(__intrin)
					 : _mm256_cvtepu32_epi64(__intrin));
	else if constexpr (__y_to_z)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm512_cvtepi32_epi64(__intrin)
					 : _mm512_cvtepu32_epi64(__intrin));
      }
    else if constexpr (__i32_to_i16) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi32_epi16(__intrin));
	else if constexpr (__x_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi32_epi16(__auto_bitcast(__v))));
	else if constexpr (__x_to_x && __have_ssse3)
	  return __intrin_bitcast<_To>(_mm_shuffle_epi8(
	    __intrin, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80,
				    -0x80, -0x80, -0x80, -0x80, -0x80, -0x80)));
	else if constexpr (__x_to_x)
	  {
	    auto __a = _mm_unpacklo_epi16(__intrin, __m128i()); // 0o.o 1o.o
	    auto __b = _mm_unpackhi_epi16(__intrin, __m128i()); // 2o.o 3o.o
	    auto __c = _mm_unpacklo_epi16(__a, __b);            // 02oo ..oo
	    auto __d = _mm_unpackhi_epi16(__a, __b);            // 13oo ..oo
	    return __intrin_bitcast<_To>(
	      _mm_unpacklo_epi16(__c, __d)); // 0123 oooo
	  }
	else if constexpr (__y_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi32_epi16(__intrin));
	else if constexpr (__y_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi32_epi16(__auto_bitcast(__v))));
	else if constexpr (__y_to_x)
	  {
	    auto __a = _mm256_shuffle_epi8(
	      __intrin,
	      _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, -0x80,
			       -0x80, -0x80, -0x80, -0x80, -0x80, 0, 1, 4, 5, 8,
			       9, 12, 13, -0x80, -0x80, -0x80, -0x80, -0x80,
			       -0x80, -0x80, -0x80));
	    return __intrin_bitcast<_To>(__lo128(
	      _mm256_permute4x64_epi64(__a,
				       0xf8))); // __a[0] __a[2] | __a[3] __a[3]
	  }
	else if constexpr (__z_to_y)
	  return __intrin_bitcast<_To>(_mm512_cvtepi32_epi16(__intrin));
      }
    else if constexpr (__i32_to_i8) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi32_epi8(__intrin));
	else if constexpr (__x_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi32_epi8(__zero_extend(__intrin))));
	else if constexpr (__x_to_x && __have_ssse3)
	  {
	    return __intrin_bitcast<_To>(
	      _mm_shuffle_epi8(__intrin,
			       _mm_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80,
					     -0x80, -0x80, -0x80, -0x80, -0x80,
					     -0x80, -0x80, -0x80, -0x80)));
	  }
	else if constexpr (__x_to_x)
	  {
	    const auto __a
	      = _mm_unpacklo_epi8(__intrin, __intrin); // 0... .... 1... ....
	    const auto __b
	      = _mm_unpackhi_epi8(__intrin, __intrin);    // 2... .... 3... ....
	    const auto __c = _mm_unpacklo_epi8(__a, __b); // 02.. .... .... ....
	    const auto __d = _mm_unpackhi_epi8(__a, __b); // 13.. .... .... ....
	    const auto __e = _mm_unpacklo_epi8(__c, __d); // 0123 .... .... ....
	    return __intrin_bitcast<_To>(__e & _mm_cvtsi32_si128(-1));
	  }
	else if constexpr (__y_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi32_epi8(__intrin));
	else if constexpr (__y_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    _mm512_cvtepi32_epi8(__zero_extend(__intrin)));
	else if constexpr (__z_to_x)
	  return __intrin_bitcast<_To>(_mm512_cvtepi32_epi8(__intrin));
      }
    else if constexpr (__i16_to_i64) //{{{2
      {
	if constexpr (__x_to_x && __have_sse4_1)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm_cvtepi16_epi64(__intrin)
					 : _mm_cvtepu16_epi64(__intrin));
	else if constexpr (__x_to_x && is_signed_v<_Tp>)
	  {
	    auto __x = _mm_srai_epi16(__intrin, 15);
	    auto __y = _mm_unpacklo_epi16(__intrin, __x);
	    __x = _mm_unpacklo_epi16(__x, __x);
	    return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__y, __x));
	  }
	else if constexpr (__x_to_x)
	  return __intrin_bitcast<_To>(
	    _mm_unpacklo_epi32(_mm_unpacklo_epi16(__intrin, __m128i()),
			       __m128i()));
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm256_cvtepi16_epi64(__intrin)
					 : _mm256_cvtepu16_epi64(__intrin));
	else if constexpr (__x_to_z)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm512_cvtepi16_epi64(__intrin)
					 : _mm512_cvtepu16_epi64(__intrin));
      }
    else if constexpr (__i16_to_i32) //{{{2
      {
	if constexpr (__x_to_x && __have_sse4_1)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm_cvtepi16_epi32(__intrin)
					 : _mm_cvtepu16_epi32(__intrin));
	else if constexpr (__x_to_x && is_signed_v<_Tp>)
	  return __intrin_bitcast<_To>(
	    _mm_srai_epi32(_mm_unpacklo_epi16(__intrin, __intrin), 16));
	else if constexpr (__x_to_x && is_unsigned_v<_Tp>)
	  return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__intrin, __m128i()));
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm256_cvtepi16_epi32(__intrin)
					 : _mm256_cvtepu16_epi32(__intrin));
	else if constexpr (__y_to_z)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm512_cvtepi16_epi32(__intrin)
					 : _mm512_cvtepu16_epi32(__intrin));
      }
    else if constexpr (__i16_to_i8) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512bw_vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi16_epi8(__intrin));
	else if constexpr (__x_to_x && __have_avx512bw)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepi16_epi8(__zero_extend(__intrin))));
	else if constexpr (__x_to_x && __have_ssse3)
	  return __intrin_bitcast<_To>(_mm_shuffle_epi8(
	    __intrin, _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -0x80, -0x80,
				    -0x80, -0x80, -0x80, -0x80, -0x80, -0x80)));
	else if constexpr (__x_to_x)
	  {
	    auto __a
	      = _mm_unpacklo_epi8(__intrin, __intrin); // 00.. 11.. 22.. 33..
	    auto __b
	      = _mm_unpackhi_epi8(__intrin, __intrin); // 44.. 55.. 66.. 77..
	    auto __c = _mm_unpacklo_epi8(__a, __b);    // 0404 .... 1515 ....
	    auto __d = _mm_unpackhi_epi8(__a, __b);    // 2626 .... 3737 ....
	    auto __e = _mm_unpacklo_epi8(__c, __d);    // 0246 0246 .... ....
	    auto __f = _mm_unpackhi_epi8(__c, __d);    // 1357 1357 .... ....
	    return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__e, __f));
	  }
	else if constexpr (__y_to_x && __have_avx512bw_vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi16_epi8(__intrin));
	else if constexpr (__y_to_x && __have_avx512bw)
	  return __intrin_bitcast<_To>(
	    __lo256(_mm512_cvtepi16_epi8(__zero_extend(__intrin))));
	else if constexpr (__y_to_x)
	  {
	    auto __a = _mm256_shuffle_epi8(
	      __intrin,
	      _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -0x80, -0x80, -0x80,
			       -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
			       -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 0, 2,
			       4, 6, 8, 10, 12, 14));
	    return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
	  }
	else if constexpr (__z_to_y && __have_avx512bw)
	  return __intrin_bitcast<_To>(_mm512_cvtepi16_epi8(__intrin));
	else if constexpr (__z_to_y)
	  __assert_unreachable<_Tp>();
      }
    else if constexpr (__i8_to_i64) //{{{2
      {
	if constexpr (__x_to_x && __have_sse4_1)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm_cvtepi8_epi64(__intrin)
					 : _mm_cvtepu8_epi64(__intrin));
	else if constexpr (__x_to_x && is_signed_v<_Tp>)
	  {
	    if constexpr (__have_ssse3)
	      {
		auto __dup = _mm_unpacklo_epi8(__intrin, __intrin);
		auto __epi16 = _mm_srai_epi16(__dup, 8);
		_mm_shuffle_epi8(__epi16,
				 _mm_setr_epi8(0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3,
					       3, 3, 3, 3, 3));
	      }
	    else
	      {
		auto __x = _mm_unpacklo_epi8(__intrin, __intrin);
		__x = _mm_unpacklo_epi16(__x, __x);
		return __intrin_bitcast<_To>(
		  _mm_unpacklo_epi32(_mm_srai_epi32(__x, 24),
				     _mm_srai_epi32(__x, 31)));
	      }
	  }
	else if constexpr (__x_to_x)
	  {
	    return __intrin_bitcast<_To>(_mm_unpacklo_epi32(
	      _mm_unpacklo_epi16(_mm_unpacklo_epi8(__intrin, __m128i()),
				 __m128i()),
	      __m128i()));
	  }
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm256_cvtepi8_epi64(__intrin)
					 : _mm256_cvtepu8_epi64(__intrin));
	else if constexpr (__x_to_z)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm512_cvtepi8_epi64(__intrin)
					 : _mm512_cvtepu8_epi64(__intrin));
      }
    else if constexpr (__i8_to_i32) //{{{2
      {
	if constexpr (__x_to_x && __have_sse4_1)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm_cvtepi8_epi32(__intrin)
					 : _mm_cvtepu8_epi32(__intrin));
	else if constexpr (__x_to_x && is_signed_v<_Tp>)
	  {
	    const auto __x = _mm_unpacklo_epi8(__intrin, __intrin);
	    return __intrin_bitcast<_To>(
	      _mm_srai_epi32(_mm_unpacklo_epi16(__x, __x), 24));
	  }
	else if constexpr (__x_to_x && is_unsigned_v<_Tp>)
	  return __intrin_bitcast<_To>(
	    _mm_unpacklo_epi16(_mm_unpacklo_epi8(__intrin, __m128i()),
			       __m128i()));
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm256_cvtepi8_epi32(__intrin)
					 : _mm256_cvtepu8_epi32(__intrin));
	else if constexpr (__x_to_z)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm512_cvtepi8_epi32(__intrin)
					 : _mm512_cvtepu8_epi32(__intrin));
      }
    else if constexpr (__i8_to_i16) //{{{2
      {
	if constexpr (__x_to_x && __have_sse4_1)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm_cvtepi8_epi16(__intrin)
					 : _mm_cvtepu8_epi16(__intrin));
	else if constexpr (__x_to_x && is_signed_v<_Tp>)
	  return __intrin_bitcast<_To>(
	    _mm_srai_epi16(_mm_unpacklo_epi8(__intrin, __intrin), 8));
	else if constexpr (__x_to_x && is_unsigned_v<_Tp>)
	  return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__intrin, __m128i()));
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm256_cvtepi8_epi16(__intrin)
					 : _mm256_cvtepu8_epi16(__intrin));
	else if constexpr (__y_to_z && __have_avx512bw)
	  return __intrin_bitcast<_To>(is_signed_v<_Tp>
					 ? _mm512_cvtepi8_epi16(__intrin)
					 : _mm512_cvtepu8_epi16(__intrin));
	else if constexpr (__y_to_z)
	  __assert_unreachable<_Tp>();
      }
    else if constexpr (__f32_to_s64) //{{{2
      {
	if constexpr (__have_avx512dq_vl && __x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvttps_epi64(__intrin));
	else if constexpr (__have_avx512dq_vl && __x_to_y)
	  return __intrin_bitcast<_To>(_mm256_cvttps_epi64(__intrin));
	else if constexpr (__have_avx512dq && __y_to_z)
	  return __intrin_bitcast<_To>(_mm512_cvttps_epi64(__intrin));
	// else use scalar fallback
      }
    else if constexpr (__f32_to_u64) //{{{2
      {
	if constexpr (__have_avx512dq_vl && __x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvttps_epu64(__intrin));
	else if constexpr (__have_avx512dq_vl && __x_to_y)
	  return __intrin_bitcast<_To>(_mm256_cvttps_epu64(__intrin));
	else if constexpr (__have_avx512dq && __y_to_z)
	  return __intrin_bitcast<_To>(_mm512_cvttps_epu64(__intrin));
	// else use scalar fallback
      }
    else if constexpr (__f32_to_s32) //{{{2
      {
	if constexpr (__x_to_x || __y_to_y || __z_to_z)
	  {
	    // go to fallback, it does the right thing
	  }
	else
	  __assert_unreachable<_Tp>();
      }
    else if constexpr (__f32_to_u32) //{{{2
      {
	if constexpr (__have_avx512vl && __x_to_x)
	  return __auto_bitcast(_mm_cvttps_epu32(__intrin));
	else if constexpr (__have_avx512f && __x_to_x)
	  return __auto_bitcast(
	    __lo128(_mm512_cvttps_epu32(__auto_bitcast(__v))));
	else if constexpr (__have_avx512vl && __y_to_y)
	  return __vector_bitcast<_Up>(_mm256_cvttps_epu32(__intrin));
	else if constexpr (__have_avx512f && __y_to_y)
	  return __vector_bitcast<_Up>(
	    __lo256(_mm512_cvttps_epu32(__auto_bitcast(__v))));
	else if constexpr (__x_to_x || __y_to_y || __z_to_z)
	  {
	    // go to fallback, it does the right thing. We can't use the
	    // _mm_floor_ps - 0x8000'0000 trick for f32->u32 because it would
	    // discard small input values (only 24 mantissa bits)
	  }
	else
	  __assert_unreachable<_Tp>();
      }
    else if constexpr (__f32_to_ibw) //{{{2
      return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _Np>>(__v));
    else if constexpr (__f64_to_s64) //{{{2
      {
	if constexpr (__have_avx512dq_vl && __x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvttpd_epi64(__intrin));
	else if constexpr (__have_avx512dq_vl && __y_to_y)
	  return __intrin_bitcast<_To>(_mm256_cvttpd_epi64(__intrin));
	else if constexpr (__have_avx512dq && __z_to_z)
	  return __intrin_bitcast<_To>(_mm512_cvttpd_epi64(__intrin));
	// else use scalar fallback
      }
    else if constexpr (__f64_to_u64) //{{{2
      {
	if constexpr (__have_avx512dq_vl && __x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvttpd_epu64(__intrin));
	else if constexpr (__have_avx512dq_vl && __y_to_y)
	  return __intrin_bitcast<_To>(_mm256_cvttpd_epu64(__intrin));
	else if constexpr (__have_avx512dq && __z_to_z)
	  return __intrin_bitcast<_To>(_mm512_cvttpd_epu64(__intrin));
	// else use scalar fallback
      }
    else if constexpr (__f64_to_s32) //{{{2
      {
	if constexpr (__x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvttpd_epi32(__intrin));
	else if constexpr (__y_to_x)
	  return __intrin_bitcast<_To>(_mm256_cvttpd_epi32(__intrin));
	else if constexpr (__z_to_y)
	  return __intrin_bitcast<_To>(_mm512_cvttpd_epi32(__intrin));
      }
    else if constexpr (__f64_to_u32) //{{{2
      {
	if constexpr (__have_avx512vl && __x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvttpd_epu32(__intrin));
	else if constexpr (__have_sse4_1 && __x_to_x)
	  return __vector_bitcast<_Up, _M>(
		   _mm_cvttpd_epi32(_mm_floor_pd(__intrin) - 0x8000'0000u))
		 ^ 0x8000'0000u;
	else if constexpr (__x_to_x)
	  {
	    // use scalar fallback: it's only 2 values to convert, can't get
	    // much better than scalar decomposition
	  }
	else if constexpr (__have_avx512vl && __y_to_x)
	  return __intrin_bitcast<_To>(_mm256_cvttpd_epu32(__intrin));
	else if constexpr (__y_to_x)
	  {
	    return __intrin_bitcast<_To>(
	      __vector_bitcast<_Up>(
		_mm256_cvttpd_epi32(_mm256_floor_pd(__intrin) - 0x8000'0000u))
	      ^ 0x8000'0000u);
	  }
	else if constexpr (__z_to_y)
	  return __intrin_bitcast<_To>(_mm512_cvttpd_epu32(__intrin));
      }
    else if constexpr (__f64_to_ibw) //{{{2
      {
	return __convert_x86<_To>(
	  __convert_x86<__vector_type_t<int, (_Np < 4 ? 4 : _Np)>>(__v));
      }
    else if constexpr (__s64_to_f32) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi64_ps(__intrin));
	else if constexpr (__y_to_x && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi64_ps(__intrin));
	else if constexpr (__z_to_y && __have_avx512dq)
	  return __intrin_bitcast<_To>(_mm512_cvtepi64_ps(__intrin));
	else if constexpr (__z_to_y)
	  return __intrin_bitcast<_To>(
	    _mm512_cvtpd_ps(__convert_x86<__vector_type_t<double, 8>>(__v)));
      }
    else if constexpr (__u64_to_f32) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm_cvtepu64_ps(__intrin));
	else if constexpr (__y_to_x && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepu64_ps(__intrin));
	else if constexpr (__z_to_y && __have_avx512dq)
	  return __intrin_bitcast<_To>(_mm512_cvtepu64_ps(__intrin));
	else if constexpr (__z_to_y)
	  {
	    return __intrin_bitcast<_To>(
	      __lo256(_mm512_cvtepu32_ps(__auto_bitcast(
		_mm512_cvtepi64_epi32(_mm512_srai_epi64(__intrin, 32)))))
		* 0x100000000LL
	      + __lo256(_mm512_cvtepu32_ps(
		__auto_bitcast(_mm512_cvtepi64_epi32(__intrin)))));
	  }
      }
    else if constexpr (__s32_to_f32) //{{{2
      {
	// use fallback (builtin conversion)
      }
    else if constexpr (__u32_to_f32) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512vl)
	  {
	    // use fallback
	  }
	else if constexpr (__x_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepu32_ps(__auto_bitcast(__v))));
	else if constexpr (__x_to_x && (__have_fma || __have_fma4))
	  // work around PR85819
	  return __auto_bitcast(0x10000
				  * _mm_cvtepi32_ps(__to_intrin(__v >> 16))
				+ _mm_cvtepi32_ps(__to_intrin(__v & 0xffff)));
	else if constexpr (__y_to_y && __have_avx512vl)
	  {
	    // use fallback
	  }
	else if constexpr (__y_to_y && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo256(_mm512_cvtepu32_ps(__auto_bitcast(__v))));
	else if constexpr (__y_to_y)
	  // work around PR85819
	  return 0x10000 * _mm256_cvtepi32_ps(__to_intrin(__v >> 16))
		 + _mm256_cvtepi32_ps(__to_intrin(__v & 0xffff));
	// else use fallback (builtin conversion)
      }
    else if constexpr (__ibw_to_f32) //{{{2
      {
	if constexpr (_M <= 4 || __have_avx2)
	  return __convert_x86<_To>(
	    __convert_x86<__vector_type_t<int, _M>>(__v));
	else
	  {
	    static_assert(__x_to_y);
	    __m128i __a, __b;
	    if constexpr (__have_sse4_1)
	      {
		__a = sizeof(_Tp) == 2
			? (is_signed_v<_Tp> ? _mm_cvtepi16_epi32(__intrin)
					    : _mm_cvtepu16_epi32(__intrin))
			: (is_signed_v<_Tp> ? _mm_cvtepi8_epi32(__intrin)
					    : _mm_cvtepu8_epi32(__intrin));
		const auto __w
		  = _mm_shuffle_epi32(__intrin, sizeof(_Tp) == 2 ? 0xee : 0xe9);
		__b = sizeof(_Tp) == 2
			? (is_signed_v<_Tp> ? _mm_cvtepi16_epi32(__w)
					    : _mm_cvtepu16_epi32(__w))
			: (is_signed_v<_Tp> ? _mm_cvtepi8_epi32(__w)
					    : _mm_cvtepu8_epi32(__w));
	      }
	    else
	      {
		__m128i __tmp;
		if constexpr (sizeof(_Tp) == 1)
		  {
		    __tmp = is_signed_v<_Tp>
			      ? _mm_srai_epi16(_mm_unpacklo_epi8(__intrin,
								 __intrin),
					       8)
			      : _mm_unpacklo_epi8(__intrin, __m128i());
		  }
		else
		  {
		    static_assert(sizeof(_Tp) == 2);
		    __tmp = __intrin;
		  }
		__a = is_signed_v<_Tp>
			? _mm_srai_epi32(_mm_unpacklo_epi16(__tmp, __tmp), 16)
			: _mm_unpacklo_epi16(__tmp, __m128i());
		__b = is_signed_v<_Tp>
			? _mm_srai_epi32(_mm_unpackhi_epi16(__tmp, __tmp), 16)
			: _mm_unpackhi_epi16(__tmp, __m128i());
	      }
	    return __convert_x86<_To>(__vector_bitcast<int>(__a),
				      __vector_bitcast<int>(__b));
	  }
      }
    else if constexpr (__s64_to_f64) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm_cvtepi64_pd(__intrin));
	else if constexpr (__y_to_y && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepi64_pd(__intrin));
	else if constexpr (__z_to_z && __have_avx512dq)
	  return __intrin_bitcast<_To>(_mm512_cvtepi64_pd(__intrin));
	else if constexpr (__z_to_z)
	  {
	    return __intrin_bitcast<_To>(
	      _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(__to_intrin(__v >> 32)))
		* 0x100000000LL
	      + _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__intrin)));
	  }
      }
    else if constexpr (__u64_to_f64) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm_cvtepu64_pd(__intrin));
	else if constexpr (__y_to_y && __have_avx512dq_vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepu64_pd(__intrin));
	else if constexpr (__z_to_z && __have_avx512dq)
	  return __intrin_bitcast<_To>(_mm512_cvtepu64_pd(__intrin));
	else if constexpr (__z_to_z)
	  {
	    return __intrin_bitcast<_To>(
	      _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__to_intrin(__v >> 32)))
		* 0x100000000LL
	      + _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__intrin)));
	  }
      }
    else if constexpr (__s32_to_f64) //{{{2
      {
	if constexpr (__x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvtepi32_pd(__intrin));
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(_mm256_cvtepi32_pd(__intrin));
	else if constexpr (__y_to_z)
	  return __intrin_bitcast<_To>(_mm512_cvtepi32_pd(__intrin));
      }
    else if constexpr (__u32_to_f64) //{{{2
      {
	if constexpr (__x_to_x && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm_cvtepu32_pd(__intrin));
	else if constexpr (__x_to_x && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo128(_mm512_cvtepu32_pd(__auto_bitcast(__v))));
	else if constexpr (__x_to_x)
	  return __intrin_bitcast<_To>(
	    _mm_cvtepi32_pd(__to_intrin(__v ^ 0x8000'0000u)) + 0x8000'0000u);
	else if constexpr (__x_to_y && __have_avx512vl)
	  return __intrin_bitcast<_To>(_mm256_cvtepu32_pd(__intrin));
	else if constexpr (__x_to_y && __have_avx512f)
	  return __intrin_bitcast<_To>(
	    __lo256(_mm512_cvtepu32_pd(__auto_bitcast(__v))));
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(
	    _mm256_cvtepi32_pd(__to_intrin(__v ^ 0x8000'0000u)) + 0x8000'0000u);
	else if constexpr (__y_to_z)
	  return __intrin_bitcast<_To>(_mm512_cvtepu32_pd(__intrin));
      }
    else if constexpr (__ibw_to_f64) //{{{2
      {
	return __convert_x86<_To>(
	  __convert_x86<__vector_type_t<int, std::max(size_t(4), _M)>>(__v));
      }
    else if constexpr (__f32_to_f64) //{{{2
      {
	if constexpr (__x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvtps_pd(__intrin));
	else if constexpr (__x_to_y)
	  return __intrin_bitcast<_To>(_mm256_cvtps_pd(__intrin));
	else if constexpr (__y_to_z)
	  return __intrin_bitcast<_To>(_mm512_cvtps_pd(__intrin));
      }
    else if constexpr (__f64_to_f32) //{{{2
      {
	if constexpr (__x_to_x)
	  return __intrin_bitcast<_To>(_mm_cvtpd_ps(__intrin));
	else if constexpr (__y_to_x)
	  return __intrin_bitcast<_To>(_mm256_cvtpd_ps(__intrin));
	else if constexpr (__z_to_y)
	  return __intrin_bitcast<_To>(_mm512_cvtpd_ps(__intrin));
      }
    else //{{{2
      __assert_unreachable<_Tp>();

    // fallback:{{{2
    return __vector_convert<_To>(__v, make_index_sequence<std::min(_M, _Np)>());
    //}}}
  }

// }}}
// 2-arg __convert_x86 {{{1
template <typename _To, typename _V, typename _Traits>
  _GLIBCXX_SIMD_INTRINSIC _To
  __convert_x86(_V __v0, _V __v1)
  {
    static_assert(__is_vector_type_v<_V>);
    using _Tp = typename _Traits::value_type;
    constexpr size_t _Np = _Traits::_S_full_size;
    [[maybe_unused]] const auto __i0 = __to_intrin(__v0);
    [[maybe_unused]] const auto __i1 = __to_intrin(__v1);
    using _Up = typename _VectorTraits<_To>::value_type;
    constexpr size_t _M = _VectorTraits<_To>::_S_full_size;

    static_assert(2 * _Np <= _M,
		  "__v1 would be discarded; use the one-argument "
		  "__convert_x86 overload instead");

    // [xyz]_to_[xyz] {{{2
    [[maybe_unused]] constexpr bool __x_to_x
      = sizeof(__v0) <= 16 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __x_to_y
      = sizeof(__v0) <= 16 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __x_to_z
      = sizeof(__v0) <= 16 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __y_to_x
      = sizeof(__v0) == 32 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __y_to_y
      = sizeof(__v0) == 32 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __y_to_z
      = sizeof(__v0) == 32 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __z_to_x
      = sizeof(__v0) == 64 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __z_to_y
      = sizeof(__v0) == 64 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __z_to_z
      = sizeof(__v0) == 64 && sizeof(_To) == 64;

    // iX_to_iX {{{2
    [[maybe_unused]] constexpr bool __i_to_i
      = is_integral_v<_Up> && is_integral_v<_Tp>;
    [[maybe_unused]] constexpr bool __i8_to_i16
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i8_to_i32
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __i8_to_i64
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i16_to_i8
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i16_to_i32
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __i16_to_i64
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i32_to_i8
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i32_to_i16
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i32_to_i64
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i64_to_i8
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i64_to_i16
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i64_to_i32
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 4;

    // [fsu]X_to_[fsu]X {{{2
    // ibw = integral && byte or word, i.e. char and short with any signedness
    [[maybe_unused]] constexpr bool __i64_to_f32
      = is_integral_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s32_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s16_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s8_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u32_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u16_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u8_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s64_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __s32_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __s16_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __s8_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u64_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u32_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u16_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u8_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __f32_to_s64
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_s32
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_u64
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_u32
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f64_to_s64
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_s32
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_u64
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_u32
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f32_to_ibw
      = is_integral_v<_Up> && sizeof(_Up) <= 2
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f64_to_ibw
      = is_integral_v<_Up> && sizeof(_Up) <= 2
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f32_to_f64
      = is_floating_point_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __f64_to_f32
      = is_floating_point_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;

    if constexpr (__i_to_i && __y_to_x && !__have_avx2) //{{{2
      // <double, 4>, <double, 4> => <short, 8>
      return __convert_x86<_To>(__lo128(__v0), __hi128(__v0), __lo128(__v1),
				__hi128(__v1));
    else if constexpr (__i_to_i) // assert ISA {{{2
      {
	static_assert(__x_to_x || __have_avx2,
		      "integral conversions with ymm registers require AVX2");
	static_assert(__have_avx512bw
			|| ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64)
			    && (sizeof(_Up) >= 4 || sizeof(_To) < 64)),
		      "8/16-bit integers in zmm registers require AVX512BW");
	static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f,
		      "integral conversions with ymm registers require AVX2");
      }
    // concat => use 1-arg __convert_x86 {{{2
    if constexpr (sizeof(__v0) < 16 || (sizeof(__v0) == 16 && __have_avx2)
		  || (sizeof(__v0) == 16 && __have_avx
		      && is_floating_point_v<_Tp>)
		  || (sizeof(__v0) == 32 && __have_avx512f
		      && (sizeof(_Tp) >= 4 || __have_avx512bw)))
      {
	// The ISA can handle wider input registers, so concat and use one-arg
	// implementation. This reduces code duplication considerably.
	return __convert_x86<_To>(__concat(__v0, __v1));
      }
    else //{{{2
      {
	// conversion using bit reinterpretation (or no conversion at all)
	// should all go through the concat branch above:
	static_assert(
	  !(is_floating_point_v<
	      _Tp> == is_floating_point_v<_Up> && sizeof(_Tp) == sizeof(_Up)));
	// handle all zero extension{{{2
	if constexpr (2 * _Np < _M && sizeof(_To) > 16)
	  {
	    constexpr size_t Min = 16 / sizeof(_Up);
	    return __zero_extend(
	      __convert_x86<
		__vector_type_t<_Up, (Min > 2 * _Np) ? Min : 2 * _Np>>(__v0,
								       __v1));
	  }
	else if constexpr (__i64_to_i32) //{{{2
	  {
	    if constexpr (__x_to_x)
	      return __auto_bitcast(_mm_shuffle_ps(__auto_bitcast(__v0),
						   __auto_bitcast(__v1), 0x88));
	    else if constexpr (__y_to_y)
	      {
		// AVX512F is not available (would concat otherwise)
		return __auto_bitcast(
		  __xzyw(_mm256_shuffle_ps(__auto_bitcast(__v0),
					   __auto_bitcast(__v1), 0x88)));
		// alternative:
		// const auto v0_abxxcdxx = _mm256_shuffle_epi32(__v0, 8);
		// const auto v1_efxxghxx = _mm256_shuffle_epi32(__v1, 8);
		// const auto v_abefcdgh = _mm256_unpacklo_epi64(v0_abxxcdxx,
		// v1_efxxghxx); return _mm256_permute4x64_epi64(v_abefcdgh,
		// 0x01 * 0 + 0x04 * 2 + 0x10 * 1 + 0x40 * 3);  // abcdefgh
	      }
	    else if constexpr (__z_to_z)
	      return __intrin_bitcast<_To>(
		__concat(_mm512_cvtepi64_epi32(__i0),
			 _mm512_cvtepi64_epi32(__i1)));
	  }
	else if constexpr (__i64_to_i16) //{{{2
	  {
	    if constexpr (__x_to_x)
	      {
		// AVX2 is not available (would concat otherwise)
		if constexpr (__have_sse4_1)
		  {
		    return __intrin_bitcast<_To>(_mm_shuffle_epi8(
		      _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 4), 0x44),
		      _mm_setr_epi8(0, 1, 8, 9, 4, 5, 12, 13, -0x80, -0x80,
				    -0x80, -0x80, -0x80, -0x80, -0x80, -0x80)));
		  }
		else
		  {
		    return __vector_type_t<_Up, _M>{_Up(__v0[0]), _Up(__v0[1]),
						    _Up(__v1[0]), _Up(__v1[1])};
		  }
	      }
	    else if constexpr (__y_to_x)
	      {
		auto __a
		  = _mm256_unpacklo_epi16(__i0, __i1); // 04.. .... 26.. ....
		auto __b
		  = _mm256_unpackhi_epi16(__i0, __i1); // 15.. .... 37.. ....
		auto __c
		  = _mm256_unpacklo_epi16(__a, __b); // 0145 .... 2367 ....
		return __intrin_bitcast<_To>(
		  _mm_unpacklo_epi32(__lo128(__c), __hi128(__c))); // 0123 4567
	      }
	    else if constexpr (__z_to_y)
	      return __intrin_bitcast<_To>(
		__concat(_mm512_cvtepi64_epi16(__i0),
			 _mm512_cvtepi64_epi16(__i1)));
	  }
	else if constexpr (__i64_to_i8) //{{{2
	  {
	    if constexpr (__x_to_x && __have_sse4_1)
	      {
		return __intrin_bitcast<_To>(_mm_shuffle_epi8(
		  _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 4), 0x44),
		  _mm_setr_epi8(0, 8, 4, 12, -0x80, -0x80, -0x80, -0x80, -0x80,
				-0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
				-0x80)));
	      }
	    else if constexpr (__x_to_x && __have_ssse3)
	      {
		return __intrin_bitcast<_To>(_mm_unpacklo_epi16(
		  _mm_shuffle_epi8(
		    __i0, _mm_setr_epi8(0, 8, -0x80, -0x80, -0x80, -0x80, -0x80,
					-0x80, -0x80, -0x80, -0x80, -0x80,
					-0x80, -0x80, -0x80, -0x80)),
		  _mm_shuffle_epi8(
		    __i1, _mm_setr_epi8(0, 8, -0x80, -0x80, -0x80, -0x80, -0x80,
					-0x80, -0x80, -0x80, -0x80, -0x80,
					-0x80, -0x80, -0x80, -0x80))));
	      }
	    else if constexpr (__x_to_x)
	      {
		return __vector_type_t<_Up, _M>{_Up(__v0[0]), _Up(__v0[1]),
						_Up(__v1[0]), _Up(__v1[1])};
	      }
	    else if constexpr (__y_to_x)
	      {
		const auto __a = _mm256_shuffle_epi8(
		  _mm256_blend_epi32(__i0, _mm256_slli_epi64(__i1, 32), 0xAA),
		  _mm256_setr_epi8(0, 8, -0x80, -0x80, 4, 12, -0x80, -0x80,
				   -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
				   -0x80, -0x80, -0x80, -0x80, 0, 8, -0x80,
				   -0x80, 4, 12, -0x80, -0x80, -0x80, -0x80,
				   -0x80, -0x80, -0x80, -0x80));
		return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
	      } // __z_to_x uses concat fallback
	  }
	else if constexpr (__i32_to_i16) //{{{2
	  {
	    if constexpr (__x_to_x)
	      {
		// AVX2 is not available (would concat otherwise)
		if constexpr (__have_sse4_1)
		  {
		    return __intrin_bitcast<_To>(_mm_shuffle_epi8(
		      _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 2), 0xaa),
		      _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10,
				    11, 14, 15)));
		  }
		else if constexpr (__have_ssse3)
		  {
		    return __intrin_bitcast<_To>(
		      _mm_hadd_epi16(__to_intrin(__v0 << 16),
				     __to_intrin(__v1 << 16)));
		    /*
		    return _mm_unpacklo_epi64(
			_mm_shuffle_epi8(__i0, _mm_setr_epi8(0, 1, 4, 5, 8, 9,
		    12, 13, 8, 9, 12, 13, 12, 13, 14, 15)),
		    _mm_shuffle_epi8(__i1, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12,
		    13, 8, 9, 12, 13, 12, 13, 14, 15)));
							   */
		  }
		else
		  {
		    auto __a = _mm_unpacklo_epi16(__i0, __i1); // 04.. 15..
		    auto __b = _mm_unpackhi_epi16(__i0, __i1); // 26.. 37..
		    auto __c = _mm_unpacklo_epi16(__a, __b);   // 0246 ....
		    auto __d = _mm_unpackhi_epi16(__a, __b);   // 1357 ....
		    return __intrin_bitcast<_To>(
		      _mm_unpacklo_epi16(__c, __d)); // 0123 4567
		  }
	      }
	    else if constexpr (__y_to_y)
	      {
		const auto __shuf
		  = _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80,
				     -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
				     0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80,
				     -0x80, -0x80, -0x80, -0x80, -0x80, -0x80);
		auto __a = _mm256_shuffle_epi8(__i0, __shuf);
		auto __b = _mm256_shuffle_epi8(__i1, __shuf);
		return __intrin_bitcast<_To>(
		  __xzyw(_mm256_unpacklo_epi64(__a, __b)));
	      } // __z_to_z uses concat fallback
	  }
	else if constexpr (__i32_to_i8) //{{{2
	  {
	    if constexpr (__x_to_x && __have_ssse3)
	      {
		const auto shufmask
		  = _mm_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80,
				  -0x80, -0x80, -0x80, -0x80, -0x80, -0x80,
				  -0x80, -0x80);
		return __intrin_bitcast<_To>(
		  _mm_unpacklo_epi32(_mm_shuffle_epi8(__i0, shufmask),
				     _mm_shuffle_epi8(__i1, shufmask)));
	      }
	    else if constexpr (__x_to_x)
	      {
		auto __a = _mm_unpacklo_epi8(__i0, __i1); // 04.. .... 15.. ....
		auto __b = _mm_unpackhi_epi8(__i0, __i1); // 26.. .... 37.. ....
		auto __c = _mm_unpacklo_epi8(__a, __b);   // 0246 .... .... ....
		auto __d = _mm_unpackhi_epi8(__a, __b);   // 1357 .... .... ....
		auto __e = _mm_unpacklo_epi8(__c, __d);   // 0123 4567 .... ....
		return __intrin_bitcast<_To>(__e & __m128i{-1, 0});
	      }
	    else if constexpr (__y_to_x)
	      {
		const auto __a = _mm256_shuffle_epi8(
		  _mm256_blend_epi16(__i0, _mm256_slli_epi32(__i1, 16), 0xAA),
		  _mm256_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80, 2,
				   6, 10, 14, -0x80, -0x80, -0x80, -0x80, -0x80,
				   -0x80, -0x80, -0x80, 0, 4, 8, 12, -0x80,
				   -0x80, -0x80, -0x80, 2, 6, 10, 14));
		return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a));
	      } // __z_to_y uses concat fallback
	  }
	else if constexpr (__i16_to_i8) //{{{2
	  {
	    if constexpr (__x_to_x && __have_ssse3)
	      {
		const auto __shuf = reinterpret_cast<__m128i>(
		  __vector_type_t<_UChar, 16>{0, 2, 4, 6, 8, 10, 12, 14, 0x80,
					      0x80, 0x80, 0x80, 0x80, 0x80,
					      0x80, 0x80});
		return __intrin_bitcast<_To>(
		  _mm_unpacklo_epi64(_mm_shuffle_epi8(__i0, __shuf),
				     _mm_shuffle_epi8(__i1, __shuf)));
	      }
	    else if constexpr (__x_to_x)
	      {
		auto __a = _mm_unpacklo_epi8(__i0, __i1); // 08.. 19.. 2A.. 3B..
		auto __b = _mm_unpackhi_epi8(__i0, __i1); // 4C.. 5D.. 6E.. 7F..
		auto __c = _mm_unpacklo_epi8(__a, __b);   // 048C .... 159D ....
		auto __d = _mm_unpackhi_epi8(__a, __b);   // 26AE .... 37BF ....
		auto __e = _mm_unpacklo_epi8(__c, __d);   // 0246 8ACE .... ....
		auto __f = _mm_unpackhi_epi8(__c, __d);   // 1357 9BDF .... ....
		return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__e, __f));
	      }
	    else if constexpr (__y_to_y)
	      {
		return __intrin_bitcast<_To>(__xzyw(_mm256_shuffle_epi8(
		  (__to_intrin(__v0) & _mm256_set1_epi32(0x00ff00ff))
		    | _mm256_slli_epi16(__i1, 8),
		  _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11,
				   13, 15, 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5,
				   7, 9, 11, 13, 15))));
	      } // __z_to_z uses concat fallback
	  }
	else if constexpr (__i64_to_f32) //{{{2
	  {
	    if constexpr (__x_to_x)
	      return __make_wrapper<float>(__v0[0], __v0[1], __v1[0], __v1[1]);
	    else if constexpr (__y_to_y)
	      {
		static_assert(__y_to_y && __have_avx2);
		const auto __a = _mm256_unpacklo_epi32(__i0, __i1); // aeAE cgCG
		const auto __b = _mm256_unpackhi_epi32(__i0, __i1); // bfBF dhDH
		const auto __lo32
		  = _mm256_unpacklo_epi32(__a, __b); // abef cdgh
		const auto __hi32 = __vector_bitcast<
		  conditional_t<is_signed_v<_Tp>, int, _UInt>>(
		  _mm256_unpackhi_epi32(__a, __b)); // ABEF CDGH
		const auto __hi
		  = 0x100000000LL
		    * __convert_x86<__vector_type_t<float, 8>>(__hi32);
		const auto __mid
		  = 0x10000 * _mm256_cvtepi32_ps(_mm256_srli_epi32(__lo32, 16));
		const auto __lo
		  = _mm256_cvtepi32_ps(_mm256_set1_epi32(0x0000ffffu) & __lo32);
		return __xzyw((__hi + __mid) + __lo);
	      }
	    else if constexpr (__z_to_z && __have_avx512dq)
	      {
		return is_signed_v<_Tp> ? __concat(_mm512_cvtepi64_ps(__i0),
						   _mm512_cvtepi64_ps(__i1))
					: __concat(_mm512_cvtepu64_ps(__i0),
						   _mm512_cvtepu64_ps(__i1));
	      }
	    else if constexpr (__z_to_z && is_signed_v<_Tp>)
	      {
		const __m512 __hi32 = _mm512_cvtepi32_ps(
		  __concat(_mm512_cvtepi64_epi32(__to_intrin(__v0 >> 32)),
			   _mm512_cvtepi64_epi32(__to_intrin(__v1 >> 32))));
		const __m512i __lo32 = __concat(_mm512_cvtepi64_epi32(__i0),
						_mm512_cvtepi64_epi32(__i1));
		// split low 32-bits, because if __hi32 is a small negative
		// number, the 24-bit mantissa may lose important information if
		// any of the high 8 bits of __lo32 is set, leading to
		// catastrophic cancelation in the FMA
		const __m512 __hi16
		  = _mm512_cvtepu32_ps(_mm512_set1_epi32(0xffff0000u) & __lo32);
		const __m512 __lo16
		  = _mm512_cvtepi32_ps(_mm512_set1_epi32(0x0000ffffu) & __lo32);
		return (__hi32 * 0x100000000LL + __hi16) + __lo16;
	      }
	    else if constexpr (__z_to_z && is_unsigned_v<_Tp>)
	      {
		return __intrin_bitcast<_To>(
		  _mm512_cvtepu32_ps(__concat(
		    _mm512_cvtepi64_epi32(_mm512_srai_epi64(__i0, 32)),
		    _mm512_cvtepi64_epi32(_mm512_srai_epi64(__i1, 32))))
		    * 0x100000000LL
		  + _mm512_cvtepu32_ps(__concat(_mm512_cvtepi64_epi32(__i0),
						_mm512_cvtepi64_epi32(__i1))));
	      }
	  }
	else if constexpr (__f64_to_s32) //{{{2
	  {
	    // use concat fallback
	  }
	else if constexpr (__f64_to_u32) //{{{2
	  {
	    if constexpr (__x_to_x && __have_sse4_1)
	      {
		return __vector_bitcast<_Up, _M>(_mm_unpacklo_epi64(
			 _mm_cvttpd_epi32(_mm_floor_pd(__i0) - 0x8000'0000u),
			 _mm_cvttpd_epi32(_mm_floor_pd(__i1) - 0x8000'0000u)))
		       ^ 0x8000'0000u;
		// without SSE4.1 just use the scalar fallback, it's only four
		// values
	      }
	    else if constexpr (__y_to_y)
	      {
		return __vector_bitcast<_Up>(
			 __concat(_mm256_cvttpd_epi32(_mm256_floor_pd(__i0)
						      - 0x8000'0000u),
				  _mm256_cvttpd_epi32(_mm256_floor_pd(__i1)
						      - 0x8000'0000u)))
		       ^ 0x8000'0000u;
	      } // __z_to_z uses fallback
	  }
	else if constexpr (__f64_to_ibw) //{{{2
	  {
	    // one-arg __f64_to_ibw goes via _SimdWrapper<int, ?>. The fallback
	    // would go via two independet conversions to _SimdWrapper<_To> and
	    // subsequent interleaving. This is better, because f64->__i32
	    // allows to combine __v0 and __v1 into one register: if constexpr
	    // (__z_to_x || __y_to_x) {
	    return __convert_x86<_To>(
	      __convert_x86<__vector_type_t<int, _Np * 2>>(__v0, __v1));
	    //}
	  }
	else if constexpr (__f32_to_ibw) //{{{2
	  {
	    return __convert_x86<_To>(
	      __convert_x86<__vector_type_t<int, _Np>>(__v0),
	      __convert_x86<__vector_type_t<int, _Np>>(__v1));
	  } //}}}

	// fallback: {{{2
	if constexpr (sizeof(_To) >= 32)
	  // if _To is ymm or zmm, then _SimdWrapper<_Up, _M / 2> is xmm or ymm
	  return __concat(__convert_x86<__vector_type_t<_Up, _M / 2>>(__v0),
			  __convert_x86<__vector_type_t<_Up, _M / 2>>(__v1));
	else if constexpr (sizeof(_To) == 16)
	  {
	    const auto __lo = __to_intrin(__convert_x86<_To>(__v0));
	    const auto __hi = __to_intrin(__convert_x86<_To>(__v1));
	    if constexpr (sizeof(_Up) * _Np == 8)
	      {
		if constexpr (is_floating_point_v<_Up>)
		  return __auto_bitcast(
		    _mm_unpacklo_pd(__vector_bitcast<double>(__lo),
				    __vector_bitcast<double>(__hi)));
		else
		  return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi));
	      }
	    else if constexpr (sizeof(_Up) * _Np == 4)
	      {
		if constexpr (is_floating_point_v<_Up>)
		  return __auto_bitcast(
		    _mm_unpacklo_ps(__vector_bitcast<float>(__lo),
				    __vector_bitcast<float>(__hi)));
		else
		  return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__lo, __hi));
	      }
	    else if constexpr (sizeof(_Up) * _Np == 2)
	      return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__lo, __hi));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return __vector_convert<_To>(__v0, __v1, make_index_sequence<_Np>());
	//}}}
      }
  }

//}}}1
// 4-arg __convert_x86 {{{1
template <typename _To, typename _V, typename _Traits>
  _GLIBCXX_SIMD_INTRINSIC _To
  __convert_x86(_V __v0, _V __v1, _V __v2, _V __v3)
  {
    static_assert(__is_vector_type_v<_V>);
    using _Tp = typename _Traits::value_type;
    constexpr size_t _Np = _Traits::_S_full_size;
    [[maybe_unused]] const auto __i0 = __to_intrin(__v0);
    [[maybe_unused]] const auto __i1 = __to_intrin(__v1);
    [[maybe_unused]] const auto __i2 = __to_intrin(__v2);
    [[maybe_unused]] const auto __i3 = __to_intrin(__v3);
    using _Up = typename _VectorTraits<_To>::value_type;
    constexpr size_t _M = _VectorTraits<_To>::_S_full_size;

    static_assert(4 * _Np <= _M,
		  "__v2/__v3 would be discarded; use the two/one-argument "
		  "__convert_x86 overload instead");

    // [xyz]_to_[xyz] {{{2
    [[maybe_unused]] constexpr bool __x_to_x
      = sizeof(__v0) <= 16 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __x_to_y
      = sizeof(__v0) <= 16 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __x_to_z
      = sizeof(__v0) <= 16 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __y_to_x
      = sizeof(__v0) == 32 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __y_to_y
      = sizeof(__v0) == 32 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __y_to_z
      = sizeof(__v0) == 32 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __z_to_x
      = sizeof(__v0) == 64 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __z_to_y
      = sizeof(__v0) == 64 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __z_to_z
      = sizeof(__v0) == 64 && sizeof(_To) == 64;

    // iX_to_iX {{{2
    [[maybe_unused]] constexpr bool __i_to_i
      = is_integral_v<_Up> && is_integral_v<_Tp>;
    [[maybe_unused]] constexpr bool __i8_to_i16
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i8_to_i32
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __i8_to_i64
      = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i16_to_i8
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i16_to_i32
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __i16_to_i64
      = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i32_to_i8
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i32_to_i16
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i32_to_i64
      = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __i64_to_i8
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __i64_to_i16
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 2;
    [[maybe_unused]] constexpr bool __i64_to_i32
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 4;

    // [fsu]X_to_[fsu]X {{{2
    // ibw = integral && byte or word, i.e. char and short with any signedness
    [[maybe_unused]] constexpr bool __i64_to_f32
      = is_integral_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s32_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s16_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s8_to_f32
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u32_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u16_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __u8_to_f32
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;
    [[maybe_unused]] constexpr bool __s64_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __s32_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __s16_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __s8_to_f64
      = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u64_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u32_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u16_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __u8_to_f64
      = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __f32_to_s64
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_s32
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_u64
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f32_to_u32
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f64_to_s64
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_s32
      = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_u64
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f64_to_u32
      = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f32_to_ibw
      = is_integral_v<_Up> && sizeof(_Up) <= 2
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 4;
    [[maybe_unused]] constexpr bool __f64_to_ibw
      = is_integral_v<_Up> && sizeof(_Up) <= 2
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;
    [[maybe_unused]] constexpr bool __f32_to_f64
      = is_floating_point_v<_Tp> && sizeof(_Tp) == 4
	&& is_floating_point_v<_Up> && sizeof(_Up) == 8;
    [[maybe_unused]] constexpr bool __f64_to_f32
      = is_floating_point_v<_Tp> && sizeof(_Tp) == 8
	&& is_floating_point_v<_Up> && sizeof(_Up) == 4;

    if constexpr (__i_to_i && __y_to_x && !__have_avx2) //{{{2
      {
	// <double, 4>, <double, 4>, <double, 4>, <double, 4> => <char, 16>
	return __convert_x86<_To>(__lo128(__v0), __hi128(__v0), __lo128(__v1),
				  __hi128(__v1), __lo128(__v2), __hi128(__v2),
				  __lo128(__v3), __hi128(__v3));
      }
    else if constexpr (__i_to_i) // assert ISA {{{2
      {
	static_assert(__x_to_x || __have_avx2,
		      "integral conversions with ymm registers require AVX2");
	static_assert(__have_avx512bw
			|| ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64)
			    && (sizeof(_Up) >= 4 || sizeof(_To) < 64)),
		      "8/16-bit integers in zmm registers require AVX512BW");
	static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f,
		      "integral conversions with ymm registers require AVX2");
      }
    // concat => use 2-arg __convert_x86 {{{2
    if constexpr (sizeof(__v0) < 16 || (sizeof(__v0) == 16 && __have_avx2)
		  || (sizeof(__v0) == 16 && __have_avx
		      && is_floating_point_v<_Tp>)
		  || (sizeof(__v0) == 32 && __have_avx512f))
      {
	// The ISA can handle wider input registers, so concat and use two-arg
	// implementation. This reduces code duplication considerably.
	return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3));
      }
    else //{{{2
      {
	// conversion using bit reinterpretation (or no conversion at all)
	// should all go through the concat branch above:
	static_assert(
	  !(is_floating_point_v<
	      _Tp> == is_floating_point_v<_Up> && sizeof(_Tp) == sizeof(_Up)));
	// handle all zero extension{{{2
	if constexpr (4 * _Np < _M && sizeof(_To) > 16)
	  {
	    constexpr size_t Min = 16 / sizeof(_Up);
	    return __zero_extend(
	      __convert_x86<
		__vector_type_t<_Up, (Min > 4 * _Np) ? Min : 4 * _Np>>(
		__v0, __v1, __v2, __v3));
	  }
	else if constexpr (__i64_to_i16) //{{{2
	  {
	    if constexpr (__x_to_x && __have_sse4_1)
	      {
		return __intrin_bitcast<_To>(_mm_shuffle_epi8(
		  _mm_blend_epi16(
		    _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 2), 0x22),
		    _mm_blend_epi16(_mm_slli_si128(__i2, 4),
				    _mm_slli_si128(__i3, 6), 0x88),
		    0xcc),
		  _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7,
				14, 15)));
	      }
	    else if constexpr (__y_to_y && __have_avx2)
	      {
		return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
		  __xzyw(_mm256_blend_epi16(
		    __auto_bitcast(
		      _mm256_shuffle_ps(__vector_bitcast<float>(__v0),
					__vector_bitcast<float>(__v2),
					0x88)), // 0.1. 8.9. 2.3. A.B.
		    __to_intrin(__vector_bitcast<int>(_mm256_shuffle_ps(
				  __vector_bitcast<float>(__v1),
				  __vector_bitcast<float>(__v3), 0x88))
				<< 16), // .4.5 .C.D .6.7 .E.F
		    0xaa)               // 0415 8C9D 2637 AEBF
			 ),             // 0415 2637 8C9D AEBF
		  _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11,
				   14, 15, 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7,
				   10, 11, 14, 15)));
		/*
		auto __a = _mm256_unpacklo_epi16(__v0, __v1);  // 04.. .... 26..
		.... auto __b = _mm256_unpackhi_epi16(__v0, __v1);  // 15..
		.... 37.. .... auto __c = _mm256_unpacklo_epi16(__v2, __v3);  //
		8C.. .... AE.. .... auto __d = _mm256_unpackhi_epi16(__v2,
		__v3);
		// 9D.. .... BF.. .... auto __e = _mm256_unpacklo_epi16(__a,
		__b);
		// 0145 .... 2367 .... auto __f = _mm256_unpacklo_epi16(__c,
		__d);
		// 89CD .... ABEF .... auto __g = _mm256_unpacklo_epi64(__e,
		__f);
		// 0145 89CD 2367 ABEF return __concat(
		    _mm_unpacklo_epi32(__lo128(__g), __hi128(__g)),
		    _mm_unpackhi_epi32(__lo128(__g), __hi128(__g)));  // 0123
		4567 89AB CDEF
		    */
	      } // else use fallback
	  }
	else if constexpr (__i64_to_i8) //{{{2
	  {
	    if constexpr (__x_to_x)
	      {
		// TODO: use fallback for now
	      }
	    else if constexpr (__y_to_x)
	      {
		auto __a
		  = _mm256_srli_epi32(_mm256_slli_epi32(__i0, 24), 24)
		    | _mm256_srli_epi32(_mm256_slli_epi32(__i1, 24), 16)
		    | _mm256_srli_epi32(_mm256_slli_epi32(__i2, 24), 8)
		    | _mm256_slli_epi32(
		      __i3, 24); // 048C .... 159D .... 26AE .... 37BF ....
		/*return _mm_shuffle_epi8(
		    _mm_blend_epi32(__lo128(__a) << 32, __hi128(__a), 0x5),
		    _mm_setr_epi8(4, 12, 0, 8, 5, 13, 1, 9, 6, 14, 2, 10, 7, 15,
		   3, 11));*/
		auto __b = _mm256_unpackhi_epi64(
		  __a, __a); // 159D .... 159D .... 37BF .... 37BF ....
		auto __c = _mm256_unpacklo_epi8(
		  __a, __b); // 0145 89CD .... .... 2367 ABEF .... ....
		return __intrin_bitcast<_To>(
		  _mm_unpacklo_epi16(__lo128(__c),
				     __hi128(__c))); // 0123 4567 89AB CDEF
	      }
	  }
	else if constexpr (__i32_to_i8) //{{{2
	  {
	    if constexpr (__x_to_x)
	      {
		if constexpr (__have_ssse3)
		  {
		    const auto __x0 = __vector_bitcast<_UInt>(__v0) & 0xff;
		    const auto __x1 = (__vector_bitcast<_UInt>(__v1) & 0xff)
				      << 8;
		    const auto __x2 = (__vector_bitcast<_UInt>(__v2) & 0xff)
				      << 16;
		    const auto __x3 = __vector_bitcast<_UInt>(__v3) << 24;
		    return __intrin_bitcast<_To>(
		      _mm_shuffle_epi8(__to_intrin(__x0 | __x1 | __x2 | __x3),
				       _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13,
						     2, 6, 10, 14, 3, 7, 11,
						     15)));
		  }
		else
		  {
		    auto __a
		      = _mm_unpacklo_epi8(__i0, __i2); // 08.. .... 19.. ....
		    auto __b
		      = _mm_unpackhi_epi8(__i0, __i2); // 2A.. .... 3B.. ....
		    auto __c
		      = _mm_unpacklo_epi8(__i1, __i3); // 4C.. .... 5D.. ....
		    auto __d
		      = _mm_unpackhi_epi8(__i1, __i3); // 6E.. .... 7F.. ....
		    auto __e
		      = _mm_unpacklo_epi8(__a, __c); // 048C .... .... ....
		    auto __f
		      = _mm_unpackhi_epi8(__a, __c); // 159D .... .... ....
		    auto __g
		      = _mm_unpacklo_epi8(__b, __d); // 26AE .... .... ....
		    auto __h
		      = _mm_unpackhi_epi8(__b, __d); // 37BF .... .... ....
		    return __intrin_bitcast<_To>(_mm_unpacklo_epi8(
		      _mm_unpacklo_epi8(__e, __g), // 0246 8ACE .... ....
		      _mm_unpacklo_epi8(__f, __h)  // 1357 9BDF .... ....
		      ));                          // 0123 4567 89AB CDEF
		  }
	      }
	    else if constexpr (__y_to_y)
	      {
		const auto __a = _mm256_shuffle_epi8(
		  __to_intrin((__vector_bitcast<_UShort>(_mm256_blend_epi16(
				 __i0, _mm256_slli_epi32(__i1, 16), 0xAA))
			       & 0xff)
			      | (__vector_bitcast<_UShort>(_mm256_blend_epi16(
				   __i2, _mm256_slli_epi32(__i3, 16), 0xAA))
				 << 8)),
		  _mm256_setr_epi8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7,
				   11, 15, 0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9,
				   13, 3, 7, 11, 15));
		return __intrin_bitcast<_To>(_mm256_permutevar8x32_epi32(
		  __a, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7)));
	      }
	  }
	else if constexpr (__i64_to_f32) //{{{2
	  {
	    // this branch is only relevant with AVX and w/o AVX2 (i.e. no ymm
	    // integers)
	    if constexpr (__x_to_y)
	      {
		return __make_wrapper<float>(__v0[0], __v0[1], __v1[0], __v1[1],
					     __v2[0], __v2[1], __v3[0],
					     __v3[1]);

		const auto __a = _mm_unpacklo_epi32(__i0, __i1);   // acAC
		const auto __b = _mm_unpackhi_epi32(__i0, __i1);   // bdBD
		const auto __c = _mm_unpacklo_epi32(__i2, __i3);   // egEG
		const auto __d = _mm_unpackhi_epi32(__i2, __i3);   // fhFH
		const auto __lo32a = _mm_unpacklo_epi32(__a, __b); // abcd
		const auto __lo32b = _mm_unpacklo_epi32(__c, __d); // efgh
		const auto __hi32 = __vector_bitcast<
		  conditional_t<is_signed_v<_Tp>, int, _UInt>>(
		  __concat(_mm_unpackhi_epi32(__a, __b),
			   _mm_unpackhi_epi32(__c, __d))); // ABCD EFGH
		const auto __hi
		  = 0x100000000LL
		    * __convert_x86<__vector_type_t<float, 8>>(__hi32);
		const auto __mid
		  = 0x10000
		    * _mm256_cvtepi32_ps(__concat(_mm_srli_epi32(__lo32a, 16),
						  _mm_srli_epi32(__lo32b, 16)));
		const auto __lo = _mm256_cvtepi32_ps(
		  __concat(_mm_set1_epi32(0x0000ffffu) & __lo32a,
			   _mm_set1_epi32(0x0000ffffu) & __lo32b));
		return (__hi + __mid) + __lo;
	      }
	  }
	else if constexpr (__f64_to_ibw) //{{{2
	  {
	    return __convert_x86<_To>(
	      __convert_x86<__vector_type_t<int, _Np * 2>>(__v0, __v1),
	      __convert_x86<__vector_type_t<int, _Np * 2>>(__v2, __v3));
	  }
	else if constexpr (__f32_to_ibw) //{{{2
	  {
	    return __convert_x86<_To>(
	      __convert_x86<__vector_type_t<int, _Np>>(__v0),
	      __convert_x86<__vector_type_t<int, _Np>>(__v1),
	      __convert_x86<__vector_type_t<int, _Np>>(__v2),
	      __convert_x86<__vector_type_t<int, _Np>>(__v3));
	  } //}}}

	// fallback: {{{2
	if constexpr (sizeof(_To) >= 32)
	  // if _To is ymm or zmm, then _SimdWrapper<_Up, _M / 2> is xmm or ymm
	  return __concat(__convert_x86<__vector_type_t<_Up, _M / 2>>(__v0,
								      __v1),
			  __convert_x86<__vector_type_t<_Up, _M / 2>>(__v2,
								      __v3));
	else if constexpr (sizeof(_To) == 16)
	  {
	    const auto __lo = __to_intrin(__convert_x86<_To>(__v0, __v1));
	    const auto __hi = __to_intrin(__convert_x86<_To>(__v2, __v3));
	    if constexpr (sizeof(_Up) * _Np * 2 == 8)
	      {
		if constexpr (is_floating_point_v<_Up>)
		  return __auto_bitcast(_mm_unpacklo_pd(__lo, __hi));
		else
		  return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi));
	      }
	    else if constexpr (sizeof(_Up) * _Np * 2 == 4)
	      {
		if constexpr (is_floating_point_v<_Up>)
		  return __auto_bitcast(_mm_unpacklo_ps(__lo, __hi));
		else
		  return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__lo, __hi));
	      }
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return __vector_convert<_To>(__v0, __v1, __v2, __v3,
				       make_index_sequence<_Np>());
	//}}}2
      }
  }

//}}}
// 8-arg __convert_x86 {{{1
template <typename _To, typename _V, typename _Traits>
  _GLIBCXX_SIMD_INTRINSIC _To
  __convert_x86(_V __v0, _V __v1, _V __v2, _V __v3, _V __v4, _V __v5, _V __v6,
		_V __v7)
  {
    static_assert(__is_vector_type_v<_V>);
    using _Tp = typename _Traits::value_type;
    constexpr size_t _Np = _Traits::_S_full_size;
    [[maybe_unused]] const auto __i0 = __to_intrin(__v0);
    [[maybe_unused]] const auto __i1 = __to_intrin(__v1);
    [[maybe_unused]] const auto __i2 = __to_intrin(__v2);
    [[maybe_unused]] const auto __i3 = __to_intrin(__v3);
    [[maybe_unused]] const auto __i4 = __to_intrin(__v4);
    [[maybe_unused]] const auto __i5 = __to_intrin(__v5);
    [[maybe_unused]] const auto __i6 = __to_intrin(__v6);
    [[maybe_unused]] const auto __i7 = __to_intrin(__v7);
    using _Up = typename _VectorTraits<_To>::value_type;
    constexpr size_t _M = _VectorTraits<_To>::_S_full_size;

    static_assert(8 * _Np <= _M,
		  "__v4-__v7 would be discarded; use the four/two/one-argument "
		  "__convert_x86 overload instead");

    // [xyz]_to_[xyz] {{{2
    [[maybe_unused]] constexpr bool __x_to_x
      = sizeof(__v0) <= 16 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __x_to_y
      = sizeof(__v0) <= 16 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __x_to_z
      = sizeof(__v0) <= 16 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __y_to_x
      = sizeof(__v0) == 32 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __y_to_y
      = sizeof(__v0) == 32 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __y_to_z
      = sizeof(__v0) == 32 && sizeof(_To) == 64;
    [[maybe_unused]] constexpr bool __z_to_x
      = sizeof(__v0) == 64 && sizeof(_To) <= 16;
    [[maybe_unused]] constexpr bool __z_to_y
      = sizeof(__v0) == 64 && sizeof(_To) == 32;
    [[maybe_unused]] constexpr bool __z_to_z
      = sizeof(__v0) == 64 && sizeof(_To) == 64;

    // [if]X_to_i8 {{{2
    [[maybe_unused]] constexpr bool __i_to_i
      = is_integral_v<_Up> && is_integral_v<_Tp>;
    [[maybe_unused]] constexpr bool __i64_to_i8
      = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1;
    [[maybe_unused]] constexpr bool __f64_to_i8
      = is_integral_v<_Up> && sizeof(_Up) == 1
	&& is_floating_point_v<_Tp> && sizeof(_Tp) == 8;

    if constexpr (__i_to_i) // assert ISA {{{2
      {
	static_assert(__x_to_x || __have_avx2,
		      "integral conversions with ymm registers require AVX2");
	static_assert(__have_avx512bw
			|| ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64)
			    && (sizeof(_Up) >= 4 || sizeof(_To) < 64)),
		      "8/16-bit integers in zmm registers require AVX512BW");
	static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f,
		      "integral conversions with ymm registers require AVX2");
      }
    // concat => use 4-arg __convert_x86 {{{2
    if constexpr (sizeof(__v0) < 16 || (sizeof(__v0) == 16 && __have_avx2)
		  || (sizeof(__v0) == 16 && __have_avx
		      && is_floating_point_v<_Tp>)
		  || (sizeof(__v0) == 32 && __have_avx512f))
      {
	// The ISA can handle wider input registers, so concat and use two-arg
	// implementation. This reduces code duplication considerably.
	return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3),
				  __concat(__v4, __v5), __concat(__v6, __v7));
      }
    else //{{{2
      {
	// conversion using bit reinterpretation (or no conversion at all)
	// should all go through the concat branch above:
	static_assert(
	  !(is_floating_point_v<
	      _Tp> == is_floating_point_v<_Up> && sizeof(_Tp) == sizeof(_Up)));
	static_assert(!(8 * _Np < _M && sizeof(_To) > 16),
		      "zero extension should be impossible");
	if constexpr (__i64_to_i8) //{{{2
	  {
	    if constexpr (__x_to_x && __have_ssse3)
	      {
		// unsure whether this is better than the variant below
		return __intrin_bitcast<_To>(_mm_shuffle_epi8(
		  __to_intrin(
		    (((__v0 & 0xff) | ((__v1 & 0xff) << 8))
		     | (((__v2 & 0xff) << 16) | ((__v3 & 0xff) << 24)))
		    | ((((__v4 & 0xff) << 32) | ((__v5 & 0xff) << 40))
		       | (((__v6 & 0xff) << 48) | (__v7 << 56)))),
		  _mm_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14,
				7, 15)));
	      }
	    else if constexpr (__x_to_x)
	      {
		const auto __a = _mm_unpacklo_epi8(__i0, __i1); // ac
		const auto __b = _mm_unpackhi_epi8(__i0, __i1); // bd
		const auto __c = _mm_unpacklo_epi8(__i2, __i3); // eg
		const auto __d = _mm_unpackhi_epi8(__i2, __i3); // fh
		const auto __e = _mm_unpacklo_epi8(__i4, __i5); // ik
		const auto __f = _mm_unpackhi_epi8(__i4, __i5); // jl
		const auto __g = _mm_unpacklo_epi8(__i6, __i7); // mo
		const auto __h = _mm_unpackhi_epi8(__i6, __i7); // np
		return __intrin_bitcast<_To>(_mm_unpacklo_epi64(
		  _mm_unpacklo_epi32(_mm_unpacklo_epi8(__a, __b),  // abcd
				     _mm_unpacklo_epi8(__c, __d)), // efgh
		  _mm_unpacklo_epi32(_mm_unpacklo_epi8(__e, __f),  // ijkl
				     _mm_unpacklo_epi8(__g, __h))  // mnop
		  ));
	      }
	    else if constexpr (__y_to_y)
	      {
		auto __a = // 048C GKOS 159D HLPT 26AE IMQU 37BF JNRV
		  __to_intrin(
		    (((__v0 & 0xff) | ((__v1 & 0xff) << 8))
		     | (((__v2 & 0xff) << 16) | ((__v3 & 0xff) << 24)))
		    | ((((__v4 & 0xff) << 32) | ((__v5 & 0xff) << 40))
		       | (((__v6 & 0xff) << 48) | ((__v7 << 56)))));
		/*
		auto __b = _mm256_unpackhi_epi64(__a, __a);  // 159D HLPT 159D
		HLPT 37BF JNRV 37BF JNRV auto __c = _mm256_unpacklo_epi8(__a,
		__b);  // 0145 89CD GHKL OPST 2367 ABEF IJMN QRUV auto __d =
		__xzyw(__c); // 0145 89CD 2367 ABEF GHKL OPST IJMN QRUV return
		_mm256_shuffle_epi8(
		    __d, _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12,
		13, 6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7,
		14, 15));
					*/
		auto __b = _mm256_shuffle_epi8( // 0145 89CD GHKL OPST 2367 ABEF
						// IJMN QRUV
		  __a, _mm256_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13,
					6, 14, 7, 15, 0, 8, 1, 9, 2, 10, 3, 11,
					4, 12, 5, 13, 6, 14, 7, 15));
		auto __c
		  = __xzyw(__b); // 0145 89CD 2367 ABEF GHKL OPST IJMN QRUV
		return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
		  __c, _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13,
					6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11,
					4, 5, 12, 13, 6, 7, 14, 15)));
	      }
	    else if constexpr (__z_to_z)
	      {
		return __concat(
		  __convert_x86<__vector_type_t<_Up, _M / 2>>(__v0, __v1, __v2,
							      __v3),
		  __convert_x86<__vector_type_t<_Up, _M / 2>>(__v4, __v5, __v6,
							      __v7));
	      }
	  }
	else if constexpr (__f64_to_i8) //{{{2
	  {
	    return __convert_x86<_To>(
	      __convert_x86<__vector_type_t<int, _Np * 2>>(__v0, __v1),
	      __convert_x86<__vector_type_t<int, _Np * 2>>(__v2, __v3),
	      __convert_x86<__vector_type_t<int, _Np * 2>>(__v4, __v5),
	      __convert_x86<__vector_type_t<int, _Np * 2>>(__v6, __v7));
	  }
	else // unreachable {{{2
	  __assert_unreachable<_Tp>();
	//}}}

	// fallback: {{{2
	if constexpr (sizeof(_To) >= 32)
	  // if _To is ymm or zmm, then _SimdWrapper<_Up, _M / 2> is xmm or ymm
	  return __concat(
	    __convert_x86<__vector_type_t<_Up, _M / 2>>(__v0, __v1, __v2, __v3),
	    __convert_x86<__vector_type_t<_Up, _M / 2>>(__v4, __v5, __v6,
							__v7));
	else if constexpr (sizeof(_To) == 16)
	  {
	    const auto __lo
	      = __to_intrin(__convert_x86<_To>(__v0, __v1, __v2, __v3));
	    const auto __hi
	      = __to_intrin(__convert_x86<_To>(__v4, __v5, __v6, __v7));
	    static_assert(sizeof(_Up) == 1 && _Np == 2);
	    return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi));
	  }
	else
	  {
	    __assert_unreachable<_Tp>();
	    // return __vector_convert<_To>(__v0, __v1, __v2, __v3, __v4, __v5,
	    // __v6, __v7,
	    //                             make_index_sequence<_Np>());
	  } //}}}2
      }
  }

//}}}
// 16-arg __convert_x86 {{{1
template <typename _To, typename _V, typename _Traits>
  _GLIBCXX_SIMD_INTRINSIC _To
  __convert_x86(_V __v0, _V __v1, _V __v2, _V __v3, _V __v4, _V __v5, _V __v6,
		_V __v7, _V __v8, _V __v9, _V __v10, _V __v11, _V __v12,
		_V __v13, _V __v14, _V __v15)
  {
    // concat => use 8-arg __convert_x86
    return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3),
			      __concat(__v4, __v5), __concat(__v6, __v7),
			      __concat(__v8, __v9), __concat(__v10, __v11),
			      __concat(__v12, __v13), __concat(__v14, __v15));
  }

//}}}

#endif // __cplusplus >= 201703L
#endif // _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H

// vim: foldmethod=marker
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      $,HD$ HD$ AH  H@@|$/ HH$a  H|$8 
      HH`  H   H%  H=  uH@   @H   AD$
I$HHT$fAD$
AH  HD$0fAL$H   HL$HD$    HT$HL$>  Hп   %  H  H   H+5    HH|$E1HHA   HH5        H  AT$
H$ID$Ã$fAT$
HAGfA9
  t=HRHI 
  AfD$,DH4vL$D
,  A~11I}  
      HHLd$@Ah  @    f|$, t*LDHfA
  K    IED D$,HH[]A\A]A^A_    =     H|$LILd$@t=L$    1HE     $fEE
fE
Ah  @c    HD$0    H   HuH   $    HH    H    $1E1Ld$@DILLd$@Z1%H5        US HLg  L  L  L
  1L
  taHc1H    H)HLfp@     f$   H@(t&HRH41F
fF
$  9|;L  rP  E1ۅt`Ic11HHi  HLG    AD    f t$HwHHF
fF
G9|AD;P  rL"i  tfH;1H   P  u
G;P  s<1҉H#u[]    HCH    H    H       HCH   {U   8HC   1HC  Ce   HC  HCSb  DL  HC  A  D  E~0H    H  1DHCD HH  D9|HC$    HC
  @  P  t1H    1HQHSH3
  2HH;P  rHCH`  
  %  HCH  d  P  t4H    H
  1FDHCD HH  ;P  rHCHx  
  %  { upHC  $     HC,    HC   HC   HC(  HC  1HC  HC  HC   4  Cd  @HCE@π   HCH  Ch% HCH  CJ  HC  H{    S,9GC,   C H{%  @    Cq%  ʀCd@E{ tC0EH{  C,HSp
|   1[]    HGH  U     P  uHC  aS^HC  Cd   {    ?   ED  A   H{       C,    HC   o HGH   ]f         AW1AVLx  AULoATUSHH H/eH%(   HD$1HGLfT$H$1fD$    LI    LLA    EuwH    CUL\uQ  L  L    HFHHD$eH+%(     H []A\A]A^A_    i   t    CU L    LIHT$Ht$    LLD$    D$uT$D$щ	f9t  u
f9v     t  LZt$L    fv  CT1ft  CPHCH  {U    HCH  {U    HCH  {U         {T un
  4sP%H{  0 	sH
 <  	  HK  HS  h      HE8}H    p
  4듋 HCH   i HC  , HC  1    ff.     f    AUATUSL'HL    HŅ^  LkH+ƃl  L      l  H  HCII1H          Ņ   h   tD  HX  L  E1t,IcAH    H)HH8      D;L  rԋP  E1t$IcAHi  H
      D;P  r|  u"HC   HCSD  HC I$     []A\A]    h       ƃl      h       l   uQP  tE1DHAD;P  r[]A\A]    I   H    H        H{    ff.         USHH  HE8tHH	  u
H[]           []    f         AT1USH_xH	    H	  HeH    HH    H    HC8u[1]A\    L	  LS1[]A\             ATUSHGHu HLHw    Hsr	[]A\    C1H     H   H H      L'KuH   H  H []A\    H    LJLbH    DD      HG8   ATAUHSH  H  s       H+rD     fD	  A  ~AD$))f  H	  HH    H  1[]A\    1    ff.         AVAUATUSH    D  L	  HH	  D  A  ~)Hd       H  f@D9|    
  AAЍ<@Hc    H  H  E  H  11HcfJH  H<vH@H  H4Hh\A9uE9  H  H  1fJH H¸  HH  fdf~HcLH  J<H~D  LDA9uIcIcI   E1H    HHx  HHtGL      Hp  H    Hx  HNH)Eq  H<HH  A~H    H    H)Hf  H  HP  Hx  H  HHHHp  H@TfRH9uE   AD$H  HH  H  Lx  H HƸ  ILAHp  HAAfALx  IHLPHp  HH\fZH9uHŅuH[]A\A]A^    A   Hǽ      H  1H	  H	           ubH	  Hx  E1Hp    H       H  Hǃp      Ht    Hǃ      H	  4H	      됽         AWAVAUATUSHH(HOGt$HLAHr8  t$m  H$HE1|$ L<LD$HT$DKILKAIM̅  Af  L1уfKHHSf9s-JJ{Of;Ku1Ƀ1HSfKf9rAl$  @  f?f  A$E1   H@I 
  L4HD$AVIvH       A$M6LH4II 
  H4N
H    fN
1fA;
  DfA
      Ht$L    fA   H$x AD$Z     t'AD$   A   fA   ffA   LHA    D$A9fH$s   H]H$E  E1DHH$    H$Hi@H    H$HCH    QDQDHA  H    H(D[]A\A]A^A_    Hct$ H<$IIIA<$M 
  ft<11ʃHIMAI
I    fAI
fA9
  DՃf9uH$Hf0
  h  @       A    A   AD$|sHϺ   H$D9l$H$H<$h  @    f  @uH$LD$h  @    A$Hct$ ;E1f         AWAVAUATUSHH@H  G|H    H      IHDAfy AptED   ALIRL+   ESp+StEA9ыQABfH    DI$	  H)HL$HD|$I  
J    L)HLD  fA9X  HD)HHHD$ f9E  t$H|$H   M$	  Iŋ   Hfx j     `<`d	     H   H+   )M	     AU	ЀAE     H      HH+   t
   AEH   Sp+StՉы   |f|$  f  LE1HI+$  HH@I$  LfAJI$	  L   LT$(H   LHL$H4$HD$0    H4$HL$LT$(d  L   L%  Hp	  H   H+5    HH|$0E1HHA   L$HH5        L$IBH"  AR
fAR
HEABfE f|$ 0  E1IL|$(LE-AV
AfAV
HEAFfE fD;l$  A   t$L\$H|$M   H$_H<$E1A   HHHE HCHEHI+$  HH@I$  L4DHHIAO8fANI$	  AW<Iw0H       L\$HIF(L|$(IML1IB    fABA$      N    MM)IMI  I)IMfE9     H$ELL\$H  AHHL,HRH  H,U
u\HuHt%1 I$	  [  R  H} Ht1    U
HE    HE     fU
A1IE     IE    fD;  DDfD;  FH$Ll$J       HL)HfE      1Df  K    I$	  fD H    H|$ H         H@[]A\A]A^A_    11 
  HL$    L$  K    J    I$	  L)HA  f   H    1E11UH   E1    L|$(MLہM   ICpe      =&  H\$       H   K    J    I$	  L)HA  f1C~tH )΃h@      H   {pHpH)f9     HFH)9     1ft
1   H   D
D
    f1     fD
AM   -  H|$t$IE     IE    L$H   1L$I   fLH      HEHH+   9Cp     1E1   HfWHw1H    fEAM 0  CpAEAFM AFAE   H   H+   	AU   H   D%  	AE   HFHH+   DLI+$  HD\$0EH@I$  LfEZH   I$	  LT$8HHT$H   HL$(H$    HT$HL$(D\$0LT$8  Hй   %  H  H   H+
    H4
H<$E1LHA   HLT$HH5    D\$(    LT$IBHkAR
D\$(fAR
IEABfAE fA9s;D)fE8t$H|$$$HIE HE IEHEL{t  HȉspH)   =  Ht$             9*H    Ht$ H      D$       9HD$       ̀AEffAUAEAEA$      =     t
IB
H       HuH   H|$0L$    HH    H    L$뱀=     uH       HuH   H<$LT$    HH    H    LT$bA$      MHL$    L$H   H   {p<A$  p    H      HH
    KH5    ff.     f    SHHX  [    f.     f    HHH  E u8 H7E H  uC	fȉBf

1u	     HH   H7E H  t	H          HH  E u(       uHH   E u      H  ͋ H   f.         FHȉ  Hf      D      AVAUATI1US   HE IH     *AA   I61ɉ  I%    
   H   x4        A~E ItH   IH   yIH  A~E    IH  A~E uk 	к   A$Et)[]A\A]A^    I6  ;1EuI  []A\A]A^    H   IH   눋 I  fff.                1A   1D)	Ѓ u        H  HH            UHS1   t7Ƨ      xt1HE H  }E tՋ HE H     uɸ   []    1[]    ff.     f    fT x    H1҉  off.     @     fT x    UHSH   ځ           aHE   R []    ff.     f    AWAAVAAUMATUSfT HyH1҉      1fAE CD`A   {E    DEEtfDH%   	艁H  D  @H  H       HH  {E ur fAE f{T x-1[]A\A]A^A_       D	   E1vDH     HA  a    HD  R     뜋 HH   끸    AWAAVAAUEATUHSfT yH1҉      ED`A   }E    DEEE	EtPAHE A   A	DH  D  @HE   H    tjf}T x*1[]A\A]A^A_       D	   E1DHE      HA  a    HE D  R     띸fD      UHSGH_VfEAfA D@DEDʀDEDDEf@
  DEEftf ڀ    EfE11   H    u 11Dù	   H    []        AVAUATLgrULSHHeH%(   HD$11fD$uAHH  {E e  D(H    uT{  LH  1SrSlAT$fSpHT$eH+%(   6  H[]A\A]A^    C    HH  {E    H  E1H   t9AAd/  
       {E HtH   HH   uǃ{W  @E111   H    uLD$   11H    DD$11   HAȀfDD$AE1    uA   11H    11A;      H    uLD$   11H    DD$11   HAfDD$AA;       uA   11H    O     .   L    CrCrAbHAD  H        C1E111   H    uLD$   11H    D$E111   H$fD$    uA   11H    11A;      H    uLD$   11H    D$11A;      H߃fD$    uA   11H    O        PHAD         HAD          HD   HH            I11ff.         D11ff.         I@־           A@־   E     UDH1S1ҹ   H    uIH߹   [11]g[]        UDƉ1S1ҹ   H    uDH߹   [11]W[]        AU1ATUSHHLgeH%(   HD$1HfT$Ml$H  E   (CU  Z   H  H Ƨ      H  H m4     C  P  CU{  A)      H߃6  11f%f6fD$    uA   11H    11A      H    uA     11H    CwXHH@  {E    H@  E1<      H߾       AM    Hߺ          E111   H    uA     11H    11A      H    uAF,     11H    11A      H    uAL     11H    11A6      H    uA     11H       11A      HfD$      HT$eH+%(     H[]A\A]    11A>      H    uLD$   11H    DD$11   HfA fDD$AA>       uA   11H    CUr  11A)      H    uLD$   11H    DD$11   HDA)         uA   11H    11A      H    uLD$   11H    DD$11   HDA          >A%Z   IA$i   g    11A
      H    uLD$   11H    DD$11   HDA
         uA   11H    CCP11A;      H    uLD$   11H    DD$11   HAfDD$AA;       uA   11H    C HH@    H      ff.     @     ATUSHHLgIl$tg11LG*           11LC,   H        CFftnv7ft^fuhA   111H    uCx1H[]A\    A   fuHTA   tA$h  t    A   A   A$h      fD      SHHeH%(   HD$11fD$tOLD$   11       11LD$H    D$fHT$eH+%(   u/H[    HH  E u	 ΋ H      ff.     f    UHSHHeH%(   HD$11fD$t_11LD$       u-L$   f f @tff tXft<HT$eH+%(   u`H[]    HH  E u2    fE 1fĸ
   1Ҁ   d    H  ø    f         AW   AVAUATUHSHLoeH%(   HD$11fT$fD$M}fL$    uG   
   E1fAt  fAv  HD$eH+%(   Z  HD[]A\A]A^A_    LD$   11H    \$   @u    u   À   DeHEx HfEHt!ffDeHAfu4   OA   111H    AƅufDeHftAd   d   XA     HuH    u'AuI   H    H        nHT$Ht$H    tfT$D$f% fI   H    H            ff.          SHrt[    HA   111[C     AW1AVAUATUSHHHOeH%(   HD$1    ft$   ft  LaDHH   E   D(HH  {E   (HH  {E q   A	fv  u %   
  AAAt  {A        LD$   11H    fHD   H  HD  HD  HD$eH+%(   +  H1[]A\A]A^A_    AWH    EAAVI$   H    H        XZ11A      H    tD  A0?H;A   
\  D   H;  H;  H  Cx DE1{(AǃG|낋 HH    HD   HH  {E I H  ;I$   H    H    H        :            1@     ATUSHeH%(   HD$11fD$GP   Hf   uyLD$            D$   HA   f%fQEAA  fQ11f%f-MfD$    uE   11H    fdu
{{    HD$eH+%(      H[]A\    { t11A          uA     11H    11A      H    uHD$eH+%(      A  HH߹   1[1]A\L11A      H    uA     11H    11A      H    HD$eH+%(   uAO  {    f.                             (          HF!     F  tF  t
N  t
N   tN    f    ATIUHSH  H  s       H+rA|$taAD$AT$=  tCdtQ1f9	  tf	  H	      u7  1[]A\    u;    @   1             t        US	  H	  f    @̀fC	     ft2Cf  CH{0C    H{@    1[]    C        Hǐ	      	    ff.     @     HG8u1    Hǀ	      1    f         F^uQǇ      F  Fu"1@H	  H       1         ָ    f.         UI1H	  SHzHHHeH%(   HD$1fL$HH)H    (  Hǂ       HF   A	   I	    H   I	  H  A	      CI	  H  A	      CI	  H  A	      CI	  H   A	      CI	  H  A	      CI	  H  A	      CI	  H  A	      CI	  HhA	      CI	  H  A	      C	I	  H  A	      C
I	  H$  A	      CI	  H  A	      CI	  H  A	   }   C
I	  H  A	   s   CI	  H  A	   i   CI	  H  A	   _   CI	  H  A	   U   CI	  H  A	   K   CI	  H  A	   A   CI	  H  A	   7   CI	  H  A	   -   CHT$1H    D$   HHT$       D$$  HD$eH+%(     H[]    H   I	  H    I	  H    I	  H    I	  H    I	  H     I	  H    I	  H   ) I	  H   3 I	  Hh = I	  H   G I	  H   Q I	  H$   [ I	  H   e I	  H   o I	  H   y I	  H    I	  H    I	  H    I	  H    I	  H    I	  H    I	            ATLfUHH    SH        H    HH tHH     H    L    H	  H   HuH       HHd    HH    H tHH     HH[]A\        AWAVAUATUSHFH$  H	  HIH       EeH	  DEe@<AAAED)؃HH<    H>         IH   D9}TA)IAD$D)Md	IM9t3B4;LH    uL    H[]A\A]A^A_    AuAUH<$L    L    1H[]A\A]A^A_    벸    HǇ              H    H    H        H        H|$H    H           HD$H    H           I   LH    H            H   H            H    L        H    L        H    L        H    L        H|$0H    $H       $    I} H    $    $    fv  H<$H    I    LDDt  HH    H   H            H<$H    H           I   H            I   H            H   H    []    H    L    H      I   H    H$    H$    D$$HD$H    H       AL$D$$    HD$AT$H    H           I$	  H    H           I$	  H            I   H            I   H            I   H    D$    D$    H   H            H   H            H   H    D$    D$    H	  H    H           H	  H    H           H        H                           atl1c_mii_ioctl atl1c_probe     atl1c_suspend   atl1c_request_irq               atl1c_vlan_mode atl1c_restore_vlan                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              `  R   `  `    `      b  R   b  b    b  X    b  R   b  b    b  X    b  e(   b  e(   s Q   s s   s X     X     Q        bv    e(                               Ph      |Tl      Xp       \t   @                   i  c                          i  b                          i  `                           i  b                           i  s                          i                                                                            atl1c_power_saving              atl1c_phy_to_ps_link            strnlen strscpy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 atl1c <atl1c_mii_ioctl> write %x %x %s: %s NIC Link is Down
 atl1c_vlan_mode
 phy power saving failed cannot enable PCI device
 cannot obtain PCI resources
 cannot map device registers
 mac address : %pM
 register netdevice failed
 alloc rx buffer failed
 include/linux/dma-mapping.h RX dma_map_single failed atl1c_restore_vlan
 Full Duplex Half Duplex reset mac failed
 atl1c_request_irq OK
 Multi rfd not support yet!
 IPV4 tso with zero data??
 IPV6 tso with zero data??
 Error get phy ID
 Wrong Media type %d
 phy autoneg failed
 get speed and duplex failed
 atl1c 1000M half is invalid
 field "adapter->hw.mac_addr" at drivers/net/ethernet/atheros/atl1c/atl1c_main.c:486     memcpy: detected field-spanning write (size %zu) of single %s (size %zu)
       drivers/net/ethernet/atheros/atl1c/atl1c_main.c atl1c hardware error (status = 0x%x)
   TX/RX overflow (status = 0x%x)
 MAC state machine can't be idle since disabled for 10ms second
 Cannot re-enable PCI device after reset
        No usable DMA configuration,aborting
   %s %s: rejecting DMA map of vmalloc memory
     %s: %s NIC Link is Up<%d Mbps %s>
      Unable to allocate MSI interrupt Error: %d
     Unable to allocate interrupt Error: %d
 Cannot bring device back up after reset
        could not get memory for DMA buffer
    wrong packet! rrs word3 is %x
  payload offset should not an event number
      tx-skb dropped due to dma error
        include/linux/dynamic_queue_limits.h    Error enable PHY linkChange Interrupt
  Error Setting up Auto-Negotiation
      %s: write phy MII_IER failed.
  %s: suspend MAC=%x,MASTER=%x,PHY=0x%x,WOL=%x
   drivers/net/ethernet/atheros/atl1c/atl1c_hw.c   ethtool speed/duplex setting failed
                                                                                                                            license=GPL description=Qualcomm Atheros 100/1000M Ethernet Network Driver author=Qualcomm Atheros Inc. author=Jie Yang alias=pci:v00001969d00001083sv*sd*bc*sc*i* alias=pci:v00001969d00001073sv*sd*bc*sc*i* alias=pci:v00001969d00002062sv*sd*bc*sc*i* alias=pci:v00001969d00002060sv*sd*bc*sc*i* alias=pci:v00001969d00001062sv*sd*bc*sc*i* alias=pci:v00001969d00001063sv*sd*bc*sc*i* depends= retpoline=Y intree=Y name=atl1c vermagic=6.1.0-37-amd64 SMP preempt mod_unload modversions                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          (    0  8  0  (                     8                         (  0  (                   0                     0               0                                   (    0  8  h  8  0  (                     h                                                                           (    0  8  0  (                     8                         (    0  8  H  8  0  (                     H                         (    0  8  X  8  0  (                     X                       (                 (                                                         (  0  (                   0                                                                                                                               (    0  8  X  8  0  (                     X  8  0  (                     X                         (    0  8    8  0  (                                                                      (    0  8  X  8  0  (                     X                       (                 (                 (                                                                                                                                                                         (  0  (                   0                         (    0  8  `  8  0  (                     `                         (    0  8  x  8  0  (                     x                                              h    X    X  (          0  `  x                                                       (  0  (                   0  (                   0                                                                                                       (    0  8  0  (                     8                   (    0  8  0  (                     8                                         (  0  @  0  (                   @                                                                                                   (  @  (                 @                     (               (                                                (          (                         (    0  8  H  8  0  (                     H                                                   (    0  8  H  8  0  (                     H  P  X  P  H                               0               0               0        @  (                                                                                                                        (          (                                                (    0  8  @  8  0  (                     @  8  0  (                     @                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   .    alloc_etherdev_mqs                                      ;JQ    free_irq                                                ΰ    is_vmalloc_addr                                         d/    device_set_wakeup_enable                                'J    pcie_capability_write_word                              ^    pci_find_ext_capability                                 %Q    pci_wake_from_d3                                        R    pci_enable_device                                       S>    skb_put                                                 V_    __napi_alloc_skb                                        O    pci_enable_device_mem                                   "4t    netdev_update_features                                  #WK    consume_skb                                             ,X    netif_napi_add_weight                                   6    queue_work_on                                           @    unregister_netdev                                       o4    dma_unmap_page_attrs                                    ׂ    __pci_register_driver                                   S9    iounmap                                                 "T͎    pci_disable_msi                                         ;䒲    pci_request_regions                                     >    ___pskb_trim                                            8߬i    memcpy                                                  z    kfree                                                   	    enable_irq                                              X    eth_validate_addr                                       j    netdev_warn                                             ܐ    timer_delete_sync                                       4~    pci_set_power_state                                     4    _raw_spin_lock_irqsave                                  n    __dynamic_dev_dbg                                       i    pci_ioremap_bar                                         d    _raw_spin_lock                                          *v    pci_unregister_driver                                       fortify_panic                                           m    __fentry__                                              y_,    pskb_expand_head                                        ם    dev_driver_string                                       (
    dev_addr_mod                                            y6    eth_type_trans                                              pci_read_config_dword                                   ٣    dma_map_page_attrs                                      "    napi_complete_done                                      V
    __stack_chk_fail                                        a*    __napi_schedule                                             strnlen                                                 BW\    netif_device_detach                                     ]    _dev_info                                               v    napi_gro_receive                                        "*    netif_device_attach                                     ^|    page_offset_base                                        ikp    dev_set_threaded                                        u#    synchronize_irq                                         -~    pci_enable_msi                                          9I    _dev_err                                                ו    softnet_data                                            Ւ    request_threaded_irq                                    [;i    crc32_le                                                MxN    dma_alloc_attrs                                         p*    napi_enable                                             xB<    register_netdev                                         m    free_netdev                                             (L    phys_base                                               *
    dql_completed                                           ^    ethtool_op_get_link                                     p\    _raw_spin_unlock_irqrestore                             a    netif_tx_wake_queue                                     ŏW    memset                                                  fo    _dev_warn                                               h>y    netif_schedule_queue                                    U<!)    pci_set_master                                          9[    __x86_return_thunk                                      0|    csum_ipv6_magic                                         #    __netdev_alloc_skb                                      P    jiffies                                                     dma_set_coherent_mask                                   le    vmemmap_base                                            9d    strscpy                                                 Tn    dma_free_attrs                                          'a    pcie_get_readrq                                         >$    pci_enable_wake                                         9c    init_timer_key                                          b    pci_release_regions                                         __const_udelay                                          @    __dev_kfree_skb_any                                     -    cancel_work_sync                                        GV    __warn_printk                                               netif_carrier_off                                       I    netif_carrier_on                                        	WN    dql_reset                                               C6    pci_disable_device                                      V+    pcie_set_readrq                                         G}    dma_set_mask                                            	7A    get_random_bytes                                        MK    dev_err_probe                                           0j    ethtool_convert_legacy_u32_to_link_mode                     napi_schedule_prep                                      ұ    napi_disable                                            4K    _raw_spin_unlock                                            msleep                                                  E:#    __kmalloc                                               l i    pci_write_config_dword                                  _5    netdev_info                                             Ӆ3-    system_wq                                               o<    disable_irq                                             zR    module_layout                                                   		        		        		        L		        L		        L		        M           M                                                                                                                                                                                                                                                                                                                                                                                                                                                                           atl1c                                                   
                                                     	                                                     L	                                                     }                                                                                                                                                               ?                                                     8                                                                                                                                                                       atl1c                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    GCC: (Debian 12.2.0-14+deb12u1) 12.2.0  GCC: (Debian 12.2.0-14+deb12u1) 12.2.0  GCC: (Debian 12.2.0-14+deb12u1) 12.2.0  GCC: (Debian 12.2.0-14+deb12u1) 12.2.0             u         
9             X    j          
Y A=      h        #       P=        W=    0   _= 
  0   h        %{            @   k=    `   x=      =      =      :"  %     =  
     = ^ @        
        %                              ] =      u +      EL +     = +      = +  @   =      = *       = +      D{ +  @   = *   `   =      > +      > +      EL +  @    +  P    > +  `   &>      = +      9>      I>    Z>    l>    ~>      >     >    >      >     >    >    >      	?     ?    -?    ??    R?    e?    x?      ?     ?    ?    ?    ?    ?    ?    ? 1    ?        ?    @   ?       7`       ?       ?    @  @      @      @       %@    @  -@      6@      C@       Q@    @  _@      n@      ~@       @    @  @      @      @       @    @  @      @      @       @    @  @      @`      @       A    @  A      A      )A       2A    @  ?A      MA      [A     	  jA    @	  zA    	  A    	  A     
  A    @
  A    
  A    
  A       A    @  A      A      A       A +     B        W k @   B g    B c    B d    (B f    u &      q3 &     3B &      @B &   0   $   @  TB &   P  \B &   `  dB *    sB *     }B $     B &     B &     B e    B K      &I
 K   (  B &   0  B &   @  B &   P  B &   `  C &   p  	C &     C &     C &     C &     .C *     8C &     CC &     NC $      XC $     bC *      kC *   @  z 	  `  xC 	    C K     C K     C K     C K     C   p   %      Z   @     i    7
 h   C \   C &   @  C    `  C      ]       C       C *   @  D K   `  ^ *     D &     D &     ) *    (D ]     0D Q     t  I     <D I  @  MD l   YD r @  bD s @P  kD t @T  tD *   @         j ~D      s  k       i  W   @           D             ơ  &   @   ]  &   P   i  W      D 
    W k     s  k   @   i  W      b  &        &        &      D &      D ]      D o @  B  f           m D       s  k       i  W   @     &        &      D &      D &      D o    D 	    W k     s  k   @   i  W      b  &        &        &      D &      D &      B  f  @             n                  p                  q       D      D &       E &      E &       E &   0   &E &   @   2E &   P   ;E *   `   BE *                 u       IE      ^E &       fE $      pE &       ~E &   0   E *   @          
w            x              
y        i        _        n        |        `        q        p        a        b        h        l        [ E       E    p   E    R  E    cO        
\  Z     X  E     E    R  F      F    R  F      ,F            
    %  :F     FF           
    W k QF           
   W k \F     eF           
}	      %  zF           
| W k x  *   F           
     %  F     F    #       
   W k x  *   F K   F     F    N  F    #       

   %  F      G           
      { D &   G           
     { G           
    W k x  *   /G           
    %  
   :    CG           
     %  h    #         OG           
    %  h    #    `G           
    %  SP    pG           
    %      G           
   // Simd x86 specific implementations -*- C++ -*-

// Copyright (C) 2020-2022 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_X86_H_
#define _GLIBCXX_EXPERIMENTAL_SIMD_X86_H_

#if __cplusplus >= 201703L

#if !_GLIBCXX_SIMD_X86INTRIN
#error                                                                         \
  "simd_x86.h may only be included when MMX or SSE on x86(_64) are available"
#endif

_GLIBCXX_SIMD_BEGIN_NAMESPACE

// __to_masktype {{{
// Given <T, N> return <__int_for_sizeof_t<T>, N>. For _SimdWrapper and
// __vector_type_t.
template <typename _Tp, size_t _Np>
  _GLIBCXX_SIMD_INTRINSIC constexpr _SimdWrapper<__int_for_sizeof_t<_Tp>, _Np>
  __to_masktype(_SimdWrapper<_Tp, _Np> __x)
  {
    return reinterpret_cast<__vector_type_t<__int_for_sizeof_t<_Tp>, _Np>>(
      __x._M_data);
  }

template <typename _TV,
	  typename _TVT
	  = enable_if_t<__is_vector_type_v<_TV>, _VectorTraits<_TV>>,
	  typename _Up = __int_for_sizeof_t<typename _TVT::value_type>>
  _GLIBCXX_SIMD_INTRINSIC constexpr __vector_type_t<_Up, _TVT::_S_full_size>
  __to_masktype(_TV __x)
  { return reinterpret_cast<__vector_type_t<_Up, _TVT::_S_full_size>>(__x); }

// }}}
// __interleave128_lo {{{
template <typename _Ap, typename _Bp, typename _Tp = common_type_t<_Ap, _Bp>,
	  typename _Trait = _VectorTraits<_Tp>>
  _GLIBCXX_SIMD_INTRINSIC constexpr _Tp
  __interleave128_lo(const _Ap& __av, const _Bp& __bv)
  {
    const _Tp __a(__av);
    const _Tp __b(__bv);
    if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 2)
      return _Tp{__a[0], __b[0]};
    else if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 4)
      return _Tp{__a[0], __b[0], __a[1], __b[1]};
    else if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 8)
      return _Tp{__a[0], __b[0], __a[1], __b[1],
		 __a[2], __b[2], __a[3], __b[3]};
    else if constexpr (sizeof(_Tp) == 16 && _Trait::_S_full_size == 16)
      return _Tp{__a[0], __b[0], __a[1], __b[1], __a[2], __b[2],
		 __a[3], __b[3], __a[4], __b[4], __a[5], __b[5],
		 __a[6], __b[6], __a[7], __b[7]};
    else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 4)
      return _Tp{__a[0], __b[0], __a[2], __b[2]};
    else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 8)
      return _Tp{__a[0], __b[0], __a[1], __b[1],
		 __a[4], __b[4], __a[5], __b[5]};
    else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 16)
      return _Tp{__a[0],  __b[0],  __a[1],  __b[1], __a[2], __b[2],
		 __a[3],  __b[3],  __a[8],  __b[8], __a[9], __b[9],
		 __a[10], __b[10], __a[11], __b[11]};
    else if constexpr (sizeof(_Tp) == 32 && _Trait::_S_full_size == 32)
      return _Tp{__a[0],  __b[0],  __a[1],  __b[1],  __a[2],  __b[2],  __a[3],
		 __b[3],  __a[4],  __b[4],  __a[5],  __b[5],  __a[6],  __b[6],
		 __a[7],  __b[7],  __a[16], __b[16], __a[17], __b[17], __a[18],
		 __b[18], __a[19], __b[19], __a[20], __b[20], __a[21], __b[21],
		 __a[22], __b[22], __a[23], __b[23]};
    else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 8)
      return _Tp{__a[0], __b[0], __a[2], __b[2],
		 __a[4], __b[4], __a[6], __b[6]};
    else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 16)
      return _Tp{__a[0],  __b[0],  __a[1],  __b[1], __a[4], __b[4],
		 __a[5],  __b[5],  __a[8],  __b[8], __a[9], __b[9],
		 __a[12], __b[12], __a[13], __b[13]};
    else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 32)
      return _Tp{__a[0],  __b[0],  __a[1],  __b[1],  __a[2],  __b[2],  __a[3],
		 __b[3],  __a[8],  __b[8],  __a[9],  __b[9],  __a[10], __b[10],
		 __a[11], __b[11], __a[16], __b[16], __a[17], __b[17], __a[18],
		 __b[18], __a[19], __b[19], __a[24], __b[24], __a[25], __b[25],
		 __a[26], __b[26], __a[27], __b[27]};
    else if constexpr (sizeof(_Tp) == 64 && _Trait::_S_full_size == 64)
      return _Tp{__a[0],  __b[0],  __a[1],  __b[1],  __a[2],  __b[2],  __a[3],
		 __b[3],  __a[4],  __b[4],  __a[5],  __b[5],  __a[6],  __b[6],
		 __a[7],  __b[7],  __a[16], __b[16], __a[17], __b[17], __a[18],
		 __b[18], __a[19], __b[19], __a[20], __b[20], __a[21], __b[21],
		 __a[22], __b[22], __a[23], __b[23], __a[32], __b[32], __a[33],
		 __b[33], __a[34], __b[34], __a[35], __b[35], __a[36], __b[36],
		 __a[37], __b[37], __a[38], __b[38], __a[39], __b[39], __a[48],
		 __b[48], __a[49], __b[49], __a[50], __b[50], __a[51], __b[51],
		 __a[52], __b[52], __a[53], __b[53], __a[54], __b[54], __a[55],
		 __b[55]};
    else
      __assert_unreachable<_Tp>();
  }

// }}}
// __is_zero{{{
template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
  _GLIBCXX_SIMD_INTRINSIC constexpr bool
  __is_zero(_Tp __a)
  {
    if (!__builtin_is_constant_evaluated())
      {
	if constexpr (__have_avx)
	  {
	    if constexpr (_TVT::template _S_is<float, 8>)
	      return _mm256_testz_ps(__a, __a);
	    else if constexpr (_TVT::template _S_is<double, 4>)
	      return _mm256_testz_pd(__a, __a);
	    else if constexpr (sizeof(_Tp) == 32)
	      return _mm256_testz_si256(__to_intrin(__a), __to_intrin(__a));
	    else if constexpr (_TVT::template _S_is<float>)
	      return _mm_testz_ps(__to_intrin(__a), __to_intrin(__a));
	    else if constexpr (_TVT::template _S_is<double, 2>)
	      return _mm_testz_pd(__a, __a);
	    else
	      return _mm_testz_si128(__to_intrin(__a), __to_intrin(__a));
	  }
	else if constexpr (__have_sse4_1)
	  return _mm_testz_si128(__intrin_bitcast<__m128i>(__a),
				 __intrin_bitcast<__m128i>(__a));
      }
    else if constexpr (sizeof(_Tp) <= 8)
      return reinterpret_cast<__int_for_sizeof_t<_Tp>>(__a) == 0;
    else
      {
	const auto __b = __vector_bitcast<_LLong>(__a);
	if constexpr (sizeof(__b) == 16)
	  return (__b[0] | __b[1]) == 0;
	else if constexpr (sizeof(__b) == 32)
	  return __is_zero(__lo128(__b) | __hi128(__b));
	else if constexpr (sizeof(__b) == 64)
	  return __is_zero(__lo256(__b) | __hi256(__b));
	else
	  __assert_unreachable<_Tp>();
      }
  }

// }}}
// __movemask{{{
template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST int
  __movemask(_Tp __a)
  {
    if constexpr (sizeof(_Tp) == 32)
      {
	if constexpr (_TVT::template _S_is<float>)
	  return _mm256_movemask_ps(__to_intrin(__a));
	else if constexpr (_TVT::template _S_is<double>)
	  return _mm256_movemask_pd(__to_intrin(__a));
	else
	  return _mm256_movemask_epi8(__to_intrin(__a));
      }
    else if constexpr (_TVT::template _S_is<float>)
      return _mm_movemask_ps(__to_intrin(__a));
    else if constexpr (_TVT::template _S_is<double>)
      return _mm_movemask_pd(__to_intrin(__a));
    else
      return _mm_movemask_epi8(__to_intrin(__a));
  }

// }}}
// __testz{{{
template <typename _TI, typename _TVT = _VectorTraits<_TI>>
  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr int
  __testz(_TI __a, _TI __b)
  {
    static_assert(is_same_v<_TI, __intrinsic_type_t<typename _TVT::value_type,
						    _TVT::_S_full_size>>);
    if (!__builtin_is_constant_evaluated())
      {
	if constexpr (sizeof(_TI) == 32)
	  {
	    if constexpr (_TVT::template _S_is<float>)
	      return _mm256_testz_ps(__to_intrin(__a), __to_intrin(__b));
	    else if constexpr (_TVT::template _S_is<double>)
	      return _mm256_testz_pd(__to_intrin(__a), __to_intrin(__b));
	    else
	      return _mm256_testz_si256(__to_intrin(__a), __to_intrin(__b));
	  }
	else if constexpr (_TVT::template _S_is<float> && __have_avx)
	  return _mm_testz_ps(__to_intrin(__a), __to_intrin(__b));
	else if constexpr (_TVT::template _S_is<double> && __have_avx)
	  return _mm_testz_pd(__to_intrin(__a), __to_intrin(__b));
	else if constexpr (__have_sse4_1)
	  return _mm_testz_si128(__intrin_bitcast<__m128i>(__to_intrin(__a)),
				 __intrin_bitcast<__m128i>(__to_intrin(__b)));
	else
	  return __movemask(0 == __and(__a, __b)) != 0;
      }
    else
      return __is_zero(__and(__a, __b));
  }

// }}}
// __testc{{{
// requires SSE4.1 or above
template <typename _TI, typename _TVT = _VectorTraits<_TI>>
  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr int
  __testc(_TI __a, _TI __b)
  {
    static_assert(is_same_v<_TI, __intrinsic_type_t<typename _TVT::value_type,
						    _TVT::_S_full_size>>);
    if (__builtin_is_constant_evaluated())
      return __is_zero(__andnot(__a, __b));

    if constexpr (sizeof(_TI) == 32)
      {
	if constexpr (_TVT::template _S_is<float>)
	  return _mm256_testc_ps(__a, __b);
	else if constexpr (_TVT::template _S_is<double>)
	  return _mm256_testc_pd(__a, __b);
	else
	  return _mm256_testc_si256(__to_intrin(__a), __to_intrin(__b));
      }
    else if constexpr (_TVT::template _S_is<float> && __have_avx)
      return _mm_testc_ps(__to_intrin(__a), __to_intrin(__b));
    else if constexpr (_TVT::template _S_is<double> && __have_avx)
      return _mm_testc_pd(__to_intrin(__a), __to_intrin(__b));
    else
      {
	static_assert(is_same_v<_TI, _TI> && __have_sse4_1);
	return _mm_testc_si128(__intrin_bitcast<__m128i>(__to_intrin(__a)),
			       __intrin_bitcast<__m128i>(__to_intrin(__b)));
      }
  }

// }}}
// __testnzc{{{
template <typename _TI, typename _TVT = _VectorTraits<_TI>>
  _GLIBCXX_SIMD_INTRINSIC _GLIBCXX_CONST constexpr int
  __testnzc(_TI __a, _TI __b)
  {
    static_assert(is_same_v<_TI, __intrinsic_type_t<typename _TVT::value_type,
						    _TVT::_S_full_size>>);
    if (!__builtin_is_constant_evaluated())
      {
	if constexpr (sizeof(_TI) == 32)
	  {
	    if constexpr (_TVT::template _S_is<float>)
	      return _mm256_testnzc_ps(__a, __b);
	    else if constexpr (_TVT::template _S_is<double>)
	      return _mm256_testnzc_pd(__a, __b);
	    else
	      return _mm256_testnzc_si256(__to_intrin(__a), __to_intrin(__b));
	  }
	else if constexpr (_TVT::template _S_is<float> && __have_avx)
	  return _mm_testnzc_ps(__to_intrin(__a), __to_intrin(__b));
	else if constexpr (_TVT::template _S_is<double> && __have_avx)
	  return _mm_testnzc_pd(__to_intrin(__a), __to_intrin(__b));
	else if constexpr (__have_sse4_1)
	  return _mm_testnzc_si128(__intrin_bitcast<__m128i>(__to_intrin(__a)),
				   __intrin_bitcast<__m128i>(__to_intrin(__b)));
	else
	  return __movemask(0 == __and(__a, __b)) == 0
		 && __movemask(0 == __andnot(__a, __b)) == 0;
      }
    else
      return !(__is_zero(__and(__a, __b)) || __is_zero(__andnot(__a, __b)));
  }

// }}}
// __xzyw{{{
// shuffles the complete vector, swapping the inner two quarters. Often useful
// for AVX for fixing up a shuffle result.
template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
  _GLIBCXX_SIMD_INTRINSIC _Tp
  __xzyw(_Tp __a)
  {
    if constexpr (sizeof(_Tp) == 16)
      {
	const auto __x = __vector_bitcast<conditional_t<
	  is_floating_point_v<typename _TVT::value_type>, float, int>>(__a);
	return reinterpret_cast<_Tp>(
	  decltype(__x){__x[0], __x[2], __x[1], __x[3]});
      }
    else if constexpr (sizeof(_Tp) == 32)
      {
	const auto __x = __vector_bitcast<conditional_t<
	  is_floating_point_v<typename _TVT::value_type>, double, _LLong>>(__a);
	return reinterpret_cast<_Tp>(
	  decltype(__x){__x[0], __x[2], __x[1], __x[3]});
      }
    else if constexpr (sizeof(_Tp) == 64)
      {
	const auto __x = __vector_bitcast<conditional_t<
	  is_floating_point_v<typename _TVT::value_type>, double, _LLong>>(__a);
	return reinterpret_cast<_Tp>(decltype(__x){__x[0], __x[1], __x[4],
						   __x[5], __x[2], __x[3],
						   __x[6], __x[7]});
      }
    else
      __assert_unreachable<_Tp>();
  }

// }}}
// __maskload_epi32{{{
template <typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC auto
  __maskload_epi32(const int* __ptr, _Tp __k)
  {
    if constexpr (sizeof(__k) == 16)
      return _mm_maskload_epi32(__ptr, __k);
    else
      return _mm256_maskload_epi32(__ptr, __k);
  }

// }}}
// __maskload_epi64{{{
template <typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC auto
  __maskload_epi64(const _LLong* __ptr, _Tp __k)
  {
    if constexpr (sizeof(__k) == 16)
      return _mm_maskload_epi64(__ptr, __k);
    else
      return _mm256_maskload_epi64(__ptr, __k);
  }

// }}}
// __maskload_ps{{{
template <typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC auto
  __maskload_ps(const float* __ptr, _Tp __k)
  {
    if constexpr (sizeof(__k) == 16)
      return _mm_maskload_ps(__ptr, __k);
    else
      return _mm256_maskload_ps(__ptr, __k);
  }

// }}}
// __maskload_pd{{{
template <typename _Tp>
  _GLIBCXX_SIMD_INTRINSIC auto
  __maskload_pd(const double* __ptr, _Tp __k)
  {
    if constexpr (sizeof(__k) == 16)
      return _mm_maskload_pd(__ptr, __k);
    else
      return _mm256_maskload_pd(__ptr, __k);
  }

// }}}

#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
#include "simd_x86_conversions.h"
#endif

// ISA & type detection {{{
template <typename _Tp, size_t _Np>
  constexpr bool
  __is_sse_ps()
  {
    return __have_sse
	   && is_same_v<_Tp,
			float> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 16;
  }

template <typename _Tp, size_t _Np>
  constexpr bool
  __is_sse_pd()
  {
    return __have_sse2
	   && is_same_v<_Tp,
			double> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 16;
  }

template <typename _Tp, size_t _Np>
  constexpr bool
  __is_avx_ps()
  {
    return __have_avx
	   && is_same_v<_Tp,
			float> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 32;
  }

template <typename _Tp, size_t _Np>
  constexpr bool
  __is_avx_pd()
  {
    return __have_avx
	   && is_same_v<_Tp,
			double> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 32;
  }

template <typename _Tp, size_t _Np>
  constexpr bool
  __is_avx512_ps()
  {
    return __have_avx512f
	   && is_same_v<_Tp,
			float> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 64;
  }

template <typename _Tp, size_t _Np>
  constexpr bool
  __is_avx512_pd()
  {
    return __have_avx512f
	   && is_same_v<_Tp,
			double> && sizeof(__intrinsic_type_t<_Tp, _Np>) == 64;
  }

// }}}
struct _MaskImplX86Mixin;

// _CommonImplX86 {{{
struct _CommonImplX86 : _CommonImplBuiltin
{
#ifdef _GLIBCXX_SIMD_WORKAROUND_PR85048
  // _S_converts_via_decomposition {{{
  template <typename _From, typename _To, size_t _ToSize>
    static constexpr bool _S_converts_via_decomposition()
    {
      if constexpr (is_integral_v<
		      _From> && is_integral_v<_To> && sizeof(_From) == 8
		    && _ToSize == 16)
	return (sizeof(_To) == 2 && !__have_ssse3)
	       || (sizeof(_To) == 1 && !__have_avx512f);
      else if constexpr (is_floating_point_v<_From> && is_integral_v<_To>)
	return ((sizeof(_From) == 4 || sizeof(_From) == 8) && sizeof(_To) == 8
		&& !__have_avx512dq)
	       || (sizeof(_From) == 8 && sizeof(_To) == 4 && !__have_sse4_1
		   && _ToSize == 16);
      else if constexpr (
	is_integral_v<_From> && is_floating_point_v<_To> && sizeof(_From) == 8
	&& !__have_avx512dq)
	return (sizeof(_To) == 4 && _ToSize == 16)
	       || (sizeof(_To) == 8 && _ToSize < 64);
      else
	return false;
    }

  template <typename _From, typename _To, size_t _ToSize>
    static inline constexpr bool __converts_via_decomposition_v
      = _S_converts_via_decomposition<_From, _To, _ToSize>();

  // }}}
#endif
  // _S_store {{{
  using _CommonImplBuiltin::_S_store;

  template <typename _Tp, size_t _Np>
    _GLIBCXX_SIMD_INTRINSIC static void _S_store(_SimdWrapper<_Tp, _Np> __x,
						 void* __addr)
    {
      constexpr size_t _Bytes = _Np * sizeof(_Tp);

      if constexpr ((_Bytes & (_Bytes - 1)) != 0 && __have_avx512bw_vl)
	{
	  const auto __v = __to_intrin(__x);

	  if constexpr (_Bytes & 1)
	    {
	      if constexpr (_Bytes < 16)
		_mm_mask_storeu_epi8(__addr, 0xffffu >> (16 - _Bytes),
				     __intrin_bitcast<__m128i>(__v));
	      else if constexpr (_Bytes < 32)
		_mm256_mask_storeu_epi8(__addr, 0xffffffffu >> (32 - _Bytes),
					__intrin_bitcast<__m256i>(__v));
	      else
		_mm512_mask_storeu_epi8(__addr,
					0xffffffffffffffffull >> (64 - _Bytes),
					__intrin_bitcast<__m512i>(__v));
	    }
	  else if constexpr (_Bytes & 2)
	    {
	      if constexpr (_Bytes < 16)
		_mm_mask_storeu_epi16(__addr, 0xffu >> (8 - _Bytes / 2),
				      __intrin_bitcast<__m128i>(__v));
	      else if constexpr (_Bytes < 32)
		_mm256_mask_storeu_epi16(__addr, 0xffffu >> (16 - _Bytes / 2),
					 __intrin_bitcast<__m256i>(__v));
	      else
		_mm512_mask_storeu_epi16(__addr,
					 0xffffffffull >> (32 - _Bytes / 2),
					 __intrin_bitcast<__m512i>(__v));
	    }
	  else if constexpr (_Bytes & 4)
	    {
	      if constexpr (_Bytes < 16)
		_mm_mask_storeu_epi32(__addr, 0xfu >> (4 - _Bytes / 4),
				      __intrin_bitcast<__m128i>(__v));
	      else if constexpr (_Bytes < 32)
		_mm256_mask_storeu_epi32(__addr, 0xffu >> (8 - _Bytes / 4),
					 __intrin_bitcast<__m256i>(__v));
	      else
		_mm512_mask_storeu_epi32(__addr, 0xffffull >> (16 - _Bytes / 4),
					 __intrin_bitcast<__m512i>(__v));
	    }
	  else
	    {
	      static_assert(
		_Bytes > 16,
		"_Bytes < 16 && (_Bytes & 7) == 0 && (_Bytes & (_Bytes "
		"- 1)) != 0 is impossible");
	      if constexpr (_Bytes < 32)
		_mm256_mask_storeu_epi64(__addr, 0xfu >> (4 - _Bytes / 8),
					 __intrin_bitcast<__m256i>(__v));
	      else
		_mm512_mask_storeu_epi64(__addr, 0xffull >> (8 - _Bytes / 8),
					 __intrin_bitcast<__m512i>(__v));
	    }
	}
      else
	_CommonImplBuiltin::_S_store(__x, __addr);
    }

  // }}}
  // _S_store_bool_array(_BitMask) {{{
  template <size_t _Np, bool _Sanitized>
    _GLIBCXX_SIMD_INTRINSIC static constexpr void
    _S_store_bool_array(const _BitMask<_Np, _Sanitized> __x, bool* __mem)
    {
      if constexpr (__have_avx512bw_vl) // don't care for BW w/o VL
	_S_store<_Np>(1 & __vector_bitcast<_UChar, _Np>([=]() constexpr {
			if constexpr (_Np <= 16)
			  return _mm_movm_epi8(__x._M_to_bits());
			else if constexpr (_Np <= 32)
			  return _mm256_movm_epi8(__x._M_to_bits());
			else if constexpr (_Np <= 64)
			  return _mm512_movm_epi8(__x._M_to_bits());
			else
			  __assert_unreachable<_SizeConstant<_Np>>();
		      }()),
		      __mem);
      else if constexpr (__have_bmi2)
	{
	  if constexpr (_Np <= 4)
	    _S_store<_Np>(_pdep_u32(__x._M_to_bits(), 0x01010101U), __mem);
	  else
	    __execute_n_times<__div_roundup(_Np, sizeof(size_t))>(
	      [&](auto __i) {
		constexpr size_t __offset = __i * sizeof(size_t);
		constexpr int __todo = std::min(sizeof(size_t), _Np - __offset);
		if constexpr (__todo == 1)
		  __mem[__offset] = __x[__offset];
		else
		  {
		    const auto __bools =
#ifdef __x86_64__
		      _pdep_u64(__x.template _M_extract<__offset>().to_ullong(),
				0x0101010101010101ULL);
#else // __x86_64__
		      _pdep_u32(
			__x.template _M_extract<__offset>()._M_to_bits(),
			0x01010101U);
#endif // __x86_64__
		    _S_store<__todo>(__bools, __mem + __offset);
		  }
	      });
	}
      else if constexpr (__have_sse2 && _Np > 7)
	__execute_n_times<__div_roundup(_Np, 16)>([&](auto __i) {
	  constexpr int __offset = __i * 16;
	  constexpr int __todo = std::min(16, int(_Np) - __offset);
	  const int __bits = __x.template _M_extract<__offset>()._M_to_bits();
	  __vector_type16_t<_UChar> __bools;
	  if constexpr (__have_avx512f)
	    {
	      auto __as32bits
		= _mm512_maskz_mov_epi32(__bits, __to_intrin(
						   __vector_broadcast<16>(1)));
	      auto __as16bits
		= __xzyw(_mm256_packs_epi32(__lo256(__as32bits),
					    __todo > 8 ? __hi256(__as32bits)
						       : __m256i()));
	      __bools = __vector_bitcast<_UChar>(
		_mm_packs_epi16(__lo128(__as16bits), __hi128(__as16bits)));
	    }
	  else
	    {
	      using _V = __vector_type_t<_UChar, 16>;
	      auto __tmp = _mm_cvtsi32_si128(__bits);
	      __tmp = _mm_unpacklo_epi8(__tmp, __tmp);
	      __tmp = _mm_unpacklo_epi16(__tmp, __tmp);
	      __tmp = _mm_unpacklo_epi32(__tmp, __tmp);
	      _V __tmp2 = reinterpret_cast<_V>(__tmp);
	      __tmp2 &= _V{1, 2, 4, 8, 16, 32, 64, 128,
			   1, 2, 4, 8, 16, 32, 64, 128}; // mask bit index
	      __bools = (__tmp2 == 0) + 1; // 0xff -> 0x00 | 0x00 -> 0x01
	    }
	  _S_store<__todo>(__bools, __mem + __offset);
	});
      else
	_CommonImplBuiltin::_S_store_bool_array(__x, __mem);
    }

  // }}}
  // _S_blend_avx512 {{{
  // Returns: __k ? __b : __a
  // TODO: reverse __a and __b to match COND_EXPR
  // Requires: _TV to be a __vector_type_t matching valuetype for the bitmask
  //           __k
  template <typename _Kp, typename _TV>
    _GLIBCXX_SIMD_INTRINSIC static _TV
    _S_blend_avx512(const _Kp __k, const _TV __a, const _TV __b) noexcept
    {
#ifdef __clang__
      // FIXME: this does a boolean choice, not a blend
      return __k ? __a : __b;
#else
      static_assert(__is_vector_type_v<_TV>);
      using _Tp = typename _VectorTraits<_TV>::value_type;
      static_assert(sizeof(_TV) >= 16);
      static_assert(sizeof(_Tp) <= 8);
      using _IntT
	= conditional_t<(sizeof(_Tp) > 2),
			conditional_t<sizeof(_Tp) == 4, int, long long>,
			conditional_t<sizeof(_Tp) == 1, char, short>>;
      [[maybe_unused]] const auto __aa = __vector_bitcast<_IntT>(__a);
      [[maybe_unused]] const auto __bb = __vector_bitcast<_IntT>(__b);
      if constexpr (sizeof(_TV) == 64)
	{
	  if constexpr (sizeof(_Tp) == 1)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmb_512_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 2)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmw_512_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 4 && is_floating_point_v<_Tp>)
	    return __builtin_ia32_blendmps_512_mask(__a, __b, __k);
	  else if constexpr (sizeof(_Tp) == 4)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmd_512_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 8 && is_floating_point_v<_Tp>)
	    return __builtin_ia32_blendmpd_512_mask(__a, __b, __k);
	  else if constexpr (sizeof(_Tp) == 8)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmq_512_mask(__aa, __bb, __k));
	}
      else if constexpr (sizeof(_TV) == 32)
	{
	  if constexpr (sizeof(_Tp) == 1)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmb_256_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 2)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmw_256_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 4 && is_floating_point_v<_Tp>)
	    return __builtin_ia32_blendmps_256_mask(__a, __b, __k);
	  else if constexpr (sizeof(_Tp) == 4)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmd_256_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 8 && is_floating_point_v<_Tp>)
	    return __builtin_ia32_blendmpd_256_mask(__a, __b, __k);
	  else if constexpr (sizeof(_Tp) == 8)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmq_256_mask(__aa, __bb, __k));
	}
      else if constexpr (sizeof(_TV) == 16)
	{
	  if constexpr (sizeof(_Tp) == 1)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmb_128_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 2)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmw_128_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 4 && is_floating_point_v<_Tp>)
	    return __builtin_ia32_blendmps_128_mask(__a, __b, __k);
	  else if constexpr (sizeof(_Tp) == 4)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmd_128_mask(__aa, __bb, __k));
	  else if constexpr (sizeof(_Tp) == 8 && is_floating_point_v<_Tp>)
	    return __builtin_ia32_blendmpd_128_mask(__a, __b, __k);
	  else if constexpr (sizeof(_Tp) == 8)
	    return reinterpret_cast<_TV>(
	      __builtin_ia32_blendmq_128_mask(__aa, __bb, __k));
	}
#endif
    }

  // }}}
  // _S_blend_intrin {{{
  // Returns: __k ? __b : __a
  // TODO: reverse __a and __b to match COND_EXPR
  // Requires: _Tp to be an intrinsic type (integers blend per byte) and 16/32
  //           Bytes wide
  template <typename _Tp>
    _GLIBCXX_SIMD_INTRINSIC static _Tp _S_blend_intrin(_Tp __k, _Tp __a,
						       _Tp __b) noexcept
    {
      static_assert(is_same_v<decltype(__to_intrin(__a)), _Tp>);
      constexpr struct
      {
	_GLIBCXX_SIMD_INTRINSIC __m128 operator()(__m128 __a, __m128 __b,
						  __m128 __k) const noexcept
	{
	  return __builtin_ia32_blendvps(__a, __b, __k);
	}
	_GLIBCXX_SIMD_INTRINSIC __m128d operator()(__m128d __a, __m128d __b,
						   __m128d __k) const noexcept
	{
	  return __builtin_ia32_blendvpd(__a, __b, __k);
	}
	_GLIBCXX_SIMD_INTRINSIC __m128i operator()(__m128i __a, __m128i __b,
						   __m128i __k) const noexcept
	{
	  return reinterpret_cast<__m128i>(
	    __builtin_ia32_pblendvb128(reinterpret_cast<__v16qi>(__a),
				       reinterpret_cast<__v16qi>(__b),
				       reinterpret_cast<__v16qi>(__k)));
	}
	_GLIBCXX_SIMD_INTRINSIC __m256 operator()(__m256 __a, __m256 __b,
						  __m256 __k) const noexcept
	{
	  return __builtin_ia32_blendvps256(__a, __b, __k);
	}
	_GLIBCXX_SIMD_INTRINSIC __m256d operator()(__m256d __a, __m256d __b,
						   __m256d __k) const noexcept
	{
	  return __builtin_ia32_blendvpd256(__a, __b, __k);
	}
	_GLIBCXX_SIMD_INTRINSIC __m256i operator()(__m256i __a, __m256i __b,
						   __m256i __k) const noexcept
	{
	  if constexpr (__have_avx2)
	    return reinterpret_cast<__m256i>(
	      __builtin_ia32_pblendvb256(reinterpret_cast<__v32qi>(__a),
					 reinterpret_cast<__v32qi>(__b),
					 reinterpret_cast<__v32qi>(__k)));
	  else
	    return reinterpret_cast<__m256i>(
	      __builtin_ia32_blendvps256(reinterpret_cast<__v8sf>(__a),
					 reinterpret_cast<__v8sf>(__b),
					 reinterpret_cast<__v8sf>(__k)));
	}
      } __eval;
      return __eval(__a, __b, __k);
    }

  // }}}
  // _S_blend {{{
  // Returns: __k ? __at1 : __at0
  // TODO: reverse __at0 and __at1 to match COND_EXPR
  template <typename _Tp, size_t _Np>
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
    _S_blend(_SimdWrapper<bool, _Np> __k, _SimdWrapper<_Tp, _Np> __at0,
	     _SimdWrapper<_Tp, _Np> __at1)
    {
      static_assert(is_same_v<_Tp, _Tp> && __have_avx512f);
      if (__k._M_is_constprop() && __at0._M_is_constprop()
	  && __at1._M_is_constprop())
	return __generate_from_n_evaluations<_Np,
					     __vector_type_t<_Tp, _Np>>([&](
	  auto __i) constexpr { return __k[__i] ? __at1[__i] : __at0[__i]; });
      else if constexpr (sizeof(__at0) == 64
			 || (__have_avx512vl && sizeof(__at0) >= 16))
	return _S_blend_avx512(__k._M_data, __at0._M_data, __at1._M_data);
      else
	{
	  static_assert((__have_avx512vl && sizeof(__at0) < 16)
			|| !__have_avx512vl);
	  constexpr size_t __size = (__have_avx512vl ? 16 : 64) / sizeof(_Tp);
	  return __vector_bitcast<_Tp, _Np>(
	    _S_blend_avx512(__k._M_data, __vector_bitcast<_Tp, __size>(__at0),
			    __vector_bitcast<_Tp, __size>(__at1)));
	}
    }

  template <typename _Tp, size_t _Np>
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
    _S_blend(_SimdWrapper<__int_for_sizeof_t<_Tp>, _Np> __k,
	     _SimdWrapper<_Tp, _Np> __at0, _SimdWrapper<_Tp, _Np> __at1)
    {
      const auto __kk = __wrapper_bitcast<_Tp>(__k);
      if (__builtin_is_constant_evaluated()
	  || (__kk._M_is_constprop() && __at0._M_is_constprop()
	      && __at1._M_is_constprop()))
	{
	  auto __r = __or(__andnot(__kk, __at0), __and(__kk, __at1));
	  if (__r._M_is_constprop())
	    return __r;
	}
      if constexpr (((__have_avx512f && sizeof(__at0) == 64) || __have_avx512vl)
		    && (sizeof(_Tp) >= 4 || __have_avx512bw))
	// convert to bitmask and call overload above
	return _S_blend(
	  _SimdWrapper<bool, _Np>(
	    __make_dependent_t<_Tp, _MaskImplX86Mixin>::_S_to_bits(__k)
	      ._M_to_bits()),
	  __at0, __at1);
      else
	{
	  // Since GCC does not assume __k to be a mask, using the builtin
	  // conditional operator introduces an extra compare against 0 before
	  // blending. So we rather call the intrinsic here.
	  if constexpr (__have_sse4_1)
	    return _S_blend_intrin(__to_intrin(__kk), __to_intrin(__at0),
				   __to_intrin(__at1));
	  else
	    return __or(__andnot(__kk, __at0), __and(__kk, __at1));
	}
    }

  // }}}
};

// }}}
// _SimdImplX86 {{{
template <typename _Abi, typename>
  struct _SimdImplX86 : _SimdImplBuiltin<_Abi>
  {
    using _Base = _SimdImplBuiltin<_Abi>;

    template <typename _Tp>
      using _MaskMember = typename _Base::template _MaskMember<_Tp>;

    template <typename _Tp>
      static constexpr size_t _S_full_size = _Abi::template _S_full_size<_Tp>;

    template <typename _Tp>
      static constexpr size_t _S_size = _Abi::template _S_size<_Tp>;

    template <typename _Tp>
      static constexpr size_t _S_max_store_size
	= (sizeof(_Tp) >= 4 && __have_avx512f) || __have_avx512bw  ? 64
	  : (is_floating_point_v<_Tp>&& __have_avx) || __have_avx2 ? 32
								   : 16;
    using _MaskImpl = typename _Abi::_MaskImpl;

    // _S_masked_load {{{
    template <typename _Tp, size_t _Np, typename _Up>
      static inline _SimdWrapper<_Tp, _Np>
      _S_masked_load(_SimdWrapper<_Tp, _Np> __merge, _MaskMember<_Tp> __k,
		     const _Up* __mem) noexcept
      {
	static_assert(_Np == _S_size<_Tp>);
	if constexpr (is_same_v<_Tp, _Up> || // no conversion
		      (sizeof(_Tp) == sizeof(_Up)
		       && is_integral_v<
			    _Tp> == is_integral_v<_Up>) // conversion via bit
							// reinterpretation
	)
	  {
	    [[maybe_unused]] const auto __intrin = __to_intrin(__merge);
	    if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512bw_vl)
			  && sizeof(_Tp) == 1)
	      {
		const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
		if constexpr (sizeof(__intrin) == 16)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm_mask_loadu_epi8(__intrin, __kk, __mem));
		else if constexpr (sizeof(__merge) == 32)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm256_mask_loadu_epi8(__intrin, __kk, __mem));
		else if constexpr (sizeof(__merge) == 64)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm512_mask_loadu_epi8(__intrin, __kk, __mem));
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512bw_vl)
			       && sizeof(_Tp) == 2)
	      {
		const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
		if constexpr (sizeof(__intrin) == 16)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm_mask_loadu_epi16(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 32)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm256_mask_loadu_epi16(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 64)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm512_mask_loadu_epi16(__intrin, __kk, __mem));
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
			       && sizeof(_Tp) == 4 && is_integral_v<_Up>)
	      {
		const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
		if constexpr (sizeof(__intrin) == 16)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm_mask_loadu_epi32(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 32)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm256_mask_loadu_epi32(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 64)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm512_mask_loadu_epi32(__intrin, __kk, __mem));
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
			       && sizeof(_Tp) == 4 && is_floating_point_v<_Up>)
	      {
		const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
		if constexpr (sizeof(__intrin) == 16)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm_mask_loadu_ps(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 32)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm256_mask_loadu_ps(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 64)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm512_mask_loadu_ps(__intrin, __kk, __mem));
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (__have_avx2 && sizeof(_Tp) == 4
			       && is_integral_v<_Up>)
	      {
		static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
		__merge
		  = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
			 __vector_bitcast<_Tp, _Np>(
			   __maskload_epi32(reinterpret_cast<const int*>(__mem),
					    __to_intrin(__k))));
	      }
	    else if constexpr (__have_avx && sizeof(_Tp) == 4)
	      {
		static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
		__merge
		  = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
			 __vector_bitcast<_Tp, _Np>(
			   __maskload_ps(reinterpret_cast<const float*>(__mem),
					 __to_intrin(__k))));
	      }
	    else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
			       && sizeof(_Tp) == 8 && is_integral_v<_Up>)
	      {
		const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
		if constexpr (sizeof(__intrin) == 16)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm_mask_loadu_epi64(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 32)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm256_mask_loadu_epi64(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 64)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm512_mask_loadu_epi64(__intrin, __kk, __mem));
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr ((__is_avx512_abi<_Abi>() || __have_avx512vl)
			       && sizeof(_Tp) == 8 && is_floating_point_v<_Up>)
	      {
		const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
		if constexpr (sizeof(__intrin) == 16)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm_mask_loadu_pd(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 32)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm256_mask_loadu_pd(__intrin, __kk, __mem));
		else if constexpr (sizeof(__intrin) == 64)
		  __merge = __vector_bitcast<_Tp, _Np>(
		    _mm512_mask_loadu_pd(__intrin, __kk, __mem));
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (__have_avx2 && sizeof(_Tp) == 8
			       && is_integral_v<_Up>)
	      {
		static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
		__merge
		  = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
			 __vector_bitcast<_Tp, _Np>(__maskload_epi64(
			   reinterpret_cast<const _LLong*>(__mem),
			   __to_intrin(__k))));
	      }
	    else if constexpr (__have_avx && sizeof(_Tp) == 8)
	      {
		static_assert(sizeof(__intrin) == 16 || sizeof(__intrin) == 32);
		__merge
		  = __or(__andnot(__vector_bitcast<_Tp>(__k), __merge._M_data),
			 __vector_bitcast<_Tp, _Np>(
			   __maskload_pd(reinterpret_cast<const double*>(__mem),
					 __to_intrin(__k))));
	      }
	    else
	      _BitOps::_S_bit_iteration(_MaskImpl::_S_to_bits(__k),
					[&](auto __i) {
					  __merge._M_set(__i, static_cast<_Tp>(
								__mem[__i]));
					});
	  }
	/* Very uncertain, that the following improves anything. Needs
	benchmarking
	 * before it's activated.
	else if constexpr (sizeof(_Up) <= 8 && // no long double
			   !__converts_via_decomposition_v<
			     _Up, _Tp,
			     sizeof(__merge)> // conversion via decomposition
					      // is better handled via the
					      // bit_iteration fallback below
	)
	  {
	    // TODO: copy pattern from _S_masked_store, which doesn't resort to
	    // fixed_size
	    using _Ap       = simd_abi::deduce_t<_Up, _Np>;
	    using _ATraits = _SimdTraits<_Up, _Ap>;
	    using _AImpl   = typename _ATraits::_SimdImpl;
	    typename _ATraits::_SimdMember __uncvted{};
	    typename _ATraits::_MaskMember __kk = _Ap::_MaskImpl::template
	_S_convert<_Up>(__k);
	    __uncvted = _AImpl::_S_masked_load(__uncvted, __kk, __mem);
	    _SimdConverter<_Up, _Ap, _Tp, _Abi> __converter;
	    _Base::_S_masked_assign(__k, __merge, __converter(__uncvted));
	  }
	  */
	else
	  __merge = _Base::_S_masked_load(__merge, __k, __mem);
	return __merge;
      }

    // }}}
    // _S_masked_store_nocvt {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem,
			    _SimdWrapper<bool, _Np> __k)
      {
	[[maybe_unused]] const auto __vi = __to_intrin(__v);
	if constexpr (sizeof(__vi) == 64)
	  {
	    static_assert(sizeof(__v) == 64 && __have_avx512f);
	    if constexpr (__have_avx512bw && sizeof(_Tp) == 1)
	      _mm512_mask_storeu_epi8(__mem, __k, __vi);
	    else if constexpr (__have_avx512bw && sizeof(_Tp) == 2)
	      _mm512_mask_storeu_epi16(__mem, __k, __vi);
	    else if constexpr (__have_avx512f && sizeof(_Tp) == 4)
	      {
		if constexpr (is_integral_v<_Tp>)
		  _mm512_mask_storeu_epi32(__mem, __k, __vi);
		else
		  _mm512_mask_storeu_ps(__mem, __k, __vi);
	      }
	    else if constexpr (__have_avx512f && sizeof(_Tp) == 8)
	      {
		if constexpr (is_integral_v<_Tp>)
		  _mm512_mask_storeu_epi64(__mem, __k, __vi);
		else
		  _mm512_mask_storeu_pd(__mem, __k, __vi);
	      }
#if 0 // with KNL either sizeof(_Tp) >= 4 or sizeof(_vi) <= 32
      // with Skylake-AVX512, __have_avx512bw is true
	  else if constexpr (__have_sse2)
	    {
	      using _M   = __vector_type_t<_Tp, _Np>;
	      using _MVT = _VectorTraits<_M>;
	      _mm_maskmoveu_si128(__auto_bitcast(__extract<0, 4>(__v._M_data)),
				  __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(__k._M_data)),
				  reinterpret_cast<char*>(__mem));
	      _mm_maskmoveu_si128(__auto_bitcast(__extract<1, 4>(__v._M_data)),
				  __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(
				    __k._M_data >> 1 * _MVT::_S_full_size)),
				  reinterpret_cast<char*>(__mem) + 1 * 16);
	      _mm_maskmoveu_si128(__auto_bitcast(__extract<2, 4>(__v._M_data)),
				  __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(
				    __k._M_data >> 2 * _MVT::_S_full_size)),
				  reinterpret_cast<char*>(__mem) + 2 * 16);
	      if constexpr (_Np > 48 / sizeof(_Tp))
		_mm_maskmoveu_si128(
		  __auto_bitcast(__extract<3, 4>(__v._M_data)),
		  __auto_bitcast(_MaskImpl::template _S_convert<_Tp, _Np>(
		    __k._M_data >> 3 * _MVT::_S_full_size)),
		  reinterpret_cast<char*>(__mem) + 3 * 16);
	    }
#endif
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (sizeof(__vi) == 32)
	  {
	    if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
	      _mm256_mask_storeu_epi8(__mem, __k, __vi);
	    else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
	      _mm256_mask_storeu_epi16(__mem, __k, __vi);
	    else if constexpr (__have_avx512vl && sizeof(_Tp) == 4)
	      {
		if constexpr (is_integral_v<_Tp>)
		  _mm256_mask_storeu_epi32(__mem, __k, __vi);
		else
		  _mm256_mask_storeu_ps(__mem, __k, __vi);
	      }
	    else if constexpr (__have_avx512vl && sizeof(_Tp) == 8)
	      {
		if constexpr (is_integral_v<_Tp>)
		  _mm256_mask_storeu_epi64(__mem, __k, __vi);
		else
		  _mm256_mask_storeu_pd(__mem, __k, __vi);
	      }
	    else if constexpr (__have_avx512f
			       && (sizeof(_Tp) >= 4 || __have_avx512bw))
	      {
		// use a 512-bit maskstore, using zero-extension of the bitmask
		_S_masked_store_nocvt(
		  _SimdWrapper64<_Tp>(
		    __intrin_bitcast<__vector_type64_t<_Tp>>(__v._M_data)),
		  __mem, _SimdWrapper<bool, 64 / sizeof(_Tp)>(__k._M_data));
	      }
	    else
	      _S_masked_store_nocvt(__v, __mem,
				    _MaskImpl::template _S_to_maskvector<
				      __int_for_sizeof_t<_Tp>, _Np>(__k));
	  }
	else if constexpr (sizeof(__vi) == 16)
	  {
	    if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
	      _mm_mask_storeu_epi8(__mem, __k, __vi);
	    else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
	      _mm_mask_storeu_epi16(__mem, __k, __vi);
	    else if constexpr (__have_avx512vl && sizeof(_Tp) == 4)
	      {
		if constexpr (is_integral_v<_Tp>)
		  _mm_mask_storeu_epi32(__mem, __k, __vi);
		else
		  _mm_mask_storeu_ps(__mem, __k, __vi);
	      }
	    else if constexpr (__have_avx512vl && sizeof(_Tp) == 8)
	      {
		if constexpr (is_integral_v<_Tp>)
		  _mm_mask_storeu_epi64(__mem, __k, __vi);
		else
		  _mm_mask_storeu_pd(__mem, __k, __vi);
	      }
	    else if constexpr (__have_avx512f
			       && (sizeof(_Tp) >= 4 || __have_avx512bw))
	      {
		// use a 512-bit maskstore, using zero-extension of the bitmask
		_S_masked_store_nocvt(
		  _SimdWrapper64<_Tp>(
		    __intrin_bitcast<__intrinsic_type64_t<_Tp>>(__v._M_data)),
		  __mem, _SimdWrapper<bool, 64 / sizeof(_Tp)>(__k._M_data));
	      }
	    else
	      _S_masked_store_nocvt(__v, __mem,
				    _MaskImpl::template _S_to_maskvector<
				      __int_for_sizeof_t<_Tp>, _Np>(__k));
	  }
	else
	  __assert_unreachable<_Tp>();
      }

    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem,
			    _SimdWrapper<__int_for_sizeof_t<_Tp>, _Np> __k)
      {
	if constexpr (sizeof(__v) <= 16)
	  {
	    [[maybe_unused]] const auto __vi
	      = __intrin_bitcast<__m128i>(__as_vector(__v));
	    [[maybe_unused]] const auto __ki
	      = __intrin_bitcast<__m128i>(__as_vector(__k));
	    if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
	      _mm_mask_storeu_epi8(__mem, _mm_movepi8_mask(__ki), __vi);
	    else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
	      _mm_mask_storeu_epi16(__mem, _mm_movepi16_mask(__ki), __vi);
	    else if constexpr (__have_avx2 && sizeof(_Tp) == 4
			       && is_integral_v<_Tp>)
	      _mm_maskstore_epi32(reinterpret_cast<int*>(__mem), __ki, __vi);
	    else if constexpr (__have_avx && sizeof(_Tp) == 4)
	      _mm_maskstore_ps(reinterpret_cast<float*>(__mem), __ki,
			       __vector_bitcast<float>(__vi));
	    else if constexpr (__have_avx2 && sizeof(_Tp) == 8
			       && is_integral_v<_Tp>)
	      _mm_maskstore_epi64(reinterpret_cast<_LLong*>(__mem), __ki, __vi);
	    else if constexpr (__have_avx && sizeof(_Tp) == 8)
	      _mm_maskstore_pd(reinterpret_cast<double*>(__mem), __ki,
			       __vector_bitcast<double>(__vi));
	    else if constexpr (__have_sse2)
	      _mm_maskmoveu_si128(__vi, __ki, reinterpret_cast<char*>(__mem));
	  }
	else if constexpr (sizeof(__v) == 32)
	  {
	    [[maybe_unused]] const auto __vi
	      = __intrin_bitcast<__m256i>(__as_vector(__v));
	    [[maybe_unused]] const auto __ki
	      = __intrin_bitcast<__m256i>(__as_vector(__k));
	    if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 1)
	      _mm256_mask_storeu_epi8(__mem, _mm256_movepi8_mask(__ki), __vi);
	    else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 2)
	      _mm256_mask_storeu_epi16(__mem, _mm256_movepi16_mask(__ki), __vi);
	    else if constexpr (__have_avx2 && sizeof(_Tp) == 4
			       && is_integral_v<_Tp>)
	      _mm256_maskstore_epi32(reinterpret_cast<int*>(__mem), __ki, __vi);
	    else if constexpr (sizeof(_Tp) == 4)
	      _mm256_maskstore_ps(reinterpret_cast<float*>(__mem), __ki,
				  __vector_bitcast<float>(__v));
	    else if constexpr (__have_avx2 && sizeof(_Tp) == 8
			       && is_integral_v<_Tp>)
	      _mm256_maskstore_epi64(reinterpret_cast<_LLong*>(__mem), __ki,
				     __vi);
	    else if constexpr (__have_avx && sizeof(_Tp) == 8)
	      _mm256_maskstore_pd(reinterpret_cast<double*>(__mem), __ki,
				  __vector_bitcast<double>(__v));
	    else if constexpr (__have_sse2)
	      {
		_mm_maskmoveu_si128(__lo128(__vi), __lo128(__ki),
				    reinterpret_cast<char*>(__mem));
		_mm_maskmoveu_si128(__hi128(__vi), __hi128(__ki),
				    reinterpret_cast<char*>(__mem) + 16);
	      }
	  }
	else
	  __assert_unreachable<_Tp>();
      }

    // }}}
    // _S_masked_store {{{
    template <typename _Tp, size_t _Np, typename _Up>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_store(const _SimdWrapper<_Tp, _Np> __v, _Up* __mem,
		      const _MaskMember<_Tp> __k) noexcept
      {
	if constexpr (is_integral_v<
			_Tp> && is_integral_v<_Up> && sizeof(_Tp) > sizeof(_Up)
		      && __have_avx512f && (sizeof(_Tp) >= 4 || __have_avx512bw)
		      && (sizeof(__v) == 64 || __have_avx512vl))
	  { // truncating store
	    const auto __vi = __to_intrin(__v);
	    const auto __kk = _MaskImpl::_S_to_bits(__k)._M_to_bits();
	    if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4
			  && sizeof(__vi) == 64)
	      _mm512_mask_cvtepi64_storeu_epi32(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4
			       && sizeof(__vi) == 32)
	      _mm256_mask_cvtepi64_storeu_epi32(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 4
			       && sizeof(__vi) == 16)
	      _mm_mask_cvtepi64_storeu_epi32(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2
			       && sizeof(__vi) == 64)
	      _mm512_mask_cvtepi64_storeu_epi16(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2
			       && sizeof(__vi) == 32)
	      _mm256_mask_cvtepi64_storeu_epi16(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 2
			       && sizeof(__vi) == 16)
	      _mm_mask_cvtepi64_storeu_epi16(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1
			       && sizeof(__vi) == 64)
	      _mm512_mask_cvtepi64_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1
			       && sizeof(__vi) == 32)
	      _mm256_mask_cvtepi64_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(_Up) == 1
			       && sizeof(__vi) == 16)
	      _mm_mask_cvtepi64_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2
			       && sizeof(__vi) == 64)
	      _mm512_mask_cvtepi32_storeu_epi16(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2
			       && sizeof(__vi) == 32)
	      _mm256_mask_cvtepi32_storeu_epi16(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 2
			       && sizeof(__vi) == 16)
	      _mm_mask_cvtepi32_storeu_epi16(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1
			       && sizeof(__vi) == 64)
	      _mm512_mask_cvtepi32_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1
			       && sizeof(__vi) == 32)
	      _mm256_mask_cvtepi32_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 4 && sizeof(_Up) == 1
			       && sizeof(__vi) == 16)
	      _mm_mask_cvtepi32_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1
			       && sizeof(__vi) == 64)
	      _mm512_mask_cvtepi16_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1
			       && sizeof(__vi) == 32)
	      _mm256_mask_cvtepi16_storeu_epi8(__mem, __kk, __vi);
	    else if constexpr (sizeof(_Tp) == 2 && sizeof(_Up) == 1
			       && sizeof(__vi) == 16)
	      _mm_mask_cvtepi16_storeu_epi8(__mem, __kk, __vi);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  _Base::_S_masked_store(__v, __mem, __k);
      }

    // }}}
    // _S_multiplies {{{
    template <typename _V, typename _VVT = _VectorTraits<_V>>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _V _S_multiplies(_V __x, _V __y)
      {
	using _Tp = typename _VVT::value_type;
	if (__builtin_is_constant_evaluated() || __x._M_is_constprop()
	    || __y._M_is_constprop())
	  return __as_vector(__x) * __as_vector(__y);
	else if constexpr (sizeof(_Tp) == 1)
	  {
	    if constexpr (sizeof(_V) == 2)
	      {
		const auto __xs = reinterpret_cast<short>(__x._M_data);
		const auto __ys = reinterpret_cast<short>(__y._M_data);
		return reinterpret_cast<__vector_type_t<_Tp, 2>>(short(
		  ((__xs * __ys) & 0xff) | ((__xs >> 8) * (__ys & 0xff00))));
	      }
	    else if constexpr (sizeof(_V) == 4 && _VVT::_S_partial_width == 3)
	      {
		const auto __xi = reinterpret_cast<int>(__x._M_data);
		const auto __yi = reinterpret_cast<int>(__y._M_data);
		return reinterpret_cast<__vector_type_t<_Tp, 3>>(
		  ((__xi * __yi) & 0xff)
		  | (((__xi >> 8) * (__yi & 0xff00)) & 0xff00)
		  | ((__xi >> 16) * (__yi & 0xff0000)));
	      }
	    else if constexpr (sizeof(_V) == 4)
	      {
		const auto __xi = reinterpret_cast<int>(__x._M_data);
		const auto __yi = reinterpret_cast<int>(__y._M_data);
		return reinterpret_cast<__vector_type_t<_Tp, 4>>(
		  ((__xi * __yi) & 0xff)
		  | (((__xi >> 8) * (__yi & 0xff00)) & 0xff00)
		  | (((__xi >> 16) * (__yi & 0xff0000)) & 0xff0000)
		  | ((__xi >> 24) * (__yi & 0xff000000u)));
	      }
	    else if constexpr (sizeof(_V) == 8 && __have_avx2
			       && is_signed_v<_Tp>)
	      return __convert<typename _VVT::type>(
		__vector_bitcast<short>(_mm_cvtepi8_epi16(__to_intrin(__x)))
		* __vector_bitcast<short>(_mm_cvtepi8_epi16(__to_intrin(__y))));
	    else if constexpr (sizeof(_V) == 8 && __have_avx2
			       && is_unsigned_v<_Tp>)
	      return __convert<typename _VVT::type>(
		__vector_bitcast<short>(_mm_cvtepu8_epi16(__to_intrin(__x)))
		* __vector_bitcast<short>(_mm_cvtepu8_epi16(__to_intrin(__y))));
	    else
	      {
		// codegen of `x*y` is suboptimal (as of GCC 9.0.1)
		constexpr size_t __full_size = _VVT::_S_full_size;
		constexpr int _Np = sizeof(_V) >= 16 ? __full_size / 2 : 8;
		using _ShortW = _SimdWrapper<short, _Np>;
		const _ShortW __even = __vector_bitcast<short, _Np>(__x)
				       * __vector_bitcast<short, _Np>(__y);
		_ShortW __high_byte = _ShortW()._M_data - 256;
		//[&]() { asm("" : "+x"(__high_byte._M_data)); }();
		const _ShortW __odd
		  = (__vector_bitcast<short, _Np>(__x) >> 8)
		    * (__vector_bitcast<short, _Np>(__y) & __high_byte._M_data);
		if constexpr (__have_avx512bw && sizeof(_V) > 2)
		  return _CommonImplX86::_S_blend_avx512(
		    0xaaaa'aaaa'aaaa'aaaaLL, __vector_bitcast<_Tp>(__even),
		    __vector_bitcast<_Tp>(__odd));
		else if constexpr (__have_sse4_1 && sizeof(_V) > 2)
		  return _CommonImplX86::_S_blend_intrin(__to_intrin(
							   __high_byte),
							 __to_intrin(__even),
							 __to_intrin(__odd));
		else
		  return __to_intrin(
		    __or(__andnot(__high_byte, __even), __odd));
	      }
	  }
	else
	  return _Base::_S_multiplies(__x, __y);
      }

    // }}}
    // _S_divides {{{
#ifdef _GLIBCXX_SIMD_WORKAROUND_PR90993
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_divides(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	if (!__builtin_is_constant_evaluated()
	    && !__builtin_constant_p(__y._M_data))
	  if constexpr (is_integral_v<_Tp> && sizeof(_Tp) <= 4)
	    { // use divps - codegen of `x/y` is suboptimal (as of GCC 9.0.1)
	      // Note that using floating-point division is likely to raise the
	      // *Inexact* exception flag and thus appears like an invalid
	      // "as-if" transformation. However, C++ doesn't specify how the
	      // fpenv can be observed and points to C. C says that function
	      // calls are assumed to potentially raise fp exceptions, unless
	      // documented otherwise. Consequently, operator/, which is a
	      // function call, may raise fp exceptions.
	      /*const struct _CsrGuard
	      {
		const unsigned _M_data = _mm_getcsr();
		_CsrGuard()
		{
		  _mm_setcsr(0x9f80); // turn off FP exceptions and
	      flush-to-zero
		}
		~_CsrGuard() { _mm_setcsr(_M_data); }
	      } __csr;*/
	      using _Float = conditional_t<sizeof(_Tp) == 4, double, float>;
	      constexpr size_t __n_intermediate
		= std::min(_Np, (__have_avx512f ? 64
				 : __have_avx   ? 32
						: 16)
				  / sizeof(_Float));
	      using _FloatV = __vector_type_t<_Float, __n_intermediate>;
	      constexpr size_t __n_floatv
		= __div_roundup(_Np, __n_intermediate);
	      using _R = __vector_type_t<_Tp, _Np>;
	      const auto __xf = __convert_all<_FloatV, __n_floatv>(__x);
	      const auto __yf = __convert_all<_FloatV, __n_floatv>(
		_Abi::__make_padding_nonzero(__as_vector(__y)));
	      return __call_with_n_evaluations<__n_floatv>(
		[](auto... __quotients) {
		  return __vector_convert<_R>(__quotients...);
		},
		[&__xf,
		 &__yf](auto __i) -> _SimdWrapper<_Float, __n_intermediate> {
#if !defined __clang__ && __GCC_IEC_559 == 0
		  // If -freciprocal-math is active, using the `/` operator is
		  // incorrect because it may be translated to an imprecise
		  // multiplication with reciprocal. We need to use inline
		  // assembly to force a real division.
		  _FloatV __r;
		  if constexpr (__have_avx) // -mno-sse2avx is irrelevant
					    // because once -mavx is given, GCC
					    // emits VEX encoded vdivp[sd]
		    {
		      if constexpr (sizeof(_Tp) == 4)
			asm("vdivpd\t{%2, %1, %0|%0, %1, %2}"
			    : "=x"(__r)
			    : "x"(__xf[__i]), "x"(__yf[__i]));
		      else
			asm("vdivps\t{%2, %1, %0|%0, %1, %2}"
			    : "=x"(__r)
			    : "x"(__xf[__i]), "x"(__yf[__i]));
		    }
		  else
		    {
		      __r = __xf[__i];
		      if constexpr (sizeof(_Tp) == 4)
			asm("divpd\t{%1, %0|%0, %1}"
			    : "=x"(__r)
			    : "x"(__yf[__i]));
		      else
			asm("divps\t{%1, %0|%0, %1}"
			    : "=x"(__r)
			    : "x"(__yf[__i]));
		    }
		  return __r;
#else
		  return __xf[__i] / __yf[__i];
#endif
		});
	    }
	/* 64-bit int division is potentially optimizable via double division if
	 * the value in __x is small enough and the conversion between
	 * int<->double is efficient enough:
	else if constexpr (is_integral_v<_Tp> && is_unsigned_v<_Tp> &&
			   sizeof(_Tp) == 8)
	  {
	    if constexpr (__have_sse4_1 && sizeof(__x) == 16)
	      {
		if (_mm_test_all_zeros(__x, __m128i{0xffe0'0000'0000'0000ull,
						    0xffe0'0000'0000'0000ull}))
		  {
		    __x._M_data | 0x __vector_convert<__m128d>(__x._M_data)
		  }
	      }
	  }
	  */
	return _Base::_S_divides(__x, __y);
      }
  #endif // _GLIBCXX_SIMD_WORKAROUND_PR90993

    // }}}
    // _S_modulus {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_modulus(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	if (__builtin_is_constant_evaluated()
	    || __builtin_constant_p(__y._M_data) || sizeof(_Tp) >= 8)
	  return _Base::_S_modulus(__x, __y);
	else
	  return _Base::_S_minus(__x, _S_multiplies(__y, _S_divides(__x, __y)));
      }

    // }}}
    // _S_bit_shift_left {{{
    // Notes on UB. C++2a [expr.shift] says:
    // -1- [...] The operands shall be of integral or unscoped enumeration type
    //     and integral promotions are performed. The type of the result is that
    //     of the promoted left operand. The behavior is undefined if the right
    //     operand is negative, or greater than or equal to the width of the
    //     promoted left operand.
    // -2- The value of E1 << E2 is the unique value congruent to E1×2^E2 modulo
    //     2^N, where N is the width of the type of the result.
    //
    // C++17 [expr.shift] says:
    // -2- The value of E1 << E2 is E1 left-shifted E2 bit positions; vacated
    //     bits are zero-filled. If E1 has an unsigned type, the value of the
    //     result is E1 × 2^E2 , reduced modulo one more than the maximum value
    //     representable in the result type. Otherwise, if E1 has a signed type
    //     and non-negative value, and E1 × 2^E2 is representable in the
    //     corresponding unsigned type of the result type, then that value,
    //     converted to the result type, is the resulting value; otherwise, the
    //     behavior is undefined.
    //
    // Consequences:
    // With C++2a signed and unsigned types have the same UB
    // characteristics:
    // - left shift is not UB for 0 <= RHS < max(32, #bits(T))
    //
    // With C++17 there's little room for optimizations because the standard
    // requires all shifts to happen on promoted integrals (i.e. int). Thus,
    // short and char shifts must assume shifts affect bits of neighboring
    // values.
  #ifndef _GLIBCXX_SIMD_NO_SHIFT_OPT
    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      inline _GLIBCXX_CONST static typename _TVT::type
      _S_bit_shift_left(_Tp __xx, int __y)
      {
	using _V = typename _TVT::type;
	using _Up = typename _TVT::value_type;
	_V __x = __xx;
	[[maybe_unused]] const auto __ix = __to_intrin(__x);
	if (__builtin_is_constant_evaluated())
	  return __x << __y;
#if __cplusplus > 201703
	// after C++17, signed shifts have no UB, and behave just like unsigned
	// shifts
	else if constexpr (sizeof(_Up) == 1 && is_signed_v<_Up>)
	  return __vector_bitcast<_Up>(
	    _S_bit_shift_left(__vector_bitcast<make_unsigned_t<_Up>>(__x),
			      __y));
#endif
	else if constexpr (sizeof(_Up) == 1)
	  {
	    // (cf. https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83894)
	    if (__builtin_constant_p(__y))
	      {
		if (__y == 0)
		  return __x;
		else if (__y == 1)
		  return __x + __x;
		else if (__y == 2)
		  {
		    __x = __x + __x;
		    return __x + __x;
		  }
		else if (__y > 2 && __y < 8)
		  {
		    if constexpr (sizeof(__x) > sizeof(unsigned))
		      {
			const _UChar __mask = 0xff << __y; // precomputed vector
			return __vector_bitcast<_Up>(
			  __vector_bitcast<_UChar>(
			    __vector_bitcast<unsigned>(__x) << __y)
			  & __mask);
		      }
		    else
		      {
			const unsigned __mask
			  = (0xff & (0xff << __y)) * 0x01010101u;
			return reinterpret_cast<_V>(
			  static_cast<__int_for_sizeof_t<_V>>(
			    unsigned(
			      reinterpret_cast<__int_for_sizeof_t<_V>>(__x)
			      << __y)
			    & __mask));
		      }
		  }
		else if (__y >= 8 && __y < 32)
		  return _V();
		else
		  __builtin_unreachable();
	      }
	    // general strategy in the following: use an sllv instead of sll
	    // instruction, because it's 2 to 4 times faster:
	    else if constexpr (__have_avx512bw_vl && sizeof(__x) == 16)
	      return __vector_bitcast<_Up>(_mm256_cvtepi16_epi8(
		_mm256_sllv_epi16(_mm256_cvtepi8_epi16(__ix),
				  _mm256_set1_epi16(__y))));
	    else if constexpr (__have_avx512bw && sizeof(__x) == 32)
	      return __vector_bitcast<_Up>(_mm512_cvtepi16_epi8(
		_mm512_sllv_epi16(_mm512_cvtepi8_epi16(__ix),
				  _mm512_set1_epi16(__y))));
	    else if constexpr (__have_avx512bw && sizeof(__x) == 64)
	      {
		const auto __shift = _mm512_set1_epi16(__y);
		return __vector_bitcast<_Up>(
		  __concat(_mm512_cvtepi16_epi8(_mm512_sllv_epi16(
			     _mm512_cvtepi8_epi16(__lo256(__ix)), __shift)),
			   _mm512_cvtepi16_epi8(_mm512_sllv_epi16(
			     _mm512_cvtepi8_epi16(__hi256(__ix)), __shift))));
	      }
	    else if constexpr (__have_avx2 && sizeof(__x) == 32)
	      {
#if 1
		const auto __shift = _mm_cvtsi32_si128(__y);
		auto __k
		  = _mm256_sll_epi16(_mm256_slli_epi16(~__m256i(), 8), __shift);
		__k |= _mm256_srli_epi16(__k, 8);
		return __vector_bitcast<_Up>(_mm256_sll_epi32(__ix, __shift)
					     & __k);
#else
		const _Up __k = 0xff << __y;
		return __vector_bitcast<_Up>(__vector_bitcast<int>(__x) << __y)
		       & __k;
#endif
	      }
	    else
	      {
		const auto __shift = _mm_cvtsi32_si128(__y);
		auto __k
		  = _mm_sll_epi16(_mm_slli_epi16(~__m128i(), 8), __shift);
		__k |= _mm_srli_epi16(__k, 8);
		return __intrin_bitcast<_V>(_mm_sll_epi16(__ix, __shift) & __k);
	      }
	  }
	return __x << __y;
      }

    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      inline _GLIBCXX_CONST static typename _TVT::type
      _S_bit_shift_left(_Tp __xx, typename _TVT::type __y)
      {
	using _V = typename _TVT::type;
	using _Up = typename _TVT::value_type;
	_V __x = __xx;
	[[maybe_unused]] const auto __ix = __to_intrin(__x);
	[[maybe_unused]] const auto __iy = __to_intrin(__y);
	if (__builtin_is_constant_evaluated())
	  return __x << __y;
#if __cplusplus > 201703
	// after C++17, signed shifts have no UB, and behave just like unsigned
	// shifts
	else if constexpr (is_signed_v<_Up>)
	  return __vector_bitcast<_Up>(
	    _S_bit_shift_left(__vector_bitcast<make_unsigned_t<_Up>>(__x),
			      __vector_bitcast<make_unsigned_t<_Up>>(__y)));
#endif
	else if constexpr (sizeof(_Up) == 1)
	  {
	    if constexpr (sizeof __ix == 64 && __have_avx512bw)
	      return __vector_bitcast<_Up>(__concat(
		_mm512_cvtepi16_epi8(
		  _mm512_sllv_epi16(_mm512_cvtepu8_epi16(__lo256(__ix)),
				    _mm512_cvtepu8_epi16(__lo256(__iy)))),
		_mm512_cvtepi16_epi8(
		  _mm512_sllv_epi16(_mm512_cvtepu8_epi16(__hi256(__ix)),
				    _mm512_cvtepu8_epi16(__hi256(__iy))))));
	    else if constexpr (sizeof __ix == 32 && __have_avx512bw)
	      return __vector_bitcast<_Up>(_mm512_cvtepi16_epi8(
		_mm512_sllv_epi16(_mm512_cvtepu8_epi16(__ix),
				  _mm512_cvtepu8_epi16(__iy))));
	    else if constexpr (sizeof __x <= 8 && __have_avx512bw_vl)
	      return __intrin_bitcast<_V>(
		_mm_cvtepi16_epi8(_mm_sllv_epi16(_mm_cvtepu8_epi16(__ix),
						 _mm_cvtepu8_epi16(__iy))));
	    else if constexpr (sizeof __ix == 16 && __have_avx512bw_vl)
	      return __intrin_bitcast<_V>(_mm256_cvtepi16_epi8(
		_mm256_sllv_epi16(_mm256_cvtepu8_epi16(__ix),
				  _mm256_cvtepu8_epi16(__iy))));
	    else if constexpr (sizeof __ix == 16 && __have_avx512bw)
	      return __intrin_bitcast<_V>(
		__lo128(_mm512_cvtepi16_epi8(_mm512_sllv_epi16(
		  _mm512_cvtepu8_epi16(_mm256_castsi128_si256(__ix)),
		  _mm512_cvtepu8_epi16(_mm256_castsi128_si256(__iy))))));
	    else if constexpr (__have_sse4_1 && sizeof(__x) == 16)
	      {
		auto __mask
		  = __vector_bitcast<_Up>(__vector_bitcast<short>(__y) << 5);
		auto __x4
		  = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 4);
		__x4 &= char(0xf0);
		__x = reinterpret_cast<_V>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__x), __to_intrin(__x4)));
		__mask += __mask;
		auto __x2
		  = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 2);
		__x2 &= char(0xfc);
		__x = reinterpret_cast<_V>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__x), __to_intrin(__x2)));
		__mask += __mask;
		auto __x1 = __x + __x;
		__x = reinterpret_cast<_V>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__x), __to_intrin(__x1)));
		return __x
		       & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
	      }
	    else if constexpr (sizeof(__x) == 16)
	      {
		auto __mask
		  = __vector_bitcast<_UChar>(__vector_bitcast<short>(__y) << 5);
		auto __x4
		  = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 4);
		__x4 &= char(0xf0);
		__x = __vector_bitcast<_SChar>(__mask) < 0 ? __x4 : __x;
		__mask += __mask;
		auto __x2
		  = __vector_bitcast<_Up>(__vector_bitcast<short>(__x) << 2);
		__x2 &= char(0xfc);
		__x = __vector_bitcast<_SChar>(__mask) < 0 ? __x2 : __x;
		__mask += __mask;
		auto __x1 = __x + __x;
		__x = __vector_bitcast<_SChar>(__mask) < 0 ? __x1 : __x;
		return __x
		       & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
	      }
	    else
	      return __x << __y;
	  }
	else if constexpr (sizeof(_Up) == 2)
	  {
	    if constexpr (sizeof __ix == 64 && __have_avx512bw)
	      return __vector_bitcast<_Up>(_mm512_sllv_epi16(__ix, __iy));
	    else if constexpr (sizeof __ix == 32 && __have_avx512bw_vl)
	      return __vector_bitcast<_Up>(_mm256_sllv_epi16(__ix, __iy));
	    else if constexpr (sizeof __ix == 32 && __have_avx512bw)
	      return __vector_bitcast<_Up>(
		__lo256(_mm512_sllv_epi16(_mm512_castsi256_si512(__ix),
					  _mm512_castsi256_si512(__iy))));
	    else if constexpr (sizeof __ix == 32 && __have_avx2)
	      {
		const auto __ux = __vector_bitcast<unsigned>(__x);
		const auto __uy = __vector_bitcast<unsigned>(__y);
		return __vector_bitcast<_Up>(_mm256_blend_epi16(
		  __auto_bitcast(__ux << (__uy & 0x0000ffffu)),
		  __auto_bitcast((__ux & 0xffff0000u) << (__uy >> 16)), 0xaa));
	      }
	    else if constexpr (sizeof __ix == 16 && __have_avx512bw_vl)
	      return __intrin_bitcast<_V>(_mm_sllv_epi16(__ix, __iy));
	    else if constexpr (sizeof __ix == 16 && __have_avx512bw)
	      return __intrin_bitcast<_V>(
		__lo128(_mm512_sllv_epi16(_mm512_castsi128_si512(__ix),
					  _mm512_castsi128_si512(__iy))));
	    else if constexpr (sizeof __ix == 16 && __have_avx2)
	      {
		const auto __ux = __vector_bitcast<unsigned>(__ix);
		const auto __uy = __vector_bitcast<unsigned>(__iy);
		return __intrin_bitcast<_V>(_mm_blend_epi16(
		  __auto_bitcast(__ux << (__uy & 0x0000ffffu)),
		  __auto_bitcast((__ux & 0xffff0000u) << (__uy >> 16)), 0xaa));
	      }
	    else if constexpr (sizeof __ix == 16)
	      {
		using _Float4 = __vector_type_t<float, 4>;
		using _Int4 = __vector_type_t<int, 4>;
		using _UInt4 = __vector_type_t<unsigned, 4>;
		const _UInt4 __yu
		  = reinterpret_cast<_UInt4>(__to_intrin(__y + (0x3f8 >> 3)));
		return __x
		       * __intrin_bitcast<_V>(
			 __vector_convert<_Int4>(_SimdWrapper<float, 4>(
			   reinterpret_cast<_Float4>(__yu << 23)))
			 | (__vector_convert<_Int4>(_SimdWrapper<float, 4>(
			      reinterpret_cast<_Float4>((__yu >> 16) << 23)))
			    << 16));
	      }
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (sizeof(_Up) == 4 && sizeof __ix == 16
			   && !__have_avx2)
	  // latency is suboptimal, but throughput is at full speedup
	  return __intrin_bitcast<_V>(
	    __vector_bitcast<unsigned>(__ix)
	    * __vector_convert<__vector_type16_t<int>>(
	      _SimdWrapper<float, 4>(__vector_bitcast<float>(
		(__vector_bitcast<unsigned, 4>(__y) << 23) + 0x3f80'0000))));
	else if constexpr (sizeof(_Up) == 8 && sizeof __ix == 16
			   && !__have_avx2)
	  {
	    const auto __lo = _mm_sll_epi64(__ix, __iy);
	    const auto __hi
	      = _mm_sll_epi64(__ix, _mm_unpackhi_epi64(__iy, __iy));
	    if constexpr (__have_sse4_1)
	      return __vector_bitcast<_Up>(_mm_blend_epi16(__lo, __hi, 0xf0));
	    else
	      return __vector_bitcast<_Up>(
		_mm_move_sd(__vector_bitcast<double>(__hi),
			    __vector_bitcast<double>(__lo)));
	  }
	else
	  return __x << __y;
      }
#endif // _GLIBCXX_SIMD_NO_SHIFT_OPT

    // }}}
    // _S_bit_shift_right {{{
#ifndef _GLIBCXX_SIMD_NO_SHIFT_OPT
    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      inline _GLIBCXX_CONST static typename _TVT::type
      _S_bit_shift_right(_Tp __xx, int __y)
      {
	using _V = typename _TVT::type;
	using _Up = typename _TVT::value_type;
	_V __x = __xx;
	[[maybe_unused]] const auto __ix = __to_intrin(__x);
	if (__builtin_is_constant_evaluated())
	  return __x >> __y;
	else if (__builtin_constant_p(__y)
		 && is_unsigned_v<
		   _Up> && __y >= int(sizeof(_Up) * __CHAR_BIT__))
	  return _V();
	else if constexpr (sizeof(_Up) == 1 && is_unsigned_v<_Up>) //{{{
	  return __intrin_bitcast<_V>(__vector_bitcast<_UShort>(__ix) >> __y)
		 & _Up(0xff >> __y);
	//}}}
	else if constexpr (sizeof(_Up) == 1 && is_signed_v<_Up>) //{{{
	  return __intrin_bitcast<_V>(
	    (__vector_bitcast<_UShort>(__vector_bitcast<short>(__ix)
				       >> (__y + 8))
	     << 8)
	    | (__vector_bitcast<_UShort>(
		 __vector_bitcast<short>(__vector_bitcast<_UShort>(__ix) << 8)
		 >> __y)
	       >> 8));
	//}}}
	// GCC optimizes sizeof == 2, 4, and unsigned 8 as expected
	else if constexpr (sizeof(_Up) == 8 && is_signed_v<_Up>) //{{{
	  {
	    if (__y > 32)
	      return (__intrin_bitcast<_V>(__vector_bitcast<int>(__ix) >> 32)
		      & _Up(0xffff'ffff'0000'0000ull))
		     | __vector_bitcast<_Up>(
		       __vector_bitcast<int>(__vector_bitcast<_ULLong>(__ix)
					     >> 32)
		       >> (__y - 32));
	    else
	      return __intrin_bitcast<_V>(__vector_bitcast<_ULLong>(__ix)
					  >> __y)
		     | __vector_bitcast<_Up>(
		       __vector_bitcast<int>(__ix & -0x8000'0000'0000'0000ll)
		       >> __y);
	  }
	//}}}
	else
	  return __x >> __y;
      }

    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      inline _GLIBCXX_CONST static typename _TVT::type
      _S_bit_shift_right(_Tp __xx, typename _TVT::type __y)
      {
	using _V = typename _TVT::type;
	using _Up = typename _TVT::value_type;
	_V __x = __xx;
	[[maybe_unused]] const auto __ix = __to_intrin(__x);
	[[maybe_unused]] const auto __iy = __to_intrin(__y);
	if (__builtin_is_constant_evaluated()
	    || (__builtin_constant_p(__x) && __builtin_constant_p(__y)))
	  return __x >> __y;
	else if constexpr (sizeof(_Up) == 1) //{{{
	  {
	    if constexpr (sizeof(__x) <= 8 && __have_avx512bw_vl)
	      return __intrin_bitcast<_V>(_mm_cvtepi16_epi8(
		is_signed_v<_Up> ? _mm_srav_epi16(_mm_cvtepi8_epi16(__ix),
						  _mm_cvtepi8_epi16(__iy))
				 : _mm_srlv_epi16(_mm_cvtepu8_epi16(__ix),
						  _mm_cvtepu8_epi16(__iy))));
	    if constexpr (sizeof(__x) == 16 && __have_avx512bw_vl)
	      return __intrin_bitcast<_V>(_mm256_cvtepi16_epi8(
		is_signed_v<_Up>
		  ? _mm256_srav_epi16(_mm256_cvtepi8_epi16(__ix),
				      _mm256_cvtepi8_epi16(__iy))
		  : _mm256_srlv_epi16(_mm256_cvtepu8_epi16(__ix),
				      _mm256_cvtepu8_epi16(__iy))));
	    else if constexpr (sizeof(__x) == 32 && __have_avx512bw)
	      return __vector_bitcast<_Up>(_mm512_cvtepi16_epi8(
		is_signed_v<_Up>
		  ? _mm512_srav_epi16(_mm512_cvtepi8_epi16(__ix),
				      _mm512_cvtepi8_epi16(__iy))
		  : _mm512_srlv_epi16(_mm512_cvtepu8_epi16(__ix),
				      _mm512_cvtepu8_epi16(__iy))));
	    else if constexpr (sizeof(__x) == 64 && is_signed_v<_Up>)
	      return __vector_bitcast<_Up>(_mm512_mask_mov_epi8(
		_mm512_srav_epi16(__ix, _mm512_srli_epi16(__iy, 8)),
		0x5555'5555'5555'5555ull,
		_mm512_srav_epi16(
		  _mm512_slli_epi16(__ix, 8),
		  _mm512_maskz_add_epi8(0x5555'5555'5555'5555ull, __iy,
					_mm512_set1_epi16(8)))));
	    else if constexpr (sizeof(__x) == 64 && is_unsigned_v<_Up>)
	      return __vector_bitcast<_Up>(_mm512_mask_mov_epi8(
		_mm512_srlv_epi16(__ix, _mm512_srli_epi16(__iy, 8)),
		0x5555'5555'5555'5555ull,
		_mm512_srlv_epi16(
		  _mm512_maskz_mov_epi8(0x5555'5555'5555'5555ull, __ix),
		  _mm512_maskz_mov_epi8(0x5555'5555'5555'5555ull, __iy))));
	    /* This has better throughput but higher latency than the impl below
	    else if constexpr (__have_avx2 && sizeof(__x) == 16 &&
			       is_unsigned_v<_Up>)
	      {
		const auto __shorts = __to_intrin(_S_bit_shift_right(
		  __vector_bitcast<_UShort>(_mm256_cvtepu8_epi16(__ix)),
		  __vector_bitcast<_UShort>(_mm256_cvtepu8_epi16(__iy))));
		return __vector_bitcast<_Up>(
		  _mm_packus_epi16(__lo128(__shorts), __hi128(__shorts)));
	      }
	      */
	    else if constexpr (__have_avx2 && sizeof(__x) > 8)
	      // the following uses vpsr[al]vd, which requires AVX2
	      if constexpr (is_signed_v<_Up>)
		{
		  const auto r3 = __vector_bitcast<_UInt>(
				    (__vector_bitcast<int>(__x)
				     >> (__vector_bitcast<_UInt>(__y) >> 24)))
				  & 0xff000000u;
		  const auto r2
		    = __vector_bitcast<_UInt>(
			((__vector_bitcast<int>(__x) << 8)
			 >> ((__vector_bitcast<_UInt>(__y) << 8) >> 24)))
		      & 0xff000000u;
		  const auto r1
		    = __vector_bitcast<_UInt>(
			((__vector_bitcast<int>(__x) << 16)
			 >> ((__vector_bitcast<_UInt>(__y) << 16) >> 24)))
		      & 0xff000000u;
		  const auto r0 = __vector_bitcast<_UInt>(
		    (__vector_bitcast<int>(__x) << 24)
		    >> ((__vector_bitcast<_UInt>(__y) << 24) >> 24));
		  return __vector_bitcast<_Up>(r3 | (r2 >> 8) | (r1 >> 16)
					       | (r0 >> 24));
		}
	      else
		{
		  const auto r3 = (__vector_bitcast<_UInt>(__x)
				   >> (__vector_bitcast<_UInt>(__y) >> 24))
				  & 0xff000000u;
		  const auto r2
		    = ((__vector_bitcast<_UInt>(__x) << 8)
		       >> ((__vector_bitcast<_UInt>(__y) << 8) >> 24))
		      & 0xff000000u;
		  const auto r1
		    = ((__vector_bitcast<_UInt>(__x) << 16)
		       >> ((__vector_bitcast<_UInt>(__y) << 16) >> 24))
		      & 0xff000000u;
		  const auto r0
		    = (__vector_bitcast<_UInt>(__x) << 24)
		      >> ((__vector_bitcast<_UInt>(__y) << 24) >> 24);
		  return __vector_bitcast<_Up>(r3 | (r2 >> 8) | (r1 >> 16)
					       | (r0 >> 24));
		}
	    else if constexpr (__have_sse4_1
			       && is_unsigned_v<_Up> && sizeof(__x) > 2)
	      {
		auto __x128 = __vector_bitcast<_Up>(__ix);
		auto __mask
		  = __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__iy) << 5);
		auto __x4 = __vector_bitcast<_Up>(
		  (__vector_bitcast<_UShort>(__x128) >> 4) & _UShort(0xff0f));
		__x128 = __vector_bitcast<_Up>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__x128), __to_intrin(__x4)));
		__mask += __mask;
		auto __x2 = __vector_bitcast<_Up>(
		  (__vector_bitcast<_UShort>(__x128) >> 2) & _UShort(0xff3f));
		__x128 = __vector_bitcast<_Up>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__x128), __to_intrin(__x2)));
		__mask += __mask;
		auto __x1 = __vector_bitcast<_Up>(
		  (__vector_bitcast<_UShort>(__x128) >> 1) & _UShort(0xff7f));
		__x128 = __vector_bitcast<_Up>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__x128), __to_intrin(__x1)));
		return __intrin_bitcast<_V>(
		  __x128
		  & ((__vector_bitcast<_Up>(__iy) & char(0xf8))
		     == 0)); // y > 7 nulls the result
	      }
	    else if constexpr (__have_sse4_1
			       && is_signed_v<_Up> && sizeof(__x) > 2)
	      {
		auto __mask = __vector_bitcast<_UChar>(
		  __vector_bitcast<_UShort>(__iy) << 5);
		auto __maskl = [&]() {
		  return __to_intrin(__vector_bitcast<_UShort>(__mask) << 8);
		};
		auto __xh = __vector_bitcast<short>(__ix);
		auto __xl = __vector_bitcast<short>(__ix) << 8;
		auto __xh4 = __xh >> 4;
		auto __xl4 = __xl >> 4;
		__xh = __vector_bitcast<short>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__xh), __to_intrin(__xh4)));
		__xl = __vector_bitcast<short>(
		  _CommonImplX86::_S_blend_intrin(__maskl(), __to_intrin(__xl),
						  __to_intrin(__xl4)));
		__mask += __mask;
		auto __xh2 = __xh >> 2;
		auto __xl2 = __xl >> 2;
		__xh = __vector_bitcast<short>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__xh), __to_intrin(__xh2)));
		__xl = __vector_bitcast<short>(
		  _CommonImplX86::_S_blend_intrin(__maskl(), __to_intrin(__xl),
						  __to_intrin(__xl2)));
		__mask += __mask;
		auto __xh1 = __xh >> 1;
		auto __xl1 = __xl >> 1;
		__xh = __vector_bitcast<short>(_CommonImplX86::_S_blend_intrin(
		  __to_intrin(__mask), __to_intrin(__xh), __to_intrin(__xh1)));
		__xl = __vector_bitcast<short>(
		  _CommonImplX86::_S_blend_intrin(__maskl(), __to_intrin(__xl),
						  __to_intrin(__xl1)));
		return __intrin_bitcast<_V>(
		  (__vector_bitcast<_Up>((__xh & short(0xff00)))
		   | __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__xl)
					   >> 8))
		  & ((__vector_bitcast<_Up>(__iy) & char(0xf8))
		     == 0)); // y > 7 nulls the result
	      }
	    else if constexpr (is_unsigned_v<_Up> && sizeof(__x) > 2) // SSE2
	      {
		auto __mask
		  = __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__y) << 5);
		auto __x4 = __vector_bitcast<_Up>(
		  (__vector_bitcast<_UShort>(__x) >> 4) & _UShort(0xff0f));
		__x = __mask > 0x7f ? __x4 : __x;
		__mask += __mask;
		auto __x2 = __vector_bitcast<_Up>(
		  (__vector_bitcast<_UShort>(__x) >> 2) & _UShort(0xff3f));
		__x = __mask > 0x7f ? __x2 : __x;
		__mask += __mask;
		auto __x1 = __vector_bitcast<_Up>(
		  (__vector_bitcast<_UShort>(__x) >> 1) & _UShort(0xff7f));
		__x = __mask > 0x7f ? __x1 : __x;
		return __x
		       & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
	      }
	    else if constexpr (sizeof(__x) > 2) // signed SSE2
	      {
		static_assert(is_signed_v<_Up>);
		auto __maskh = __vector_bitcast<_UShort>(__y) << 5;
		auto __maskl = __vector_bitcast<_UShort>(__y) << (5 + 8);
		auto __xh = __vector_bitcast<short>(__x);
		auto __xl = __vector_bitcast<short>(__x) << 8;
		auto __xh4 = __xh >> 4;
		auto __xl4 = __xl >> 4;
		__xh = __maskh > 0x7fff ? __xh4 : __xh;
		__xl = __maskl > 0x7fff ? __xl4 : __xl;
		__maskh += __maskh;
		__maskl += __maskl;
		auto __xh2 = __xh >> 2;
		auto __xl2 = __xl >> 2;
		__xh = __maskh > 0x7fff ? __xh2 : __xh;
		__xl = __maskl > 0x7fff ? __xl2 : __xl;
		__maskh += __maskh;
		__maskl += __maskl;
		auto __xh1 = __xh >> 1;
		auto __xl1 = __xl >> 1;
		__xh = __maskh > 0x7fff ? __xh1 : __xh;
		__xl = __maskl > 0x7fff ? __xl1 : __xl;
		__x = __vector_bitcast<_Up>((__xh & short(0xff00)))
		      | __vector_bitcast<_Up>(__vector_bitcast<_UShort>(__xl)
					      >> 8);
		return __x
		       & ((__y & char(0xf8)) == 0); // y > 7 nulls the result
	      }
	    else
	      return __x >> __y;
	  }                                                      //}}}
	else if constexpr (sizeof(_Up) == 2 && sizeof(__x) >= 4) //{{{
	  {
	    [[maybe_unused]] auto __blend_0xaa = [](auto __a, auto __b) {
	      if constexpr (sizeof(__a) == 16)
		return _mm_blend_epi16(__to_intrin(__a), __to_intrin(__b),
				       0xaa);
	      else if constexpr (sizeof(__a) == 32)
		return _mm256_blend_epi16(__to_intrin(__a), __to_intrin(__b),
					  0xaa);
	      else if constexpr (sizeof(__a) == 64)
		return _mm512_mask_blend_epi16(0xaaaa'aaaaU, __to_intrin(__a),
					       __to_intrin(__b));
	      else
		__assert_unreachable<decltype(__a)>();
	    };
	    if constexpr (__have_avx512bw_vl && sizeof(_Tp) <= 16)
	      return __intrin_bitcast<_V>(is_signed_v<_Up>
					    ? _mm_srav_epi16(__ix, __iy)
					    : _mm_srlv_epi16(__ix, __iy));
	    else if constexpr (__have_avx512bw_vl && sizeof(_Tp) == 32)
	      return __vector_bitcast<_Up>(is_signed_v<_Up>
					     ? _mm256_srav_epi16(__ix, __iy)
					     : _mm256_srlv_epi16(__ix, __iy));
	    else if constexpr (__have_avx512bw && sizeof(_Tp) == 64)
	      return __vector_bitcast<_Up>(is_signed_v<_Up>
					     ? _mm512_srav_epi16(__ix, __iy)
					     : _mm512_srlv_epi16(__ix, __iy));
	    else if constexpr (__have_avx2 && is_signed_v<_Up>)
	      return __intrin_bitcast<_V>(
		__blend_0xaa(((__vector_bitcast<int>(__ix) << 16)
			      >> (__vector_bitcast<int>(__iy) & 0xffffu))
			       >> 16,
			     __vector_bitcast<int>(__ix)
			       >> (__vector_bitcast<int>(__iy) >> 16)));
	    else if constexpr (__have_avx2 && is_unsigned_v<_Up>)
	      return __intrin_bitcast<_V>(
		__blend_0xaa((__vector_bitcast<_UInt>(__ix) & 0xffffu)
			       >> (__vector_bitcast<_UInt>(__iy) & 0xffffu),
			     __vector_bitcast<_UInt>(__ix)
			       >> (__vector_bitcast<_UInt>(__iy) >> 16)));
	    else if constexpr (__have_sse4_1)
	      {
		auto __mask = __vector_bitcast<_UShort>(__iy);
		auto __x128 = __vector_bitcast<_Up>(__ix);
		//__mask *= 0x0808;
		__mask = (__mask << 3) | (__mask << 11);
		// do __x128 = 0 where __y[4] is set
		__x128 = __vector_bitcast<_Up>(
		  _mm_blendv_epi8(__to_intrin(__x128), __m128i(),
				  __to_intrin(__mask)));
		// do __x128 =>> 8 where __y[3] is set
		__x128 = __vector_bitcast<_Up>(
		  _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 8),
				  __to_intrin(__mask += __mask)));
		// do __x128 =>> 4 where __y[2] is set
		__x128 = __vector_bitcast<_Up>(
		  _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 4),
				  __to_intrin(__mask += __mask)));
		// do __x128 =>> 2 where __y[1] is set
		__x128 = __vector_bitcast<_Up>(
		  _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 2),
				  __to_intrin(__mask += __mask)));
		// do __x128 =>> 1 where __y[0] is set
		return __intrin_bitcast<_V>(
		  _mm_blendv_epi8(__to_intrin(__x128), __to_intrin(__x128 >> 1),
				  __to_intrin(__mask + __mask)));
	      }
	    else
	      {
		auto __k = __vector_bitcast<_UShort>(__iy) << 11;
		auto __x128 = __vector_bitcast<_Up>(__ix);
		auto __mask = [](__vector_type16_t<_UShort> __kk) {
		  return __vector_bitcast<short>(__kk) < 0;
		};
		// do __x128 = 0 where __y[4] is set
		__x128 = __mask(__k) ? decltype(__x128)() : __x128;
		// do __x128 =>> 8 where __y[3] is set
		__x128 = __mask(__k += __k) ? __x128 >> 8 : __x128;
		// do __x128 =>> 4 where __y[2] is set
		__x128 = __mask(__k += __k) ? __x128 >> 4 : __x128;
		// do __x128 =>> 2 where __y[1] is set
		__x128 = __mask(__k += __k) ? __x128 >> 2 : __x128;
		// do __x128 =>> 1 where __y[0] is set
		return __intrin_bitcast<_V>(__mask(__k + __k) ? __x128 >> 1
							      : __x128);
	      }
	  }                                                  //}}}
	else if constexpr (sizeof(_Up) == 4 && !__have_avx2) //{{{
	  {
	    if constexpr (is_unsigned_v<_Up>)
	      {
		// x >> y == x * 2^-y == (x * 2^(31-y)) >> 31
		const __m128 __factor_f = reinterpret_cast<__m128>(
		  0x4f00'0000u - (__vector_bitcast<unsigned, 4>(__y) << 23));
		const __m128i __factor
		  = __builtin_constant_p(__factor_f)
		      ? __to_intrin(
			__make_vector<unsigned>(__factor_f[0], __factor_f[1],
						__factor_f[2], __factor_f[3]))
		      : _mm_cvttps_epi32(__factor_f);
		const auto __r02
		  = _mm_srli_epi64(_mm_mul_epu32(__ix, __factor), 31);
		const auto __r13 = _mm_mul_epu32(_mm_srli_si128(__ix, 4),
						 _mm_srli_si128(__factor, 4));
		if constexpr (__have_sse4_1)
		  return __intrin_bitcast<_V>(
		    _mm_blend_epi16(_mm_slli_epi64(__r13, 1), __r02, 0x33));
		else
		  return __intrin_bitcast<_V>(
		    __r02 | _mm_slli_si128(_mm_srli_epi64(__r13, 31), 4));
	      }
	    else
	      {
		auto __shift = [](auto __a, auto __b) {
		  if constexpr (is_signed_v<_Up>)
		    return _mm_sra_epi32(__a, __b);
		  else
		    return _mm_srl_epi32(__a, __b);
		};
		const auto __r0
		  = __shift(__ix, _mm_unpacklo_epi32(__iy, __m128i()));
		const auto __r1 = __shift(__ix, _mm_srli_epi64(__iy, 32));
		const auto __r2
		  = __shift(__ix, _mm_unpackhi_epi32(__iy, __m128i()));
		const auto __r3 = __shift(__ix, _mm_srli_si128(__iy, 12));
		if constexpr (__have_sse4_1)
		  return __intrin_bitcast<_V>(
		    _mm_blend_epi16(_mm_blend_epi16(__r1, __r0, 0x3),
				    _mm_blend_epi16(__r3, __r2, 0x30), 0xf0));
		else
		  return __intrin_bitcast<_V>(_mm_unpacklo_epi64(
		    _mm_unpacklo_epi32(__r0, _mm_srli_si128(__r1, 4)),
		    _mm_unpackhi_epi32(__r2, _mm_srli_si128(__r3, 4))));
	      }
	  } //}}}
	else
	  return __x >> __y;
      }
#endif // _GLIBCXX_SIMD_NO_SHIFT_OPT

    // }}}
    // compares {{{
    // _S_equal_to {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
      _S_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	if constexpr (__is_avx512_abi<_Abi>()) // {{{
	  {
	    if (__builtin_is_constant_evaluated()
		|| (__x._M_is_constprop() && __y._M_is_constprop()))
	      return _MaskImpl::_S_to_bits(
		__as_wrapper<_Np>(__x._M_data == __y._M_data));

	    constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    [[maybe_unused]] const auto __xi = __to_intrin(__x);
	    [[maybe_unused]] const auto __yi = __to_intrin(__y);
	    if constexpr (is_floating_point_v<_Tp>)
	      {
		if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
		  return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
		else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
		  return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
		else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
		  return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
		else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
		  return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
		else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
		  return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
		else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
		  return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_EQ_OQ);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 2)
	      return _mm512_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 1)
	      return _mm512_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 2)
	      return _mm256_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 1)
	      return _mm256_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 2)
	      return _mm_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 1)
	      return _mm_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
	    else
	      __assert_unreachable<_Tp>();
	  } // }}}
	else if (__builtin_is_constant_evaluated())
	  return _Base::_S_equal_to(__x, __y);
	else if constexpr (sizeof(__x) == 8) // {{{
	  {
	    const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
				== __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
	    _MaskMember<_Tp> __r64;
	    __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
	    return __r64;
	  } // }}}
	else
	  return _Base::_S_equal_to(__x, __y);
      }

    // }}}
    // _S_not_equal_to {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
      _S_not_equal_to(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	if constexpr (__is_avx512_abi<_Abi>()) // {{{
	  {
	    if (__builtin_is_constant_evaluated()
		|| (__x._M_is_constprop() && __y._M_is_constprop()))
	      return _MaskImpl::_S_to_bits(
		__as_wrapper<_Np>(__x._M_data != __y._M_data));

	    constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    [[maybe_unused]] const auto __xi = __to_intrin(__x);
	    [[maybe_unused]] const auto __yi = __to_intrin(__y);
	    if constexpr (is_floating_point_v<_Tp>)
	      {
		if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
		  return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
		else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
		  return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
		else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
		  return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
		else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
		  return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
		else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
		  return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
		else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
		  return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_UQ);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return ~_mm512_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return ~_mm512_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 2)
	      return ~_mm512_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 1)
	      return ~_mm512_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return ~_mm256_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return ~_mm256_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 2)
	      return ~_mm256_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 1)
	      return ~_mm256_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return ~_mm_mask_cmpeq_epi64_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return ~_mm_mask_cmpeq_epi32_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 2)
	      return ~_mm_mask_cmpeq_epi16_mask(__k1, __xi, __yi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 1)
	      return ~_mm_mask_cmpeq_epi8_mask(__k1, __xi, __yi);
	    else
	      __assert_unreachable<_Tp>();
	  }                                                   // }}}
	else if constexpr (!__builtin_is_constant_evaluated() // {{{
			   && sizeof(__x) == 8)
	  {
	    const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
				!= __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
	    _MaskMember<_Tp> __r64;
	    __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
	    return __r64;
	  } // }}}
	else
	  return _Base::_S_not_equal_to(__x, __y);
      }

    // }}}
    // _S_less {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
      _S_less(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	if constexpr (__is_avx512_abi<_Abi>()) // {{{
	  {
	    if (__builtin_is_constant_evaluated()
		|| (__x._M_is_constprop() && __y._M_is_constprop()))
	      return _MaskImpl::_S_to_bits(
		__as_wrapper<_Np>(__x._M_data < __y._M_data));

	    constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    [[maybe_unused]] const auto __xi = __to_intrin(__x);
	    [[maybe_unused]] const auto __yi = __to_intrin(__y);
	    if constexpr (sizeof(__xi) == 64)
	      {
		if constexpr (is_same_v<_Tp, float>)
		  return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OS);
		else if constexpr (is_same_v<_Tp, double>)
		  return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OS);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm512_mask_cmplt_epi8_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm512_mask_cmplt_epi16_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm512_mask_cmplt_epi32_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm512_mask_cmplt_epi64_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm512_mask_cmplt_epu8_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm512_mask_cmplt_epu16_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm512_mask_cmplt_epu32_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm512_mask_cmplt_epu64_mask(__k1, __xi, __yi);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (sizeof(__xi) == 32)
	      {
		if constexpr (is_same_v<_Tp, float>)
		  return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OS);
		else if constexpr (is_same_v<_Tp, double>)
		  return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OS);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm256_mask_cmplt_epi8_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm256_mask_cmplt_epi16_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm256_mask_cmplt_epi32_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm256_mask_cmplt_epi64_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm256_mask_cmplt_epu8_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm256_mask_cmplt_epu16_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm256_mask_cmplt_epu32_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm256_mask_cmplt_epu64_mask(__k1, __xi, __yi);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (sizeof(__xi) == 16)
	      {
		if constexpr (is_same_v<_Tp, float>)
		  return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OS);
		else if constexpr (is_same_v<_Tp, double>)
		  return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OS);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm_mask_cmplt_epi8_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm_mask_cmplt_epi16_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm_mask_cmplt_epi32_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm_mask_cmplt_epi64_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm_mask_cmplt_epu8_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm_mask_cmplt_epu16_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm_mask_cmplt_epu32_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm_mask_cmplt_epu64_mask(__k1, __xi, __yi);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else
	      __assert_unreachable<_Tp>();
	  }                                                   // }}}
	else if constexpr (!__builtin_is_constant_evaluated() // {{{
			   && sizeof(__x) == 8)
	  {
	    const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
				< __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
	    _MaskMember<_Tp> __r64;
	    __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
	    return __r64;
	  } // }}}
	else
	  return _Base::_S_less(__x, __y);
      }

    // }}}
    // _S_less_equal {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
      _S_less_equal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	if constexpr (__is_avx512_abi<_Abi>()) // {{{
	  {
	    if (__builtin_is_constant_evaluated()
		|| (__x._M_is_constprop() && __y._M_is_constprop()))
	      return _MaskImpl::_S_to_bits(
		__as_wrapper<_Np>(__x._M_data <= __y._M_data));

	    constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    [[maybe_unused]] const auto __xi = __to_intrin(__x);
	    [[maybe_unused]] const auto __yi = __to_intrin(__y);
	    if constexpr (sizeof(__xi) == 64)
	      {
		if constexpr (is_same_v<_Tp, float>)
		  return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OS);
		else if constexpr (is_same_v<_Tp, double>)
		  return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OS);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm512_mask_cmple_epi8_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm512_mask_cmple_epi16_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm512_mask_cmple_epi32_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm512_mask_cmple_epi64_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm512_mask_cmple_epu8_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm512_mask_cmple_epu16_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm512_mask_cmple_epu32_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm512_mask_cmple_epu64_mask(__k1, __xi, __yi);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (sizeof(__xi) == 32)
	      {
		if constexpr (is_same_v<_Tp, float>)
		  return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OS);
		else if constexpr (is_same_v<_Tp, double>)
		  return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OS);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm256_mask_cmple_epi8_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm256_mask_cmple_epi16_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm256_mask_cmple_epi32_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm256_mask_cmple_epi64_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm256_mask_cmple_epu8_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm256_mask_cmple_epu16_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm256_mask_cmple_epu32_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm256_mask_cmple_epu64_mask(__k1, __xi, __yi);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else if constexpr (sizeof(__xi) == 16)
	      {
		if constexpr (is_same_v<_Tp, float>)
		  return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OS);
		else if constexpr (is_same_v<_Tp, double>)
		  return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OS);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm_mask_cmple_epi8_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm_mask_cmple_epi16_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm_mask_cmple_epi32_mask(__k1, __xi, __yi);
		else if constexpr (is_signed_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm_mask_cmple_epi64_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 1)
		  return _mm_mask_cmple_epu8_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 2)
		  return _mm_mask_cmple_epu16_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 4)
		  return _mm_mask_cmple_epu32_mask(__k1, __xi, __yi);
		else if constexpr (is_unsigned_v<_Tp> && sizeof(_Tp) == 8)
		  return _mm_mask_cmple_epu64_mask(__k1, __xi, __yi);
		else
		  __assert_unreachable<_Tp>();
	      }
	    else
	      __assert_unreachable<_Tp>();
	  }                                                   // }}}
	else if constexpr (!__builtin_is_constant_evaluated() // {{{
			   && sizeof(__x) == 8)
	  {
	    const auto __r128 = __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__x)
				<= __vector_bitcast<_Tp, 16 / sizeof(_Tp)>(__y);
	    _MaskMember<_Tp> __r64;
	    __builtin_memcpy(&__r64._M_data, &__r128, sizeof(__r64));
	    return __r64;
	  } // }}}
	else
	  return _Base::_S_less_equal(__x, __y);
      }

    // }}} }}}
    // negation {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
      _S_negate(_SimdWrapper<_Tp, _Np> __x) noexcept
      {
	if constexpr (__is_avx512_abi<_Abi>())
	  return _S_equal_to(__x, _SimdWrapper<_Tp, _Np>());
	else
	  return _Base::_S_negate(__x);
      }

    // }}}
    // math {{{
    using _Base::_S_abs;

    // _S_sqrt {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
      _S_sqrt(_SimdWrapper<_Tp, _Np> __x)
      {
	if constexpr (__is_sse_ps<_Tp, _Np>())
	  return __auto_bitcast(_mm_sqrt_ps(__to_intrin(__x)));
	else if constexpr (__is_sse_pd<_Tp, _Np>())
	  return _mm_sqrt_pd(__x);
	else if constexpr (__is_avx_ps<_Tp, _Np>())
	  return _mm256_sqrt_ps(__x);
	else if constexpr (__is_avx_pd<_Tp, _Np>())
	  return _mm256_sqrt_pd(__x);
	else if constexpr (__is_avx512_ps<_Tp, _Np>())
	  return _mm512_sqrt_ps(__x);
	else if constexpr (__is_avx512_pd<_Tp, _Np>())
	  return _mm512_sqrt_pd(__x);
	else
	  __assert_unreachable<_Tp>();
      }

    // }}}
    // _S_ldexp {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
      _S_ldexp(_SimdWrapper<_Tp, _Np> __x,
	       __fixed_size_storage_t<int, _Np> __exp)
      {
	if constexpr (sizeof(__x) == 64 || __have_avx512vl)
	  {
	    const auto __xi = __to_intrin(__x);
	    constexpr _SimdConverter<int, simd_abi::fixed_size<_Np>, _Tp, _Abi>
	      __cvt;
	    const auto __expi = __to_intrin(__cvt(__exp));
	    using _Up = __bool_storage_member_type_t<_Np>;
	    constexpr _Up __k1 = _Np < sizeof(_Up) * __CHAR_BIT__ ? _Up((1ULL << _Np) - 1) : ~_Up();
	    if constexpr (sizeof(__xi) == 16)
	      {
		if constexpr (sizeof(_Tp) == 8)
		  return _mm_maskz_scalef_pd(__k1, __xi, __expi);
		else
		  return _mm_maskz_scalef_ps(__k1, __xi, __expi);
	      }
	    else if constexpr (sizeof(__xi) == 32)
	      {
		if constexpr (sizeof(_Tp) == 8)
		  return _mm256_maskz_scalef_pd(__k1, __xi, __expi);
		else
		  return _mm256_maskz_scalef_ps(__k1, __xi, __expi);
	      }
	    else
	      {
		static_assert(sizeof(__xi) == 64);
		if constexpr (sizeof(_Tp) == 8)
		  return _mm512_maskz_scalef_pd(__k1, __xi, __expi);
		else
		  return _mm512_maskz_scalef_ps(__k1, __xi, __expi);
	      }
	  }
	else
	  return _Base::_S_ldexp(__x, __exp);
      }

    // }}}
    // _S_trunc {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
      _S_trunc(_SimdWrapper<_Tp, _Np> __x)
      {
	if constexpr (__is_avx512_ps<_Tp, _Np>())
	  return _mm512_roundscale_ps(__x, 0x0b);
	else if constexpr (__is_avx512_pd<_Tp, _Np>())
	  return _mm512_roundscale_pd(__x, 0x0b);
	else if constexpr (__is_avx_ps<_Tp, _Np>())
	  return _mm256_round_ps(__x, 0xb);
	else if constexpr (__is_avx_pd<_Tp, _Np>())
	  return _mm256_round_pd(__x, 0xb);
	else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
	  return __auto_bitcast(_mm_round_ps(__to_intrin(__x), 0xb));
	else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
	  return _mm_round_pd(__x, 0xb);
	else if constexpr (__is_sse_ps<_Tp, _Np>())
	  {
	    auto __truncated
	      = _mm_cvtepi32_ps(_mm_cvttps_epi32(__to_intrin(__x)));
	    const auto __no_fractional_values
	      = __vector_bitcast<int>(__vector_bitcast<_UInt>(__to_intrin(__x))
				      & 0x7f800000u)
		< 0x4b000000; // the exponent is so large that no mantissa bits
			      // signify fractional values (0x3f8 + 23*8 =
			      // 0x4b0)
	    return __no_fractional_values ? __truncated : __to_intrin(__x);
	  }
	else
	  return _Base::_S_trunc(__x);
      }

    // }}}
    // _S_round {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
      _S_round(_SimdWrapper<_Tp, _Np> __x)
      {
	// Note that _MM_FROUND_TO_NEAREST_INT rounds ties to even, not away
	// from zero as required by std::round. Therefore this function is more
	// complicated.
	using _V = __vector_type_t<_Tp, _Np>;
	_V __truncated;
	if constexpr (__is_avx512_ps<_Tp, _Np>())
	  __truncated = _mm512_roundscale_ps(__x._M_data, 0x0b);
	else if constexpr (__is_avx512_pd<_Tp, _Np>())
	  __truncated = _mm512_roundscale_pd(__x._M_data, 0x0b);
	else if constexpr (__is_avx_ps<_Tp, _Np>())
	  __truncated = _mm256_round_ps(__x._M_data,
					_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
	else if constexpr (__is_avx_pd<_Tp, _Np>())
	  __truncated = _mm256_round_pd(__x._M_data,
					_MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
	else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
	  __truncated = __auto_bitcast(
	    _mm_round_ps(__to_intrin(__x),
			 _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC));
	else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
	  __truncated
	    = _mm_round_pd(__x._M_data, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
	else if constexpr (__is_sse_ps<_Tp, _Np>())
	  __truncated = __auto_bitcast(
	    _mm_cvtepi32_ps(_mm_cvttps_epi32(__to_intrin(__x))));
	else
	  return _Base::_S_round(__x);

	// x < 0 => truncated <= 0 && truncated >= x => x - truncated <= 0
	// x > 0 => truncated >= 0 && truncated <= x => x - truncated >= 0

	const _V __rounded
	  = __truncated
	    + (__and(_S_absmask<_V>, __x._M_data - __truncated) >= _Tp(.5)
		 ? __or(__and(_S_signmask<_V>, __x._M_data), _V() + 1)
		 : _V());
	if constexpr (__have_sse4_1)
	  return __rounded;
	else // adjust for missing range in cvttps_epi32
	  return __and(_S_absmask<_V>, __x._M_data) < 0x1p23f ? __rounded
							      : __x._M_data;
      }

    // }}}
    // _S_nearbyint {{{
    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      _GLIBCXX_SIMD_INTRINSIC static _Tp _S_nearbyint(_Tp __x) noexcept
      {
	if constexpr (_TVT::template _S_is<float, 16>)
	  return _mm512_roundscale_ps(__x, 0x0c);
	else if constexpr (_TVT::template _S_is<double, 8>)
	  return _mm512_roundscale_pd(__x, 0x0c);
	else if constexpr (_TVT::template _S_is<float, 8>)
	  return _mm256_round_ps(__x,
				 _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
	else if constexpr (_TVT::template _S_is<double, 4>)
	  return _mm256_round_pd(__x,
				 _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
	else if constexpr (__have_sse4_1 && _TVT::template _S_is<float, 4>)
	  return _mm_round_ps(__x,
			      _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
	else if constexpr (__have_sse4_1 && _TVT::template _S_is<double, 2>)
	  return _mm_round_pd(__x,
			      _MM_FROUND_CUR_DIRECTION | _MM_FROUND_NO_EXC);
	else
	  return _Base::_S_nearbyint(__x);
      }

    // }}}
    // _S_rint {{{
    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      _GLIBCXX_SIMD_INTRINSIC static _Tp _S_rint(_Tp __x) noexcept
      {
	if constexpr (_TVT::template _S_is<float, 16>)
	  return _mm512_roundscale_ps(__x, 0x04);
	else if constexpr (_TVT::template _S_is<double, 8>)
	  return _mm512_roundscale_pd(__x, 0x04);
	else if constexpr (_TVT::template _S_is<float, 8>)
	  return _mm256_round_ps(__x, _MM_FROUND_CUR_DIRECTION);
	else if constexpr (_TVT::template _S_is<double, 4>)
	  return _mm256_round_pd(__x, _MM_FROUND_CUR_DIRECTION);
	else if constexpr (__have_sse4_1 && _TVT::template _S_is<float, 4>)
	  return _mm_round_ps(__x, _MM_FROUND_CUR_DIRECTION);
	else if constexpr (__have_sse4_1 && _TVT::template _S_is<double, 2>)
	  return _mm_round_pd(__x, _MM_FROUND_CUR_DIRECTION);
	else
	  return _Base::_S_rint(__x);
      }

    // }}}
    // _S_floor {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
      _S_floor(_SimdWrapper<_Tp, _Np> __x)
      {
	if constexpr (__is_avx512_ps<_Tp, _Np>())
	  return _mm512_roundscale_ps(__x, 0x09);
	else if constexpr (__is_avx512_pd<_Tp, _Np>())
	  return _mm512_roundscale_pd(__x, 0x09);
	else if constexpr (__is_avx_ps<_Tp, _Np>())
	  return _mm256_round_ps(__x, 0x9);
	else if constexpr (__is_avx_pd<_Tp, _Np>())
	  return _mm256_round_pd(__x, 0x9);
	else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
	  return __auto_bitcast(_mm_round_ps(__to_intrin(__x), 0x9));
	else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
	  return _mm_round_pd(__x, 0x9);
	else
	  return _Base::_S_floor(__x);
      }

    // }}}
    // _S_ceil {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
      _S_ceil(_SimdWrapper<_Tp, _Np> __x)
      {
	if constexpr (__is_avx512_ps<_Tp, _Np>())
	  return _mm512_roundscale_ps(__x, 0x0a);
	else if constexpr (__is_avx512_pd<_Tp, _Np>())
	  return _mm512_roundscale_pd(__x, 0x0a);
	else if constexpr (__is_avx_ps<_Tp, _Np>())
	  return _mm256_round_ps(__x, 0xa);
	else if constexpr (__is_avx_pd<_Tp, _Np>())
	  return _mm256_round_pd(__x, 0xa);
	else if constexpr (__have_sse4_1 && __is_sse_ps<_Tp, _Np>())
	  return __auto_bitcast(_mm_round_ps(__to_intrin(__x), 0xa));
	else if constexpr (__have_sse4_1 && __is_sse_pd<_Tp, _Np>())
	  return _mm_round_pd(__x, 0xa);
	else
	  return _Base::_S_ceil(__x);
      }

    // }}}
    // _S_signbit {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
      _S_signbit(_SimdWrapper<_Tp, _Np> __x)
      {
	if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
	  {
	    if constexpr (sizeof(__x) == 64 && sizeof(_Tp) == 4)
	      return _mm512_movepi32_mask(
		__intrin_bitcast<__m512i>(__x._M_data));
	    else if constexpr (sizeof(__x) == 64 && sizeof(_Tp) == 8)
	      return _mm512_movepi64_mask(
		__intrin_bitcast<__m512i>(__x._M_data));
	    else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) == 4)
	      return _mm256_movepi32_mask(
		__intrin_bitcast<__m256i>(__x._M_data));
	    else if constexpr (sizeof(__x) == 32 && sizeof(_Tp) == 8)
	      return _mm256_movepi64_mask(
		__intrin_bitcast<__m256i>(__x._M_data));
	    else if constexpr (sizeof(__x) <= 16 && sizeof(_Tp) == 4)
	      return _mm_movepi32_mask(__intrin_bitcast<__m128i>(__x._M_data));
	    else if constexpr (sizeof(__x) <= 16 && sizeof(_Tp) == 8)
	      return _mm_movepi64_mask(__intrin_bitcast<__m128i>(__x._M_data));
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  {
	    const auto __xi = __to_intrin(__x);
	    [[maybe_unused]] constexpr auto __k1
	      = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_movemask_ps(__xi);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_movemask_pd(__xi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_movemask_ps(__xi);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_movemask_pd(__xi);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmplt_epi32_mask(
		__k1, __intrin_bitcast<__m512i>(__xi), __m512i());
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmplt_epi64_mask(
		__k1, __intrin_bitcast<__m512i>(__xi), __m512i());
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_signbit(__x);
	/*{
	  using _I = __int_for_sizeof_t<_Tp>;
	  if constexpr (sizeof(__x) == 64)
	    return _S_less(__vector_bitcast<_I>(__x), _I());
	  else
	    {
	      const auto __xx = __vector_bitcast<_I>(__x._M_data);
	      [[maybe_unused]] constexpr _I __signmask = __finite_min_v<_I>;
	      if constexpr ((sizeof(_Tp) == 4 &&
			     (__have_avx2 || sizeof(__x) == 16)) ||
			    __have_avx512vl)
		{
		  return __vector_bitcast<_Tp>(__xx >> __digits_v<_I>);
		}
	      else if constexpr ((__have_avx2 ||
				  (__have_ssse3 && sizeof(__x) == 16)))
		{
		  return __vector_bitcast<_Tp>((__xx & __signmask) ==
					       __signmask);
		}
	      else
		{ // SSE2/3 or AVX (w/o AVX2)
		  constexpr auto __one = __vector_broadcast<_Np, _Tp>(1);
		  return __vector_bitcast<_Tp>(
		    __vector_bitcast<_Tp>(
		      (__xx & __signmask) |
		      __vector_bitcast<_I>(__one)) // -1 or 1
		    != __one);
		}
	    }
	}*/
      }

    // }}}
    // _S_isnonzerovalue_mask {{{
    // (isnormal | is subnormal == !isinf & !isnan & !is zero)
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static auto _S_isnonzerovalue_mask(_Tp __x)
      {
	using _Traits = _VectorTraits<_Tp>;
	if constexpr (__have_avx512dq_vl)
	  {
	    if constexpr (_Traits::template _S_is<
			    float, 2> || _Traits::template _S_is<float, 4>)
	      return _knot_mask8(_mm_fpclass_ps_mask(__to_intrin(__x), 0x9f));
	    else if constexpr (_Traits::template _S_is<float, 8>)
	      return _knot_mask8(_mm256_fpclass_ps_mask(__x, 0x9f));
	    else if constexpr (_Traits::template _S_is<float, 16>)
	      return _knot_mask16(_mm512_fpclass_ps_mask(__x, 0x9f));
	    else if constexpr (_Traits::template _S_is<double, 2>)
	      return _knot_mask8(_mm_fpclass_pd_mask(__x, 0x9f));
	    else if constexpr (_Traits::template _S_is<double, 4>)
	      return _knot_mask8(_mm256_fpclass_pd_mask(__x, 0x9f));
	    else if constexpr (_Traits::template _S_is<double, 8>)
	      return _knot_mask8(_mm512_fpclass_pd_mask(__x, 0x9f));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  {
	    using _Up = typename _Traits::value_type;
	    constexpr size_t _Np = _Traits::_S_full_size;
	    const auto __a = __x * __infinity_v<_Up>; // NaN if __x == 0
	    const auto __b = __x * _Up();             // NaN if __x == inf
	    if constexpr (__have_avx512vl && __is_sse_ps<_Up, _Np>())
	      return _mm_cmp_ps_mask(__to_intrin(__a), __to_intrin(__b),
				     _CMP_ORD_Q);
	    else if constexpr (__have_avx512f && __is_sse_ps<_Up, _Np>())
	      return __mmask8(0xf
			      & _mm512_cmp_ps_mask(__auto_bitcast(__a),
						   __auto_bitcast(__b),
						   _CMP_ORD_Q));
	    else if constexpr (__have_avx512vl && __is_sse_pd<_Up, _Np>())
	      return _mm_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
	    else if constexpr (__have_avx512f && __is_sse_pd<_Up, _Np>())
	      return __mmask8(0x3
			      & _mm512_cmp_pd_mask(__auto_bitcast(__a),
						   __auto_bitcast(__b),
						   _CMP_ORD_Q));
	    else if constexpr (__have_avx512vl && __is_avx_ps<_Up, _Np>())
	      return _mm256_cmp_ps_mask(__a, __b, _CMP_ORD_Q);
	    else if constexpr (__have_avx512f && __is_avx_ps<_Up, _Np>())
	      return __mmask8(_mm512_cmp_ps_mask(__auto_bitcast(__a),
						 __auto_bitcast(__b),
						 _CMP_ORD_Q));
	    else if constexpr (__have_avx512vl && __is_avx_pd<_Up, _Np>())
	      return _mm256_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
	    else if constexpr (__have_avx512f && __is_avx_pd<_Up, _Np>())
	      return __mmask8(0xf
			      & _mm512_cmp_pd_mask(__auto_bitcast(__a),
						   __auto_bitcast(__b),
						   _CMP_ORD_Q));
	    else if constexpr (__is_avx512_ps<_Up, _Np>())
	      return _mm512_cmp_ps_mask(__a, __b, _CMP_ORD_Q);
	    else if constexpr (__is_avx512_pd<_Up, _Np>())
	      return _mm512_cmp_pd_mask(__a, __b, _CMP_ORD_Q);
	    else
	      __assert_unreachable<_Tp>();
	  }
      }

    // }}}
    // _S_isfinite {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
      _S_isfinite(_SimdWrapper<_Tp, _Np> __x)
      {
	static_assert(is_floating_point_v<_Tp>);
#if !__FINITE_MATH_ONLY__
	if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
	  {
	    const auto __xi = __to_intrin(__x);
	    constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return __k1 ^ _mm512_mask_fpclass_ps_mask(__k1, __xi, 0x99);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return __k1 ^ _mm512_mask_fpclass_pd_mask(__k1, __xi, 0x99);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return __k1 ^ _mm256_mask_fpclass_ps_mask(__k1, __xi, 0x99);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return __k1 ^ _mm256_mask_fpclass_pd_mask(__k1, __xi, 0x99);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return __k1 ^ _mm_mask_fpclass_ps_mask(__k1, __xi, 0x99);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return __k1 ^ _mm_mask_fpclass_pd_mask(__k1, __xi, 0x99);
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  {
	    // if all exponent bits are set, __x is either inf or NaN
	    using _I = __int_for_sizeof_t<_Tp>;
	    const auto __inf = __vector_bitcast<_I>(
	      __vector_broadcast<_Np>(__infinity_v<_Tp>));
	    return _S_less<_I, _Np>(__vector_bitcast<_I>(__x) & __inf, __inf);
	  }
	else
#endif
	  return _Base::_S_isfinite(__x);
      }

    // }}}
    // _S_isinf {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
      _S_isinf(_SimdWrapper<_Tp, _Np> __x)
      {
#if !__FINITE_MATH_ONLY__
	if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
	  {
	    const auto __xi = __to_intrin(__x);
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_fpclass_ps_mask(__xi, 0x18);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_fpclass_pd_mask(__xi, 0x18);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_fpclass_ps_mask(__xi, 0x18);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_fpclass_pd_mask(__xi, 0x18);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_fpclass_ps_mask(__xi, 0x18);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_fpclass_pd_mask(__xi, 0x18);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_avx512dq_vl)
	  {
	    if constexpr (__is_sse_pd<_Tp, _Np>())
	      return _mm_movm_epi64(_mm_fpclass_pd_mask(__x, 0x18));
	    else if constexpr (__is_avx_pd<_Tp, _Np>())
	      return _mm256_movm_epi64(_mm256_fpclass_pd_mask(__x, 0x18));
	    else if constexpr (__is_sse_ps<_Tp, _Np>())
	      return _mm_movm_epi32(
		_mm_fpclass_ps_mask(__to_intrin(__x), 0x18));
	    else if constexpr (__is_avx_ps<_Tp, _Np>())
	      return _mm256_movm_epi32(_mm256_fpclass_ps_mask(__x, 0x18));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
#endif
	  return _Base::_S_isinf(__x);
      }

    // }}}
    // _S_isnormal {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
      _S_isnormal(_SimdWrapper<_Tp, _Np> __x)
      {
#if __FINITE_MATH_ONLY__
	[[maybe_unused]] constexpr int __mode = 0x26;
#else
      [[maybe_unused]] constexpr int __mode = 0xbf;
#endif
	if constexpr (__is_avx512_abi<_Abi>() && __have_avx512dq)
	  {
	    const auto __xi = __to_intrin(__x);
	    const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return __k1 ^ _mm512_mask_fpclass_ps_mask(__k1, __xi, __mode);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return __k1 ^ _mm512_mask_fpclass_pd_mask(__k1, __xi, __mode);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return __k1 ^ _mm256_mask_fpclass_ps_mask(__k1, __xi, __mode);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return __k1 ^ _mm256_mask_fpclass_pd_mask(__k1, __xi, __mode);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return __k1 ^ _mm_mask_fpclass_ps_mask(__k1, __xi, __mode);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return __k1 ^ _mm_mask_fpclass_pd_mask(__k1, __xi, __mode);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_avx512dq)
	  {
	    if constexpr (__have_avx512vl && __is_sse_ps<_Tp, _Np>())
	      return _mm_movm_epi32(
		_knot_mask8(_mm_fpclass_ps_mask(__to_intrin(__x), __mode)));
	    else if constexpr (__have_avx512vl && __is_avx_ps<_Tp, _Np>())
	      return _mm256_movm_epi32(
		_knot_mask8(_mm256_fpclass_ps_mask(__x, __mode)));
	    else if constexpr (__is_avx512_ps<_Tp, _Np>())
	      return _knot_mask16(_mm512_fpclass_ps_mask(__x, __mode));
	    else if constexpr (__have_avx512vl && __is_sse_pd<_Tp, _Np>())
	      return _mm_movm_epi64(
		_knot_mask8(_mm_fpclass_pd_mask(__x, __mode)));
	    else if constexpr (__have_avx512vl && __is_avx_pd<_Tp, _Np>())
	      return _mm256_movm_epi64(
		_knot_mask8(_mm256_fpclass_pd_mask(__x, __mode)));
	    else if constexpr (__is_avx512_pd<_Tp, _Np>())
	      return _knot_mask8(_mm512_fpclass_pd_mask(__x, __mode));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  {
	    using _I = __int_for_sizeof_t<_Tp>;
	    const auto absn = __vector_bitcast<_I>(_S_abs(__x));
	    const auto minn = __vector_bitcast<_I>(
	      __vector_broadcast<_Np>(__norm_min_v<_Tp>));
#if __FINITE_MATH_ONLY__
	    return _S_less_equal<_I, _Np>(minn, absn);
#else
	  const auto infn
	    = __vector_bitcast<_I>(__vector_broadcast<_Np>(__infinity_v<_Tp>));
	  return __and(_S_less_equal<_I, _Np>(minn, absn),
		       _S_less<_I, _Np>(absn, infn));
#endif
	  }
	else
	  return _Base::_S_isnormal(__x);
      }

    // }}}
    // _S_isnan {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
      _S_isnan(_SimdWrapper<_Tp, _Np> __x)
      { return _S_isunordered(__x, __x); }

    // }}}
    // _S_isunordered {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
      _S_isunordered([[maybe_unused]] _SimdWrapper<_Tp, _Np> __x,
		     [[maybe_unused]] _SimdWrapper<_Tp, _Np> __y)
      {
#if __FINITE_MATH_ONLY__
	return {}; // false
#else
	const auto __xi = __to_intrin(__x);
	const auto __yi = __to_intrin(__y);
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    constexpr auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_UNORD_Q);
	  }
      else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_UNORD_Q));
      else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_UNORD_Q));
      else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	return __auto_bitcast(_mm_cmpunord_ps(__xi, __yi));
      else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	return __to_masktype(_mm_cmpunord_pd(__xi, __yi));
      else
	__assert_unreachable<_Tp>();
#endif
      }

    // }}}
    // _S_isgreater {{{
    template <typename _Tp, size_t _Np>
      static constexpr _MaskMember<_Tp> _S_isgreater(_SimdWrapper<_Tp, _Np> __x,
						     _SimdWrapper<_Tp, _Np> __y)
      {
	const auto __xi = __to_intrin(__x);
	const auto __yi = __to_intrin(__y);
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GT_OQ);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GT_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GT_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GT_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GT_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GT_OQ);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_avx)
	  {
	    if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_GT_OQ));
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_GT_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_GT_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_GT_OQ));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 4)
	  {
	    const auto __xn = __vector_bitcast<int>(__xi);
	    const auto __yn = __vector_bitcast<int>(__yi);
	    const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
	    const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
	    return __auto_bitcast(
	      __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp > __yp));
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 8)
	  return __vector_type_t<__int_with_sizeof_t<8>, 2>{
	    -_mm_ucomigt_sd(__xi, __yi),
	    -_mm_ucomigt_sd(_mm_unpackhi_pd(__xi, __xi),
			    _mm_unpackhi_pd(__yi, __yi))};
	else
	  return _Base::_S_isgreater(__x, __y);
      }

    // }}}
    // _S_isgreaterequal {{{
    template <typename _Tp, size_t _Np>
      static constexpr _MaskMember<_Tp>
      _S_isgreaterequal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	const auto __xi = __to_intrin(__x);
	const auto __yi = __to_intrin(__y);
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GE_OQ);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GE_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GE_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GE_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_GE_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_GE_OQ);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_avx)
	  {
	    if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_GE_OQ));
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_GE_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_GE_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_GE_OQ));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 4)
	  {
	    const auto __xn = __vector_bitcast<int>(__xi);
	    const auto __yn = __vector_bitcast<int>(__yi);
	    const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
	    const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
	    return __auto_bitcast(
	      __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp >= __yp));
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 8)
	  return __vector_type_t<__int_with_sizeof_t<8>, 2>{
	    -_mm_ucomige_sd(__xi, __yi),
	    -_mm_ucomige_sd(_mm_unpackhi_pd(__xi, __xi),
			    _mm_unpackhi_pd(__yi, __yi))};
	else
	  return _Base::_S_isgreaterequal(__x, __y);
      }

    // }}}
    // _S_isless {{{
    template <typename _Tp, size_t _Np>
      static constexpr _MaskMember<_Tp> _S_isless(_SimdWrapper<_Tp, _Np> __x,
						  _SimdWrapper<_Tp, _Np> __y)
      {
	const auto __xi = __to_intrin(__x);
	const auto __yi = __to_intrin(__y);
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OQ);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LT_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LT_OQ);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_avx)
	  {
	    if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_LT_OQ));
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_LT_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_LT_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_LT_OQ));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 4)
	  {
	    const auto __xn = __vector_bitcast<int>(__xi);
	    const auto __yn = __vector_bitcast<int>(__yi);
	    const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
	    const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
	    return __auto_bitcast(
	      __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp < __yp));
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 8)
	  return __vector_type_t<__int_with_sizeof_t<8>, 2>{
	    -_mm_ucomigt_sd(__yi, __xi),
	    -_mm_ucomigt_sd(_mm_unpackhi_pd(__yi, __yi),
			    _mm_unpackhi_pd(__xi, __xi))};
	else
	  return _Base::_S_isless(__x, __y);
      }

    // }}}
    // _S_islessequal {{{
    template <typename _Tp, size_t _Np>
      static constexpr _MaskMember<_Tp>
      _S_islessequal(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	const auto __xi = __to_intrin(__x);
	const auto __yi = __to_intrin(__y);
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OQ);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_LE_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_LE_OQ);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_avx)
	  {
	    if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_LE_OQ));
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_LE_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_LE_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_LE_OQ));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 4)
	  {
	    const auto __xn = __vector_bitcast<int>(__xi);
	    const auto __yn = __vector_bitcast<int>(__yi);
	    const auto __xp = __xn < 0 ? -(__xn & 0x7fff'ffff) : __xn;
	    const auto __yp = __yn < 0 ? -(__yn & 0x7fff'ffff) : __yn;
	    return __auto_bitcast(
	      __and(__to_masktype(_mm_cmpord_ps(__xi, __yi)), __xp <= __yp));
	  }
	else if constexpr (__have_sse2 && sizeof(__xi) == 16
			   && sizeof(_Tp) == 8)
	  return __vector_type_t<__int_with_sizeof_t<8>, 2>{
	    -_mm_ucomige_sd(__yi, __xi),
	    -_mm_ucomige_sd(_mm_unpackhi_pd(__yi, __yi),
			    _mm_unpackhi_pd(__xi, __xi))};
	else
	  return _Base::_S_islessequal(__x, __y);
      }

    // }}}
    // _S_islessgreater {{{
    template <typename _Tp, size_t _Np>
      static constexpr _MaskMember<_Tp>
      _S_islessgreater(_SimdWrapper<_Tp, _Np> __x, _SimdWrapper<_Tp, _Np> __y)
      {
	const auto __xi = __to_intrin(__x);
	const auto __yi = __to_intrin(__y);
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    const auto __k1 = _Abi::template _S_implicit_mask_intrin<_Tp>();
	    if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 4)
	      return _mm512_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
	    else if constexpr (sizeof(__xi) == 64 && sizeof(_Tp) == 8)
	      return _mm512_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return _mm256_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return _mm256_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return _mm_mask_cmp_ps_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return _mm_mask_cmp_pd_mask(__k1, __xi, __yi, _CMP_NEQ_OQ);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__have_avx)
	  {
	    if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 4)
	      return __to_masktype(_mm256_cmp_ps(__xi, __yi, _CMP_NEQ_OQ));
	    else if constexpr (sizeof(__xi) == 32 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm256_cmp_pd(__xi, __yi, _CMP_NEQ_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	      return __auto_bitcast(_mm_cmp_ps(__xi, __yi, _CMP_NEQ_OQ));
	    else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	      return __to_masktype(_mm_cmp_pd(__xi, __yi, _CMP_NEQ_OQ));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 4)
	  return __auto_bitcast(
	    __and(_mm_cmpord_ps(__xi, __yi), _mm_cmpneq_ps(__xi, __yi)));
	else if constexpr (sizeof(__xi) == 16 && sizeof(_Tp) == 8)
	  return __to_masktype(
	    __and(_mm_cmpord_pd(__xi, __yi), _mm_cmpneq_pd(__xi, __yi)));
	else
	  __assert_unreachable<_Tp>();
      }

    //}}} }}}
  };

// }}}
// _MaskImplX86Mixin {{{
struct _MaskImplX86Mixin
{
  template <typename _Tp>
    using _TypeTag = _Tp*;

  using _Base = _MaskImplBuiltinMixin;

  // _S_to_maskvector(bool) {{{
  template <typename _Up, size_t _ToN = 1, typename _Tp>
    _GLIBCXX_SIMD_INTRINSIC static constexpr enable_if_t<
      is_same_v<_Tp, bool>, _SimdWrapper<_Up, _ToN>>
    _S_to_maskvector(_Tp __x)
    {
      static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
      return __x ? __vector_type_t<_Up, _ToN>{~_Up()}
		 : __vector_type_t<_Up, _ToN>();
    }

  // }}}
  // _S_to_maskvector(_SanitizedBitMask) {{{
  template <typename _Up, size_t _UpN = 0, size_t _Np,
	    size_t _ToN = _UpN == 0 ? _Np : _UpN>
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN>
    _S_to_maskvector(_SanitizedBitMask<_Np> __x)
    {
      static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
      using _UV = __vector_type_t<_Up, _ToN>;
      using _UI = __intrinsic_type_t<_Up, _ToN>;
      [[maybe_unused]] const auto __k = __x._M_to_bits();
      if constexpr (_Np == 1)
	return _S_to_maskvector<_Up, _ToN>(__k);
      else if (__x._M_is_constprop() || __builtin_is_constant_evaluated())
	return __generate_from_n_evaluations<std::min(_ToN, _Np), _UV>(
	  [&](auto __i) -> _Up { return -__x[__i.value]; });
      else if constexpr (sizeof(_Up) == 1)
	{
	  if constexpr (sizeof(_UI) == 16)
	    {
	      if constexpr (__have_avx512bw_vl)
		return __intrin_bitcast<_UV>(_mm_movm_epi8(__k));
	      else if constexpr (__have_avx512bw)
		return __intrin_bitcast<_UV>(__lo128(_mm512_movm_epi8(__k)));
	      else if constexpr (__have_avx512f)
		{
		  auto __as32bits = _mm512_maskz_mov_epi32(__k, ~__m512i());
		  auto __as16bits
		    = __xzyw(_mm256_packs_epi32(__lo256(__as32bits),
						__hi256(__as32bits)));
		  return __intrin_bitcast<_UV>(
		    _mm_packs_epi16(__lo128(__as16bits), __hi128(__as16bits)));
		}
	      else if constexpr (__have_ssse3)
		{
		  const auto __bitmask = __to_intrin(
		    __make_vector<_UChar>(1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4,
					  8, 16, 32, 64, 128));
		  return __intrin_bitcast<_UV>(
		    __vector_bitcast<_Up>(
		      _mm_shuffle_epi8(__to_intrin(
					 __vector_type_t<_ULLong, 2>{__k}),
				       _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1,
						     1, 1, 1, 1, 1, 1, 1))
		      & __bitmask)
		    != 0);
		}
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 32)
	    {
	      if constexpr (__have_avx512bw_vl)
		return __vector_bitcast<_Up>(_mm256_movm_epi8(__k));
	      else if constexpr (__have_avx512bw)
		return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi8(__k)));
	      else if constexpr (__have_avx512f)
		{
		  auto __as16bits = // 0 16 1 17 ... 15 31
		    _mm512_srli_epi32(_mm512_maskz_mov_epi32(__k, ~__m512i()),
				      16)
		    | _mm512_slli_epi32(_mm512_maskz_mov_epi32(__k >> 16,
							       ~__m512i()),
					16);
		  auto __0_16_1_17 = __xzyw(_mm256_packs_epi16(
		    __lo256(__as16bits),
		    __hi256(__as16bits)) // 0 16 1 17 2 18 3 19 8 24 9 25 ...
		  );
		  // deinterleave:
		  return __vector_bitcast<_Up>(__xzyw(_mm256_shuffle_epi8(
		    __0_16_1_17, // 0 16 1 17 2 ...
		    _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9,
				     11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 14, 1,
				     3, 5, 7, 9, 11, 13,
				     15)))); // 0-7 16-23 8-15 24-31 -> xzyw
					     // 0-3  8-11 16-19 24-27
					     // 4-7 12-15 20-23 28-31
		}
	      else if constexpr (__have_avx2)
		{
		  const auto __bitmask
		    = _mm256_broadcastsi128_si256(__to_intrin(
		      __make_vector<_UChar>(1, 2, 4, 8, 16, 32, 64, 128, 1, 2,
					    4, 8, 16, 32, 64, 128)));
		  return __vector_bitcast<_Up>(
		    __vector_bitcast<_Up>(
		      _mm256_shuffle_epi8(
			_mm256_broadcastsi128_si256(
			  __to_intrin(__vector_type_t<_ULLong, 2>{__k})),
			_mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
					 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3,
					 3, 3, 3, 3, 3, 3))
		      & __bitmask)
		    != 0);
		}
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 64)
	    return reinterpret_cast<_UV>(_mm512_movm_epi8(__k));
	  if constexpr (std::min(_ToN, _Np) <= 4)
	    {
	      if constexpr (_Np > 7) // avoid overflow
		__x &= _SanitizedBitMask<_Np>(0x0f);
	      const _UInt __char_mask
		= ((_UInt(__x.to_ulong()) * 0x00204081U) & 0x01010101ULL)
		  * 0xff;
	      _UV __r = {};
	      __builtin_memcpy(&__r, &__char_mask,
			       std::min(sizeof(__r), sizeof(__char_mask)));
	      return __r;
	    }
	  else if constexpr (std::min(_ToN, _Np) <= 7)
	    {
	      if constexpr (_Np > 7) // avoid overflow
		__x &= _SanitizedBitMask<_Np>(0x7f);
	      const _ULLong __char_mask
		= ((__x.to_ulong() * 0x40810204081ULL) & 0x0101010101010101ULL)
		  * 0xff;
	      _UV __r = {};
	      __builtin_memcpy(&__r, &__char_mask,
			       std::min(sizeof(__r), sizeof(__char_mask)));
	      return __r;
	    }
	}
      else if constexpr (sizeof(_Up) == 2)
	{
	  if constexpr (sizeof(_UI) == 16)
	    {
	      if constexpr (__have_avx512bw_vl)
		return __intrin_bitcast<_UV>(_mm_movm_epi16(__k));
	      else if constexpr (__have_avx512bw)
		return __intrin_bitcast<_UV>(__lo128(_mm512_movm_epi16(__k)));
	      else if constexpr (__have_avx512f)
		{
		  __m256i __as32bits = {};
		  if constexpr (__have_avx512vl)
		    __as32bits = _mm256_maskz_mov_epi32(__k, ~__m256i());
		  else
		    __as32bits
		      = __lo256(_mm512_maskz_mov_epi32(__k, ~__m512i()));
		  return __intrin_bitcast<_UV>(
		    _mm_packs_epi32(__lo128(__as32bits), __hi128(__as32bits)));
		}
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 32)
	    {
	      if constexpr (__have_avx512bw_vl)
		return __vector_bitcast<_Up>(_mm256_movm_epi16(__k));
	      else if constexpr (__have_avx512bw)
		return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi16(__k)));
	      else if constexpr (__have_avx512f)
		{
		  auto __as32bits = _mm512_maskz_mov_epi32(__k, ~__m512i());
		  return __vector_bitcast<_Up>(
		    __xzyw(_mm256_packs_epi32(__lo256(__as32bits),
					      __hi256(__as32bits))));
		}
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 64)
	    return __vector_bitcast<_Up>(_mm512_movm_epi16(__k));
	}
      else if constexpr (sizeof(_Up) == 4)
	{
	  if constexpr (sizeof(_UI) == 16)
	    {
	      if constexpr (__have_avx512dq_vl)
		return __intrin_bitcast<_UV>(_mm_movm_epi32(__k));
	      else if constexpr (__have_avx512dq)
		return __intrin_bitcast<_UV>(__lo128(_mm512_movm_epi32(__k)));
	      else if constexpr (__have_avx512vl)
		return __intrin_bitcast<_UV>(
		  _mm_maskz_mov_epi32(__k, ~__m128i()));
	      else if constexpr (__have_avx512f)
		return __intrin_bitcast<_UV>(
		  __lo128(_mm512_maskz_mov_epi32(__k, ~__m512i())));
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 32)
	    {
	      if constexpr (__have_avx512dq_vl)
		return __vector_bitcast<_Up>(_mm256_movm_epi32(__k));
	      else if constexpr (__have_avx512dq)
		return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi32(__k)));
	      else if constexpr (__have_avx512vl)
		return __vector_bitcast<_Up>(
		  _mm256_maskz_mov_epi32(__k, ~__m256i()));
	      else if constexpr (__have_avx512f)
		return __vector_bitcast<_Up>(
		  __lo256(_mm512_maskz_mov_epi32(__k, ~__m512i())));
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 64)
	    return __vector_bitcast<_Up>(
	      __have_avx512dq ? _mm512_movm_epi32(__k)
			      : _mm512_maskz_mov_epi32(__k, ~__m512i()));
	}
      else if constexpr (sizeof(_Up) == 8)
	{
	  if constexpr (sizeof(_UI) == 16)
	    {
	      if constexpr (__have_avx512dq_vl)
		return __vector_bitcast<_Up>(_mm_movm_epi64(__k));
	      else if constexpr (__have_avx512dq)
		return __vector_bitcast<_Up>(__lo128(_mm512_movm_epi64(__k)));
	      else if constexpr (__have_avx512vl)
		return __vector_bitcast<_Up>(
		  _mm_maskz_mov_epi64(__k, ~__m128i()));
	      else if constexpr (__have_avx512f)
		return __vector_bitcast<_Up>(
		  __lo128(_mm512_maskz_mov_epi64(__k, ~__m512i())));
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 32)
	    {
	      if constexpr (__have_avx512dq_vl)
		return __vector_bitcast<_Up>(_mm256_movm_epi64(__k));
	      else if constexpr (__have_avx512dq)
		return __vector_bitcast<_Up>(__lo256(_mm512_movm_epi64(__k)));
	      else if constexpr (__have_avx512vl)
		return __vector_bitcast<_Up>(
		  _mm256_maskz_mov_epi64(__k, ~__m256i()));
	      else if constexpr (__have_avx512f)
		return __vector_bitcast<_Up>(
		  __lo256(_mm512_maskz_mov_epi64(__k, ~__m512i())));
	      // else fall through
	    }
	  else if constexpr (sizeof(_UI) == 64)
	    return __vector_bitcast<_Up>(
	      __have_avx512dq ? _mm512_movm_epi64(__k)
			      : _mm512_maskz_mov_epi64(__k, ~__m512i()));
	}

      using _UpUInt = make_unsigned_t<_Up>;
      using _V = __vector_type_t<_UpUInt, _ToN>;
      constexpr size_t __bits_per_element = sizeof(_Up) * __CHAR_BIT__;
      if constexpr (_ToN == 2)
	{
	  return __vector_bitcast<_Up>(_V{_UpUInt(-__x[0]), _UpUInt(-__x[1])});
	}
      else if constexpr (!__have_avx2 && __have_avx && sizeof(_V) == 32)
	{
	  if constexpr (sizeof(_Up) == 4)
	    return __vector_bitcast<_Up>(_mm256_cmp_ps(
	      _mm256_and_ps(_mm256_castsi256_ps(_mm256_set1_epi32(__k)),
			    _mm256_castsi256_ps(_mm256_setr_epi32(
			      0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80))),
	      _mm256_setzero_ps(), _CMP_NEQ_UQ));
	  else if constexpr (sizeof(_Up) == 8)
	    return __vector_bitcast<_Up>(_mm256_cmp_pd(
	      _mm256_and_pd(_mm256_castsi256_pd(_mm256_set1_epi64x(__k)),
			    _mm256_castsi256_pd(
			      _mm256_setr_epi64x(0x01, 0x02, 0x04, 0x08))),
	      _mm256_setzero_pd(), _CMP_NEQ_UQ));
	  else
	    __assert_unreachable<_Up>();
	}
      else if constexpr (__bits_per_element >= _ToN)
	{
	  constexpr auto __bitmask
	    = __generate_vector<_V>([](auto __i) constexpr->_UpUInt {
		return __i < _ToN ? 1ull << __i : 0;
	      });
	  const auto __bits
	    = __vector_broadcast<_ToN, _UpUInt>(__k) & __bitmask;
	  if constexpr (__bits_per_element > _ToN)
	    return __vector_bitcast<_Up>(__bits) > 0;
	  else
	    return __vector_bitcast<_Up>(__bits != 0);
	}
      else
	{
	  const _V __tmp
	    = __generate_vector<_V>([&](auto __i) constexpr {
		return static_cast<_UpUInt>(
		  __k >> (__bits_per_element * (__i / __bits_per_element)));
	      })
	      & __generate_vector<_V>([](auto __i) constexpr {
		  return static_cast<_UpUInt>(1ull
					      << (__i % __bits_per_element));
		}); // mask bit index
	  return __intrin_bitcast<_UV>(__tmp != _V());
	}
    }

  // }}}
  // _S_to_maskvector(_SimdWrapper) {{{
  template <typename _Up, size_t _UpN = 0, typename _Tp, size_t _Np,
	    size_t _ToN = _UpN == 0 ? _Np : _UpN>
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Up, _ToN>
    _S_to_maskvector(_SimdWrapper<_Tp, _Np> __x)
    {
      static_assert(is_same_v<_Up, __int_for_sizeof_t<_Up>>);
      using _TW = _SimdWrapper<_Tp, _Np>;
      using _UW = _SimdWrapper<_Up, _ToN>;
      using _UI = __intrinsic_type_t<_Up, _ToN>;
      if constexpr (is_same_v<_Tp, bool>) // bits -> vector
	return _S_to_maskvector<_Up, _ToN>(
	  _BitMask<_Np>(__x._M_data)._M_sanitized());
      // vector -> vector bitcast
      else if constexpr (sizeof(_Up) == sizeof(_Tp)
			 && sizeof(_TW) == sizeof(_UW))
	return __wrapper_bitcast<_Up, _ToN>(
	  _ToN <= _Np
	    ? __x
	    : simd_abi::_VecBuiltin<sizeof(_Tp) * _Np>::_S_masked(__x));
      else // vector -> vector {{{
	{
	  if (__x._M_is_constprop() || __builtin_is_constant_evaluated())
	    {
	      const auto __y = __vector_bitcast<__int_for_sizeof_t<_Tp>>(__x);
	      return __generate_from_n_evaluations<std::min(_ToN, _Np),
						   __vector_type_t<_Up, _ToN>>(
		[&](auto __i) -> _Up { return __y[__i.value]; });
	    }
	  using _To = __vector_type_t<_Up, _ToN>;
	  [[maybe_unused]] constexpr size_t _FromN = _Np;
	  constexpr int _FromBytes = sizeof(_Tp);
	  constexpr int _ToBytes = sizeof(_Up);
	  const auto __k = __x._M_data;

	  if constexpr (_FromBytes == _ToBytes)
	    return __intrin_bitcast<_To>(__k);
	  else if constexpr (sizeof(_UI) == 16 && sizeof(__k) == 16)
	    { // SSE -> SSE {{{
	      if constexpr (_FromBytes == 4 && _ToBytes == 8)
		return __intrin_bitcast<_To>(__interleave128_lo(__k, __k));
	      else if constexpr (_FromBytes == 2 && _ToBytes == 8)
		{
		  const auto __y
		    = __vector_bitcast<int>(__interleave128_lo(__k, __k));
		  return __intrin_bitcast<_To>(__interleave128_lo(__y, __y));
		}
	      else if constexpr (_FromBytes == 1 && _ToBytes == 8)
		{
		  auto __y
		    = __vector_bitcast<short>(__interleave128_lo(__k, __k));
		  auto __z
		    = __vector_bitcast<int>(__interleave128_lo(__y, __y));
		  return __intrin_bitcast<_To>(__interleave128_lo(__z, __z));
		}
	      else if constexpr (_FromBytes == 8 && _ToBytes == 4
				 && __have_sse2)
		return __intrin_bitcast<_To>(
		  _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()));
	      else if constexpr (_FromBytes == 8 && _ToBytes == 4)
		return __vector_shuffle<1, 3, 6, 7>(__vector_bitcast<_Up>(__k),
						    _UI());
	      else if constexpr (_FromBytes == 2 && _ToBytes == 4)
		return __intrin_bitcast<_To>(__interleave128_lo(__k, __k));
	      else if constexpr (_FromBytes == 1 && _ToBytes == 4)
		{
		  const auto __y
		    = __vector_bitcast<short>(__interleave128_lo(__k, __k));
		  return __intrin_bitcast<_To>(__interleave128_lo(__y, __y));
		}
	      else if constexpr (_FromBytes == 8 && _ToBytes == 2)
		{
		  if constexpr (__have_sse2 && !__have_ssse3)
		    return __intrin_bitcast<_To>(_mm_packs_epi32(
		      _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()),
		      __m128i()));
		  else
		    return __intrin_bitcast<_To>(
		      __vector_permute<3, 7, -1, -1, -1, -1, -1, -1>(
			__vector_bitcast<_Up>(__k)));
		}
	      else if constexpr (_FromBytes == 4 && _ToBytes == 2)
		return __intrin_bitcast<_To>(
		  _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i()));
	      else if constexpr (_FromBytes == 1 && _ToBytes == 2)
		return __intrin_bitcast<_To>(__interleave128_lo(__k, __k));
	      else if constexpr (_FromBytes == 8 && _ToBytes == 1
				 && __have_ssse3)
		return __intrin_bitcast<_To>(
		  _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
				   _mm_setr_epi8(7, 15, -1, -1, -1, -1, -1, -1,
						 -1, -1, -1, -1, -1, -1, -1,
						 -1)));
	      else if constexpr (_FromBytes == 8 && _ToBytes == 1)
		{
		  auto __y
		    = _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i());
		  __y = _mm_packs_epi32(__y, __m128i());
		  return __intrin_bitcast<_To>(_mm_packs_epi16(__y, __m128i()));
		}
	      else if constexpr (_FromBytes == 4 && _ToBytes == 1
				 && __have_ssse3)
		return __intrin_bitcast<_To>(
		  _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
				   _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1,
						 -1, -1, -1, -1, -1, -1, -1,
						 -1)));
	      else if constexpr (_FromBytes == 4 && _ToBytes == 1)
		{
		  const auto __y
		    = _mm_packs_epi32(__vector_bitcast<_LLong>(__k), __m128i());
		  return __intrin_bitcast<_To>(_mm_packs_epi16(__y, __m128i()));
		}
	      else if constexpr (_FromBytes == 2 && _ToBytes == 1)
		return __intrin_bitcast<_To>(
		  _mm_packs_epi16(__vector_bitcast<_LLong>(__k), __m128i()));
	      else
		__assert_unreachable<_Tp>();
	    } // }}}
	  else if constexpr (sizeof(_UI) == 32 && sizeof(__k) == 32)
	    { // AVX -> AVX {{{
	      if constexpr (_FromBytes == _ToBytes)
		__assert_unreachable<_Tp>();
	      else if constexpr (_FromBytes == _ToBytes * 2)
		{
		  const auto __y = __vector_bitcast<_LLong>(__k);
		  return __intrin_bitcast<_To>(_mm256_castsi128_si256(
		    _mm_packs_epi16(__lo128(__y), __hi128(__y))));
		}
	      else if constexpr (_FromBytes == _ToBytes * 4)
		{
		  const auto __y = __vector_bitcast<_LLong>(__k);
		  return __intrin_bitcast<_To>(_mm256_castsi128_si256(
		    _mm_packs_epi16(_mm_packs_epi16(__lo128(__y), __hi128(__y)),
				    __m128i())));
		}
	      else if constexpr (_FromBytes == _ToBytes * 8)
		{
		  const auto __y = __vector_bitcast<_LLong>(__k);
		  return __intrin_bitcast<_To>(
		    _mm256_castsi128_si256(_mm_shuffle_epi8(
		      _mm_packs_epi16(__lo128(__y), __hi128(__y)),
		      _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1,
				    -1, -1, -1, -1, -1))));
		}
	      else if constexpr (_FromBytes * 2 == _ToBytes)
		{
		  auto __y = __xzyw(__to_intrin(__k));
		  if constexpr (is_floating_point_v<
				  _Tp> || (!__have_avx2 && _FromBytes == 4))
		    {
		      const auto __yy = __vector_bitcast<float>(__y);
		      return __intrin_bitcast<_To>(
			_mm256_unpacklo_ps(__yy, __yy));
		    }
		  else
		    return __intrin_bitcast<_To>(
		      _mm256_unpacklo_epi8(__y, __y));
		}
	      else if constexpr (_FromBytes * 4 == _ToBytes)
		{
		  auto __y
		    = _mm_unpacklo_epi8(__lo128(__vector_bitcast<_LLong>(__k)),
					__lo128(__vector_bitcast<_LLong>(
					  __k))); // drops 3/4 of input
		  return __intrin_bitcast<_To>(
		    __concat(_mm_unpacklo_epi16(__y, __y),
			     _mm_unpackhi_epi16(__y, __y)));
		}
	      else if constexpr (_FromBytes == 1 && _ToBytes == 8)
		{
		  auto __y
		    = _mm_unpacklo_epi8(__lo128(__vector_bitcast<_LLong>(__k)),
					__lo128(__vector_bitcast<_LLong>(
					  __k))); // drops 3/4 of input
		  __y
		    = _mm_unpacklo_epi16(__y,
					 __y); // drops another 1/2 => 7/8 total
		  return __intrin_bitcast<_To>(
		    __concat(_mm_unpacklo_epi32(__y, __y),
			     _mm_unpackhi_epi32(__y, __y)));
		}
	      else
		__assert_unreachable<_Tp>();
	    } // }}}
	  else if constexpr (sizeof(_UI) == 32 && sizeof(__k) == 16)
	    { // SSE -> AVX {{{
	      if constexpr (_FromBytes == _ToBytes)
		return __intrin_bitcast<_To>(
		  __intrinsic_type_t<_Tp, 32 / sizeof(_Tp)>(
		    __zero_extend(__to_intrin(__k))));
	      else if constexpr (_FromBytes * 2 == _ToBytes)
		{ // keep all
		  return __intrin_bitcast<_To>(
		    __concat(_mm_unpacklo_epi8(__vector_bitcast<_LLong>(__k),
					       __vector_bitcast<_LLong>(__k)),
			     _mm_unpackhi_epi8(__vector_bitcast<_LLong>(__k),
					       __vector_bitcast<_LLong>(__k))));
		}
	      else if constexpr (_FromBytes * 4 == _ToBytes)
		{
		  if constexpr (__have_avx2)
		    {
		      return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
			__concat(__vector_bitcast<_LLong>(__k),
				 __vector_bitcast<_LLong>(__k)),
			_mm256_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3,
					 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6,
					 6, 6, 7, 7, 7, 7)));
		    }
		  else
		    {
		      return __intrin_bitcast<_To>(__concat(
			_mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
					 _mm_setr_epi8(0, 0, 0, 0, 1, 1, 1, 1,
						       2, 2, 2, 2, 3, 3, 3, 3)),
			_mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
					 _mm_setr_epi8(4, 4, 4, 4, 5, 5, 5, 5,
						       6, 6, 6, 6, 7, 7, 7,
						       7))));
		    }
		}
	      else if constexpr (_FromBytes * 8 == _ToBytes)
		{
		  if constexpr (__have_avx2)
		    {
		      return __intrin_bitcast<_To>(_mm256_shuffle_epi8(
			__concat(__vector_bitcast<_LLong>(__k),
				 __vector_bitcast<_LLong>(__k)),
			_mm256_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
					 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3,
					 3, 3, 3, 3, 3, 3)));
		    }
		  else
		    {
		      return __intrin_bitcast<_To>(__concat(
			_mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
					 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0,
						       1, 1, 1, 1, 1, 1, 1, 1)),
			_mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
					 _mm_setr_epi8(2, 2, 2, 2, 2, 2, 2, 2,
						       3, 3, 3, 3, 3, 3, 3,
						       3))));
		    }
		}
	      else if constexpr (_FromBytes == _ToBytes * 2)
		return __intrin_bitcast<_To>(__m256i(__zero_extend(
		  _mm_packs_epi16(__vector_bitcast<_LLong>(__k), __m128i()))));
	      else if constexpr (_FromBytes == 8 && _ToBytes == 2)
		{
		  return __intrin_bitcast<_To>(__m256i(__zero_extend(
		    _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
				     _mm_setr_epi8(6, 7, 14, 15, -1, -1, -1, -1,
						   -1, -1, -1, -1, -1, -1, -1,
						   -1)))));
		}
	      else if constexpr (_FromBytes == 4 && _ToBytes == 1)
		{
		  return __intrin_bitcast<_To>(__m256i(__zero_extend(
		    _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
				     _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1,
						   -1, -1, -1, -1, -1, -1, -1,
						   -1)))));
		}
	      else if constexpr (_FromBytes == 8 && _ToBytes == 1)
		{
		  return __intrin_bitcast<_To>(__m256i(__zero_extend(
		    _mm_shuffle_epi8(__vector_bitcast<_LLong>(__k),
				     _mm_setr_epi8(7, 15, -1, -1, -1, -1, -1,
						   -1, -1, -1, -1, -1, -1, -1,
						   -1, -1)))));
		}
	      else
		static_assert(!is_same_v<_Tp, _Tp>, "should be unreachable");
	    } // }}}
	  else if constexpr (sizeof(_UI) == 16 && sizeof(__k) == 32)
	    { // AVX -> SSE {{{
	      if constexpr (_FromBytes == _ToBytes)
		{ // keep low 1/2
		  return __intrin_bitcast<_To>(__lo128(__k));
		}
	      else if constexpr (_FromBytes == _ToBytes * 2)
		{ // keep all
		  auto __y = __vector_bitcast<_LLong>(__k);
		  return __intrin_bitcast<_To>(
		    _mm_packs_epi16(__lo128(__y), __hi128(__y)));
		}
	      else if constexpr (_FromBytes == _ToBytes * 4)
		{ // add 1/2 undef
		  auto __y = __vector_bitcast<_LLong>(__k);
		  return __intrin_bitcast<_To>(
		    _mm_packs_epi16(_mm_packs_epi16(__lo128(__y), __hi128(__y)),
				    __m128i()));
		}
	      else if constexpr (_FromBytes == 8 && _ToBytes == 1)
		{ // add 3/4 undef
		  auto __y = __vector_bitcast<_LLong>(__k);
		  return __intrin_bitcast<_To>(_mm_shuffle_epi8(
		    _mm_packs_epi16(__lo128(__y), __hi128(__y)),
		    _mm_setr_epi8(3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1,
				  -1, -1, -1, -1)));
		}
	      else if constexpr (_FromBytes * 2 == _ToBytes)
		{ // keep low 1/4
		  auto __y = __lo128(__vector_bitcast<_LLong>(__k));
		  return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__y, __y));
		}
	      else if constexpr (_FromBytes * 4 == _ToBytes)
		{ // keep low 1/8
		  auto __y = __lo128(__vector_bitcast<_LLong>(__k));
		  __y = _mm_unpacklo_epi8(__y, __y);
		  return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__y, __y));
		}
	      else if constexpr (_FromBytes * 8 == _ToBytes)
		{ // keep low 1/16
		  auto __y = __lo128(__vector_bitcast<_LLong>(__k));
		  __y = _mm_unpacklo_epi8(__y, __y);
		  __y = _mm_unpacklo_epi8(__y, __y);
		  return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__y, __y));
		}
	      else
		static_assert(!is_same_v<_Tp, _Tp>, "should be unreachable");
	    } // }}}
	  else
	    return _Base::template _S_to_maskvector<_Up, _ToN>(__x);
	  /*
	  if constexpr (_FromBytes > _ToBytes) {
	      const _To     __y      = __vector_bitcast<_Up>(__k);
	      return [&] <size_t... _Is> (index_sequence<_Is...>) {
		constexpr int _Stride = _FromBytes / _ToBytes;
		return _To{__y[(_Is + 1) * _Stride - 1]...};
	      }(make_index_sequence<std::min(_ToN, _FromN)>());
	  } else {
	      // {0, 0, 1, 1} (_Dups = 2, _Is<4>)
	      // {0, 0, 0, 0, 1, 1, 1, 1} (_Dups = 4, _Is<8>)
	      // {0, 0, 1, 1, 2, 2, 3, 3} (_Dups = 2, _Is<8>)
	      // ...
	      return [&] <size_t... _Is> (index_sequence<_Is...>) {
		constexpr int __dup = _ToBytes / _FromBytes;
		return __intrin_bitcast<_To>(_From{__k[_Is / __dup]...});
	      }(make_index_sequence<_FromN>());
	  }
	  */
	} // }}}
    }

  // }}}
  // _S_to_bits {{{
  template <typename _Tp, size_t _Np>
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
    _S_to_bits(_SimdWrapper<_Tp, _Np> __x)
    {
      if constexpr (is_same_v<_Tp, bool>)
	return _BitMask<_Np>(__x._M_data)._M_sanitized();
      else
	{
	  static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>);
	  if (__builtin_is_constant_evaluated()
	      || __builtin_constant_p(__x._M_data))
	    {
	      const auto __bools = -__x._M_data;
	      const _ULLong __k = __call_with_n_evaluations<_Np>(
		[](auto... __bits) { return (__bits | ...); },
		[&](auto __i) { return _ULLong(__bools[+__i]) << __i; });
	      if (__builtin_is_constant_evaluated()
		  || __builtin_constant_p(__k))
		return __k;
	    }
	  const auto __xi = __to_intrin(__x);
	  if constexpr (sizeof(_Tp) == 1)
	    if constexpr (sizeof(__xi) == 16)
	      if constexpr (__have_avx512bw_vl)
		return _BitMask<_Np>(_mm_movepi8_mask(__xi));
	      else // implies SSE2
		return _BitMask<_Np>(_mm_movemask_epi8(__xi));
	    else if constexpr (sizeof(__xi) == 32)
	      if constexpr (__have_avx512bw_vl)
		return _BitMask<_Np>(_mm256_movepi8_mask(__xi));
	      else // implies AVX2
		return _BitMask<_Np>(_mm256_movemask_epi8(__xi));
	    else // implies AVX512BW
	      return _BitMask<_Np>(_mm512_movepi8_mask(__xi));

	  else if constexpr (sizeof(_Tp) == 2)
	    if constexpr (sizeof(__xi) == 16)
	      if constexpr (__have_avx512bw_vl)
		return _BitMask<_Np>(_mm_movepi16_mask(__xi));
	      else if constexpr (__have_avx512bw)
		return _BitMask<_Np>(_mm512_movepi16_mask(__zero_extend(__xi)));
	      else // implies SSE2
		return _BitMask<_Np>(
		  _mm_movemask_epi8(_mm_packs_epi16(__xi, __m128i())));
	    else if constexpr (sizeof(__xi) == 32)
	      if constexpr (__have_avx512bw_vl)
		return _BitMask<_Np>(_mm256_movepi16_mask(__xi));
	      else if constexpr (__have_avx512bw)
		return _BitMask<_Np>(_mm512_movepi16_mask(__zero_extend(__xi)));
	      else // implies SSE2
		return _BitMask<_Np>(_mm_movemask_epi8(
		  _mm_packs_epi16(__lo128(__xi), __hi128(__xi))));
	    else // implies AVX512BW
	      return _BitMask<_Np>(_mm512_movepi16_mask(__xi));

	  else if constexpr (sizeof(_Tp) == 4)
	    if constexpr (sizeof(__xi) == 16)
	      if constexpr (__have_avx512dq_vl)
		return _BitMask<_Np>(_mm_movepi32_mask(__xi));
	      else if constexpr (__have_avx512vl)
		return _BitMask<_Np>(_mm_cmplt_epi32_mask(__xi, __m128i()));
	      else if constexpr (__have_avx512dq)
		return _BitMask<_Np>(_mm512_movepi32_mask(__zero_extend(__xi)));
	      else if constexpr (__have_avx512f)
		return _BitMask<_Np>(
		  _mm512_cmplt_epi32_mask(__zero_extend(__xi), __m512i()));
	      else // implies SSE
		return _BitMask<_Np>(
		  _mm_movemask_ps(reinterpret_cast<__m128>(__xi)));
	    else if constexpr (sizeof(__xi) == 32)
	      if constexpr (__have_avx512dq_vl)
		return _BitMask<_Np>(_mm256_movepi32_mask(__xi));
	      else if constexpr (__have_avx512dq)
		return _BitMask<_Np>(_mm512_movepi32_mask(__zero_extend(__xi)));
	      else if constexpr (__have_avx512vl)
		return _BitMask<_Np>(_mm256_cmplt_epi32_mask(__xi, __m256i()));
	      else if constexpr (__have_avx512f)
		return _BitMask<_Np>(
		  _mm512_cmplt_epi32_mask(__zero_extend(__xi), __m512i()));
	      else // implies AVX
		return _BitMask<_Np>(
		  _mm256_movemask_ps(reinterpret_cast<__m256>(__xi)));
	    else // implies AVX512??
	      if constexpr (__have_avx512dq)
	      return _BitMask<_Np>(_mm512_movepi32_mask(__xi));
	    else // implies AVX512F
	      return _BitMask<_Np>(_mm512_cmplt_epi32_mask(__xi, __m512i()));

	  else if constexpr (sizeof(_Tp) == 8)
	    if constexpr (sizeof(__xi) == 16)
	      if constexpr (__have_avx512dq_vl)
		return _BitMask<_Np>(_mm_movepi64_mask(__xi));
	      else if constexpr (__have_avx512dq)
		return _BitMask<_Np>(_mm512_movepi64_mask(__zero_extend(__xi)));
	      else if constexpr (__have_avx512vl)
		return _BitMask<_Np>(_mm_cmplt_epi64_mask(__xi, __m128i()));
	      else if constexpr (__have_avx512f)
		return _BitMask<_Np>(
		  _mm512_cmplt_epi64_mask(__zero_extend(__xi), __m512i()));
	      else // implies SSE2
		return _BitMask<_Np>(
		  _mm_movemask_pd(reinterpret_cast<__m128d>(__xi)));
	    else if constexpr (sizeof(__xi) == 32)
	      if constexpr (__have_avx512dq_vl)
		return _BitMask<_Np>(_mm256_movepi64_mask(__xi));
	      else if constexpr (__have_avx512dq)
		return _BitMask<_Np>(_mm512_movepi64_mask(__zero_extend(__xi)));
	      else if constexpr (__have_avx512vl)
		return _BitMask<_Np>(_mm256_cmplt_epi64_mask(__xi, __m256i()));
	      else if constexpr (__have_avx512f)
		return _BitMask<_Np>(
		  _mm512_cmplt_epi64_mask(__zero_extend(__xi), __m512i()));
	      else // implies AVX
		return _BitMask<_Np>(
		  _mm256_movemask_pd(reinterpret_cast<__m256d>(__xi)));
	    else // implies AVX512??
	      if constexpr (__have_avx512dq)
	      return _BitMask<_Np>(_mm512_movepi64_mask(__xi));
	    else // implies AVX512F
	      return _BitMask<_Np>(_mm512_cmplt_epi64_mask(__xi, __m512i()));

	  else
	    __assert_unreachable<_Tp>();
	}
    }
  // }}}
};

// }}}
// _MaskImplX86 {{{
template <typename _Abi, typename>
  struct _MaskImplX86 : _MaskImplX86Mixin, _MaskImplBuiltin<_Abi>
  {
    using _MaskImplX86Mixin::_S_to_bits;
    using _MaskImplX86Mixin::_S_to_maskvector;
    using _MaskImplBuiltin<_Abi>::_S_convert;

    // member types {{{
    template <typename _Tp>
      using _SimdMember = typename _Abi::template __traits<_Tp>::_SimdMember;

    template <typename _Tp>
      using _MaskMember = typename _Abi::template _MaskMember<_Tp>;

    template <typename _Tp>
      static constexpr size_t _S_size = simd_size_v<_Tp, _Abi>;

    using _Base = _MaskImplBuiltin<_Abi>;

    // }}}
    // _S_broadcast {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
      _S_broadcast(bool __x)
      {
	if constexpr (__is_avx512_abi<_Abi>())
	  return __x ? _Abi::_S_masked(_MaskMember<_Tp>(-1))
		     : _MaskMember<_Tp>();
	else
	  return _Base::template _S_broadcast<_Tp>(__x);
      }

    // }}}
    // _S_load {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _MaskMember<_Tp>
      _S_load(const bool* __mem)
      {
	static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>);
	if constexpr (__have_avx512bw)
	  {
	    const auto __to_vec_or_bits = [](auto __bits) -> decltype(auto) {
	      if constexpr (__is_avx512_abi<_Abi>())
		return __bits;
	      else
		return _S_to_maskvector<_Tp>(
		  _BitMask<_S_size<_Tp>>(__bits)._M_sanitized());
	    };

	    if constexpr (_S_size<_Tp> <= 16 && __have_avx512vl)
	      {
		__m128i __a = {};
		__builtin_memcpy(&__a, __mem, _S_size<_Tp>);
		return __to_vec_or_bits(_mm_test_epi8_mask(__a, __a));
	      }
	    else if constexpr (_S_size<_Tp> <= 32 && __have_avx512vl)
	      {
		__m256i __a = {};
		__builtin_memcpy(&__a, __mem, _S_size<_Tp>);
		return __to_vec_or_bits(_mm256_test_epi8_mask(__a, __a));
	      }
	    else if constexpr (_S_size<_Tp> <= 64)
	      {
		__m512i __a = {};
		__builtin_memcpy(&__a, __mem, _S_size<_Tp>);
		return __to_vec_or_bits(_mm512_test_epi8_mask(__a, __a));
	      }
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  {
	    if constexpr (_S_size<_Tp> <= 8)
	      {
		__m128i __a = {};
		__builtin_memcpy(&__a, __mem, _S_size<_Tp>);
		const auto __b = _mm512_cvtepi8_epi64(__a);
		return _mm512_test_epi64_mask(__b, __b);
	      }
	    else if constexpr (_S_size<_Tp> <= 16)
	      {
		__m128i __a = {};
		__builtin_memcpy(&__a, __mem, _S_size<_Tp>);
		const auto __b = _mm512_cvtepi8_epi32(__a);
		return _mm512_test_epi32_mask(__b, __b);
	      }
	    else if constexpr (_S_size<_Tp> <= 32)
	      {
		__m128i __a = {};
		__builtin_memcpy(&__a, __mem, 16);
		const auto __b = _mm512_cvtepi8_epi32(__a);
		__builtin_memcpy(&__a, __mem + 16, _S_size<_Tp> - 16);
		const auto __c = _mm512_cvtepi8_epi32(__a);
		return _mm512_test_epi32_mask(__b, __b)
		       | (_mm512_test_epi32_mask(__c, __c) << 16);
	      }
	    else if constexpr (_S_size<_Tp> <= 64)
	      {
		__m128i __a = {};
		__builtin_memcpy(&__a, __mem, 16);
		const auto __b = _mm512_cvtepi8_epi32(__a);
		__builtin_memcpy(&__a, __mem + 16, 16);
		const auto __c = _mm512_cvtepi8_epi32(__a);
		if constexpr (_S_size<_Tp> <= 48)
		  {
		    __builtin_memcpy(&__a, __mem + 32, _S_size<_Tp> - 32);
		    const auto __d = _mm512_cvtepi8_epi32(__a);
		    return _mm512_test_epi32_mask(__b, __b)
			   | (_mm512_test_epi32_mask(__c, __c) << 16)
			   | (_ULLong(_mm512_test_epi32_mask(__d, __d)) << 32);
		  }
		else
		  {
		    __builtin_memcpy(&__a, __mem + 16, 16);
		    const auto __d = _mm512_cvtepi8_epi32(__a);
		    __builtin_memcpy(&__a, __mem + 32, _S_size<_Tp> - 48);
		    const auto __e = _mm512_cvtepi8_epi32(__a);
		    return _mm512_test_epi32_mask(__b, __b)
			   | (_mm512_test_epi32_mask(__c, __c) << 16)
			   | (_ULLong(_mm512_test_epi32_mask(__d, __d)) << 32)
			   | (_ULLong(_mm512_test_epi32_mask(__e, __e)) << 48);
		  }
	      }
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (sizeof(_Tp) == 8 && _S_size<_Tp> == 2)
	  return __vector_bitcast<_Tp>(
	    __vector_type16_t<int>{-int(__mem[0]), -int(__mem[0]),
				   -int(__mem[1]), -int(__mem[1])});
	else if constexpr (sizeof(_Tp) == 8 && _S_size<_Tp> <= 4 && __have_avx)
	  {
	    int __bool4 = 0;
	    __builtin_memcpy(&__bool4, __mem, _S_size<_Tp>);
	    const auto __k = __to_intrin(
	      (__vector_broadcast<4>(__bool4)
	       & __make_vector<int>(0x1, 0x100, 0x10000,
				    _S_size<_Tp> == 4 ? 0x1000000 : 0))
	      != 0);
	    return __vector_bitcast<_Tp>(
	      __concat(_mm_unpacklo_epi32(__k, __k),
		       _mm_unpackhi_epi32(__k, __k)));
	  }
	else if constexpr (sizeof(_Tp) == 4 && _S_size<_Tp> <= 4)
	  {
	    int __bools = 0;
	    __builtin_memcpy(&__bools, __mem, _S_size<_Tp>);
	    if constexpr (__have_sse2)
	      {
		__m128i __k = _mm_cvtsi32_si128(__bools);
		__k = _mm_cmpgt_epi16(_mm_unpacklo_epi8(__k, __k), __m128i());
		return __vector_bitcast<_Tp, _S_size<_Tp>>(
		  _mm_unpacklo_epi16(__k, __k));
	      }
	    else
	      {
		__m128 __k = _mm_cvtpi8_ps(_mm_cvtsi32_si64(__bools));
		_mm_empty();
		return __vector_bitcast<_Tp, _S_size<_Tp>>(
		  _mm_cmpgt_ps(__k, __m128()));
	      }
	  }
	else if constexpr (sizeof(_Tp) == 4 && _S_size<_Tp> <= 8)
	  {
	    __m128i __k = {};
	    __builtin_memcpy(&__k, __mem, _S_size<_Tp>);
	    __k = _mm_cmpgt_epi16(_mm_unpacklo_epi8(__k, __k), __m128i());
	    return __vector_bitcast<_Tp>(
	      __concat(_mm_unpacklo_epi16(__k, __k),
		       _mm_unpackhi_epi16(__k, __k)));
	  }
	else if constexpr (sizeof(_Tp) == 2 && _S_size<_Tp> <= 16)
	  {
	    __m128i __k = {};
	    __builtin_memcpy(&__k, __mem, _S_size<_Tp>);
	    __k = _mm_cmpgt_epi8(__k, __m128i());
	    if constexpr (_S_size<_Tp> <= 8)
	      return __vector_bitcast<_Tp, _S_size<_Tp>>(
		_mm_unpacklo_epi8(__k, __k));
	    else
	      return __concat(_mm_unpacklo_epi8(__k, __k),
			      _mm_unpackhi_epi8(__k, __k));
	  }
	else
	  return _Base::template _S_load<_Tp>(__mem);
      }

    // }}}
    // _S_from_bitmask{{{
    template <size_t _Np, typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static _MaskMember<_Tp>
      _S_from_bitmask(_SanitizedBitMask<_Np> __bits, _TypeTag<_Tp>)
      {
	static_assert(is_same_v<_Tp, __int_for_sizeof_t<_Tp>>);
	if constexpr (__is_avx512_abi<_Abi>())
	  return __bits._M_to_bits();
	else
	  return _S_to_maskvector<_Tp, _S_size<_Tp>>(__bits);
      }

    // }}}
    // _S_masked_load {{{2
    template <typename _Tp, size_t _Np>
      static inline _SimdWrapper<_Tp, _Np>
      _S_masked_load(_SimdWrapper<_Tp, _Np> __merge,
		     _SimdWrapper<_Tp, _Np> __mask, const bool* __mem) noexcept
      {
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    if constexpr (__have_avx512bw_vl)
	      {
		if constexpr (_Np <= 16)
		  {
		    const auto __a
		      = _mm_mask_loadu_epi8(__m128i(), __mask, __mem);
		    return (__merge & ~__mask) | _mm_test_epi8_mask(__a, __a);
		  }
		else if constexpr (_Np <= 32)
		  {
		    const auto __a
		      = _mm256_mask_loadu_epi8(__m256i(), __mask, __mem);
		    return (__merge & ~__mask)
			   | _mm256_test_epi8_mask(__a, __a);
		  }
		else if constexpr (_Np <= 64)
		  {
		    const auto __a
		      = _mm512_mask_loadu_epi8(__m512i(), __mask, __mem);
		    return (__merge & ~__mask)
			   | _mm512_test_epi8_mask(__a, __a);
		  }
		else
		  __assert_unreachable<_Tp>();
	      }
	    else
	      {
		_BitOps::_S_bit_iteration(__mask, [&](auto __i) {
		  __merge._M_set(__i, __mem[__i]);
		});
		return __merge;
	      }
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 32 && sizeof(_Tp) == 1)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge = _mm256_mask_sub_epi8(__to_intrin(__merge), __k, __m256i(),
					   _mm256_mask_loadu_epi8(__m256i(),
								  __k, __mem));
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 16 && sizeof(_Tp) == 1)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge
	      = _mm_mask_sub_epi8(__vector_bitcast<_LLong>(__merge), __k,
				  __m128i(),
				  _mm_mask_loadu_epi8(__m128i(), __k, __mem));
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 16 && sizeof(_Tp) == 2)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge = _mm256_mask_sub_epi16(
	      __vector_bitcast<_LLong>(__merge), __k, __m256i(),
	      _mm256_cvtepi8_epi16(_mm_mask_loadu_epi8(__m128i(), __k, __mem)));
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 8 && sizeof(_Tp) == 2)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge = _mm_mask_sub_epi16(
	      __vector_bitcast<_LLong>(__merge), __k, __m128i(),
	      _mm_cvtepi8_epi16(_mm_mask_loadu_epi8(__m128i(), __k, __mem)));
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 8 && sizeof(_Tp) == 4)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge = __vector_bitcast<_Tp>(_mm256_mask_sub_epi32(
	      __vector_bitcast<_LLong>(__merge), __k, __m256i(),
	      _mm256_cvtepi8_epi32(
		_mm_mask_loadu_epi8(__m128i(), __k, __mem))));
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 4 && sizeof(_Tp) == 4)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge = __vector_bitcast<_Tp>(_mm_mask_sub_epi32(
	      __vector_bitcast<_LLong>(__merge), __k, __m128i(),
	      _mm_cvtepi8_epi32(_mm_mask_loadu_epi8(__m128i(), __k, __mem))));
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 4 && sizeof(_Tp) == 8)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge = __vector_bitcast<_Tp>(_mm256_mask_sub_epi64(
	      __vector_bitcast<_LLong>(__merge), __k, __m256i(),
	      _mm256_cvtepi8_epi64(
		_mm_mask_loadu_epi8(__m128i(), __k, __mem))));
	  }
	else if constexpr (__have_avx512bw_vl && _Np == 2 && sizeof(_Tp) == 8)
	  {
	    const auto __k = _S_to_bits(__mask)._M_to_bits();
	    __merge = __vector_bitcast<_Tp>(_mm_mask_sub_epi64(
	      __vector_bitcast<_LLong>(__merge), __k, __m128i(),
	      _mm_cvtepi8_epi64(_mm_mask_loadu_epi8(__m128i(), __k, __mem))));
	  }
	else
	  return _Base::_S_masked_load(__merge, __mask, __mem);
	return __merge;
      }

    // _S_store {{{2
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static void _S_store(_SimdWrapper<_Tp, _Np> __v,
						   bool* __mem) noexcept
      {
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    if constexpr (__have_avx512bw_vl)
	      _CommonImplX86::_S_store<_Np>(
		__vector_bitcast<char>([](auto __data) {
		  if constexpr (_Np <= 16)
		    return _mm_maskz_set1_epi8(__data, 1);
		  else if constexpr (_Np <= 32)
		    return _mm256_maskz_set1_epi8(__data, 1);
		  else
		    return _mm512_maskz_set1_epi8(__data, 1);
		}(__v._M_data)),
		__mem);
	    else if constexpr (_Np <= 8)
	      _CommonImplX86::_S_store<_Np>(
		__vector_bitcast<char>(
#if defined __x86_64__
		  __make_wrapper<_ULLong>(
		    _pdep_u64(__v._M_data, 0x0101010101010101ULL), 0ull)
#else
		  __make_wrapper<_UInt>(_pdep_u32(__v._M_data, 0x01010101U),
					_pdep_u32(__v._M_data >> 4,
						  0x01010101U))
#endif
		    ),
		__mem);
	    else if constexpr (_Np <= 16)
	      _mm512_mask_cvtepi32_storeu_epi8(
		__mem, 0xffffu >> (16 - _Np),
		_mm512_maskz_set1_epi32(__v._M_data, 1));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (__is_sse_abi<_Abi>()) //{{{
	  {
	    if constexpr (_Np == 2 && sizeof(_Tp) == 8)
	      {
		const auto __k = __vector_bitcast<int>(__v);
		__mem[0] = -__k[1];
		__mem[1] = -__k[3];
	      }
	    else if constexpr (_Np <= 4 && sizeof(_Tp) == 4)
	      {
		if constexpr (__have_sse2)
		  {
		    const unsigned __bool4
		      = __vector_bitcast<_UInt>(_mm_packs_epi16(
			  _mm_packs_epi32(__intrin_bitcast<__m128i>(
					    __to_intrin(__v)),
					  __m128i()),
			  __m128i()))[0]
			& 0x01010101u;
		    __builtin_memcpy(__mem, &__bool4, _Np);
		  }
		else if constexpr (__have_mmx)
		  {
		    const __m64 __k = _mm_cvtps_pi8(
		      __and(__to_intrin(__v), _mm_set1_ps(1.f)));
		    __builtin_memcpy(__mem, &__k, _Np);
		    _mm_empty();
		  }
		else
		  return _Base::_S_store(__v, __mem);
	      }
	    else if constexpr (_Np <= 8 && sizeof(_Tp) == 2)
	      {
		_CommonImplX86::_S_store<_Np>(
		  __vector_bitcast<char>(_mm_packs_epi16(
		    __to_intrin(__vector_bitcast<_UShort>(__v) >> 15),
		    __m128i())),
		  __mem);
	      }
	    else if constexpr (_Np <= 16 && sizeof(_Tp) == 1)
	      _CommonImplX86::_S_store<_Np>(__v._M_data & 1, __mem);
	    else
	      __assert_unreachable<_Tp>();
	  }                                      // }}}
	else if constexpr (__is_avx_abi<_Abi>()) // {{{
	  {
	    if constexpr (_Np <= 4 && sizeof(_Tp) == 8)
	      {
		auto __k = __intrin_bitcast<__m256i>(__to_intrin(__v));
		int __bool4;
		if constexpr (__have_avx2)
		  __bool4 = _mm256_movemask_epi8(__k);
		else
		  __bool4 = (_mm_movemask_epi8(__lo128(__k))
			     | (_mm_movemask_epi8(__hi128(__k)) << 16));
		__bool4 &= 0x01010101;
		__builtin_memcpy(__mem, &__bool4, _Np);
	      }
	    else if constexpr (_Np <= 8 && sizeof(_Tp) == 4)
	      {
		const auto __k = __intrin_bitcast<__m256i>(__to_intrin(__v));
		const auto __k2
		  = _mm_srli_epi16(_mm_packs_epi16(__lo128(__k), __hi128(__k)),
				   15);
		const auto __k3
		  = __vector_bitcast<char>(_mm_packs_epi16(__k2, __m128i()));
		_CommonImplX86::_S_store<_Np>(__k3, __mem);
	      }
	    else if constexpr (_Np <= 16 && sizeof(_Tp) == 2)
	      {
		if constexpr (__have_avx2)
		  {
		    const auto __x = _mm256_srli_epi16(__to_intrin(__v), 15);
		    const auto __bools = __vector_bitcast<char>(
		      _mm_packs_epi16(__lo128(__x), __hi128(__x)));
		    _CommonImplX86::_S_store<_Np>(__bools, __mem);
		  }
		else
		  {
		    const auto __bools
		      = 1
			& __vector_bitcast<_UChar>(
			  _mm_packs_epi16(__lo128(__to_intrin(__v)),
					  __hi128(__to_intrin(__v))));
		    _CommonImplX86::_S_store<_Np>(__bools, __mem);
		  }
	      }
	    else if constexpr (_Np <= 32 && sizeof(_Tp) == 1)
	      _CommonImplX86::_S_store<_Np>(1 & __v._M_data, __mem);
	    else
	      __assert_unreachable<_Tp>();
	  } // }}}
	else
	  __assert_unreachable<_Tp>();
      }

    // _S_masked_store {{{2
    template <typename _Tp, size_t _Np>
      static inline void
      _S_masked_store(const _SimdWrapper<_Tp, _Np> __v, bool* __mem,
		      const _SimdWrapper<_Tp, _Np> __k) noexcept
      {
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    static_assert(is_same_v<_Tp, bool>);
	    if constexpr (_Np <= 16 && __have_avx512bw_vl)
	      _mm_mask_storeu_epi8(__mem, __k, _mm_maskz_set1_epi8(__v, 1));
	    else if constexpr (_Np <= 16)
	      _mm512_mask_cvtepi32_storeu_epi8(__mem, __k,
					       _mm512_maskz_set1_epi32(__v, 1));
	    else if constexpr (_Np <= 32 && __have_avx512bw_vl)
	      _mm256_mask_storeu_epi8(__mem, __k,
				      _mm256_maskz_set1_epi8(__v, 1));
	    else if constexpr (_Np <= 32 && __have_avx512bw)
	      _mm256_mask_storeu_epi8(__mem, __k,
				      __lo256(_mm512_maskz_set1_epi8(__v, 1)));
	    else if constexpr (_Np <= 64 && __have_avx512bw)
	      _mm512_mask_storeu_epi8(__mem, __k,
				      _mm512_maskz_set1_epi8(__v, 1));
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  _Base::_S_masked_store(__v, __mem, __k);
      }

    // logical and bitwise operators {{{2
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_logical_and(const _SimdWrapper<_Tp, _Np>& __x,
		     const _SimdWrapper<_Tp, _Np>& __y)
      {
	if constexpr (is_same_v<_Tp, bool>)
	  {
	    if constexpr (__have_avx512dq && _Np <= 8)
	      return _kand_mask8(__x._M_data, __y._M_data);
	    else if constexpr (_Np <= 16)
	      return _kand_mask16(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 32)
	      return _kand_mask32(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 64)
	      return _kand_mask64(__x._M_data, __y._M_data);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_logical_and(__x, __y);
      }

    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_logical_or(const _SimdWrapper<_Tp, _Np>& __x,
		    const _SimdWrapper<_Tp, _Np>& __y)
      {
	if constexpr (is_same_v<_Tp, bool>)
	  {
	    if constexpr (__have_avx512dq && _Np <= 8)
	      return _kor_mask8(__x._M_data, __y._M_data);
	    else if constexpr (_Np <= 16)
	      return _kor_mask16(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 32)
	      return _kor_mask32(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 64)
	      return _kor_mask64(__x._M_data, __y._M_data);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_logical_or(__x, __y);
      }

    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_bit_not(const _SimdWrapper<_Tp, _Np>& __x)
      {
	if constexpr (is_same_v<_Tp, bool>)
	  {
	    if constexpr (__have_avx512dq && _Np <= 8)
	      return _kandn_mask8(__x._M_data,
				  _Abi::template __implicit_mask_n<_Np>());
	    else if constexpr (_Np <= 16)
	      return _kandn_mask16(__x._M_data,
				   _Abi::template __implicit_mask_n<_Np>());
	    else if constexpr (__have_avx512bw && _Np <= 32)
	      return _kandn_mask32(__x._M_data,
				   _Abi::template __implicit_mask_n<_Np>());
	    else if constexpr (__have_avx512bw && _Np <= 64)
	      return _kandn_mask64(__x._M_data,
				   _Abi::template __implicit_mask_n<_Np>());
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_bit_not(__x);
      }

    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_bit_and(const _SimdWrapper<_Tp, _Np>& __x,
		 const _SimdWrapper<_Tp, _Np>& __y)
      {
	if constexpr (is_same_v<_Tp, bool>)
	  {
	    if constexpr (__have_avx512dq && _Np <= 8)
	      return _kand_mask8(__x._M_data, __y._M_data);
	    else if constexpr (_Np <= 16)
	      return _kand_mask16(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 32)
	      return _kand_mask32(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 64)
	      return _kand_mask64(__x._M_data, __y._M_data);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_bit_and(__x, __y);
      }

    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_bit_or(const _SimdWrapper<_Tp, _Np>& __x,
		const _SimdWrapper<_Tp, _Np>& __y)
      {
	if constexpr (is_same_v<_Tp, bool>)
	  {
	    if constexpr (__have_avx512dq && _Np <= 8)
	      return _kor_mask8(__x._M_data, __y._M_data);
	    else if constexpr (_Np <= 16)
	      return _kor_mask16(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 32)
	      return _kor_mask32(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 64)
	      return _kor_mask64(__x._M_data, __y._M_data);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_bit_or(__x, __y);
      }

    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _SimdWrapper<_Tp, _Np>
      _S_bit_xor(const _SimdWrapper<_Tp, _Np>& __x,
		 const _SimdWrapper<_Tp, _Np>& __y)
      {
	if constexpr (is_same_v<_Tp, bool>)
	  {
	    if constexpr (__have_avx512dq && _Np <= 8)
	      return _kxor_mask8(__x._M_data, __y._M_data);
	    else if constexpr (_Np <= 16)
	      return _kxor_mask16(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 32)
	      return _kxor_mask32(__x._M_data, __y._M_data);
	    else if constexpr (__have_avx512bw && _Np <= 64)
	      return _kxor_mask64(__x._M_data, __y._M_data);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_bit_xor(__x, __y);
      }

    //}}}2
    // _S_masked_assign{{{
    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_assign(_SimdWrapper<bool, _Np> __k,
		       _SimdWrapper<bool, _Np>& __lhs,
		       _SimdWrapper<bool, _Np> __rhs)
      {
	__lhs._M_data
	  = (~__k._M_data & __lhs._M_data) | (__k._M_data & __rhs._M_data);
      }

    template <size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_assign(_SimdWrapper<bool, _Np> __k,
		       _SimdWrapper<bool, _Np>& __lhs, bool __rhs)
      {
	if (__rhs)
	  __lhs._M_data = __k._M_data | __lhs._M_data;
	else
	  __lhs._M_data = ~__k._M_data & __lhs._M_data;
      }

    using _MaskImplBuiltin<_Abi>::_S_masked_assign;

    //}}}
    // _S_all_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_all_of(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
	  {
	    constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
	    using _TI = __intrinsic_type_t<_Tp, _Np>;
	    const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
	    if constexpr (__have_sse4_1)
	      {
		_GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
		  = _Abi::template _S_implicit_mask_intrin<_Tp>();
		return 0 != __testc(__a, __b);
	      }
	    else if constexpr (is_same_v<_Tp, float>)
	      return (_mm_movemask_ps(__a) & ((1 << _Np) - 1))
		     == (1 << _Np) - 1;
	    else if constexpr (is_same_v<_Tp, double>)
	      return (_mm_movemask_pd(__a) & ((1 << _Np) - 1))
		     == (1 << _Np) - 1;
	    else
	      return (_mm_movemask_epi8(__a) & ((1 << (_Np * sizeof(_Tp))) - 1))
		     == (1 << (_Np * sizeof(_Tp))) - 1;
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  {
	    constexpr auto _Mask = _Abi::template _S_implicit_mask<_Tp>();
	    const auto __kk = __k._M_data._M_data;
	    if constexpr (sizeof(__kk) == 1)
	      {
		if constexpr (__have_avx512dq)
		  return _kortestc_mask8_u8(__kk, _Mask == 0xff
						    ? __kk
						    : __mmask8(~_Mask));
		else
		  return _kortestc_mask16_u8(__kk, __mmask16(~_Mask));
	      }
	    else if constexpr (sizeof(__kk) == 2)
	      return _kortestc_mask16_u8(__kk, _Mask == 0xffff
						 ? __kk
						 : __mmask16(~_Mask));
	    else if constexpr (sizeof(__kk) == 4 && __have_avx512bw)
	      return _kortestc_mask32_u8(__kk, _Mask == 0xffffffffU
						 ? __kk
						 : __mmask32(~_Mask));
	    else if constexpr (sizeof(__kk) == 8 && __have_avx512bw)
	      return _kortestc_mask64_u8(__kk, _Mask == 0xffffffffffffffffULL
						 ? __kk
						 : __mmask64(~_Mask));
	    else
	      __assert_unreachable<_Tp>();
	  }
      }

    // }}}
    // _S_any_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_any_of(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
	  {
	    constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
	    using _TI = __intrinsic_type_t<_Tp, _Np>;
	    const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
	    if constexpr (__have_sse4_1)
	      {
		if constexpr (_Abi::template _S_is_partial<
				_Tp> || sizeof(__k) < 16)
		  {
		    _GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
		      = _Abi::template _S_implicit_mask_intrin<_Tp>();
		    return 0 == __testz(__a, __b);
		  }
		else
		  return 0 == __testz(__a, __a);
	      }
	    else if constexpr (is_same_v<_Tp, float>)
	      return (_mm_movemask_ps(__a) & ((1 << _Np) - 1)) != 0;
	    else if constexpr (is_same_v<_Tp, double>)
	      return (_mm_movemask_pd(__a) & ((1 << _Np) - 1)) != 0;
	    else
	      return (_mm_movemask_epi8(__a) & ((1 << (_Np * sizeof(_Tp))) - 1))
		     != 0;
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  return (__k._M_data._M_data & _Abi::template _S_implicit_mask<_Tp>())
		 != 0;
      }

    // }}}
    // _S_none_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_none_of(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
	  {
	    constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
	    using _TI = __intrinsic_type_t<_Tp, _Np>;
	    const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
	    if constexpr (__have_sse4_1)
	      {
		if constexpr (_Abi::template _S_is_partial<
				_Tp> || sizeof(__k) < 16)
		  {
		    _GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
		      = _Abi::template _S_implicit_mask_intrin<_Tp>();
		    return 0 != __testz(__a, __b);
		  }
		else
		  return 0 != __testz(__a, __a);
	      }
	    else if constexpr (is_same_v<_Tp, float>)
	      return (__movemask(__a) & ((1 << _Np) - 1)) == 0;
	    else if constexpr (is_same_v<_Tp, double>)
	      return (__movemask(__a) & ((1 << _Np) - 1)) == 0;
	    else
	      return (__movemask(__a) & int((1ull << (_Np * sizeof(_Tp))) - 1))
		     == 0;
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  return (__k._M_data._M_data & _Abi::template _S_implicit_mask<_Tp>())
		 == 0;
      }

    // }}}
    // _S_some_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_some_of(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (__is_sse_abi<_Abi>() || __is_avx_abi<_Abi>())
	  {
	    constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
	    using _TI = __intrinsic_type_t<_Tp, _Np>;
	    const _TI __a = reinterpret_cast<_TI>(__to_intrin(__data(__k)));
	    if constexpr (__have_sse4_1)
	      {
		_GLIBCXX_SIMD_USE_CONSTEXPR _TI __b
		  = _Abi::template _S_implicit_mask_intrin<_Tp>();
		return 0 != __testnzc(__a, __b);
	      }
	    else if constexpr (is_same_v<_Tp, float>)
	      {
		constexpr int __allbits = (1 << _Np) - 1;
		const auto __tmp = _mm_movemask_ps(__a) & __allbits;
		return __tmp > 0 && __tmp < __allbits;
	      }
	    else if constexpr (is_same_v<_Tp, double>)
	      {
		constexpr int __allbits = (1 << _Np) - 1;
		const auto __tmp = _mm_movemask_pd(__a) & __allbits;
		return __tmp > 0 && __tmp < __allbits;
	      }
	    else
	      {
		constexpr int __allbits = (1 << (_Np * sizeof(_Tp))) - 1;
		const auto __tmp = _mm_movemask_epi8(__a) & __allbits;
		return __tmp > 0 && __tmp < __allbits;
	      }
	  }
	else if constexpr (__is_avx512_abi<_Abi>())
	  return _S_any_of(__k) && !_S_all_of(__k);
	else
	  __assert_unreachable<_Tp>();
      }

    // }}}
    // _S_popcount {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int _S_popcount(simd_mask<_Tp, _Abi> __k)
      {
	constexpr size_t _Np = simd_size_v<_Tp, _Abi>;
	const auto __kk = _Abi::_S_masked(__k._M_data)._M_data;
	if constexpr (__is_avx512_abi<_Abi>())
	  {
	    if constexpr (_Np > 32)
	      return __builtin_popcountll(__kk);
	    else
	      return __builtin_popcount(__kk);
	  }
	else
	  {
	    if constexpr (__have_popcnt)
	      {
		int __bits
		  = __movemask(__to_intrin(__vector_bitcast<_Tp>(__kk)));
		const int __count = __builtin_popcount(__bits);
		return is_integral_v<_Tp> ? __count / sizeof(_Tp) : __count;
	      }
	    else if constexpr (_Np == 2 && sizeof(_Tp) == 8)
	      {
		const int mask = _mm_movemask_pd(__auto_bitcast(__kk));
		return mask - (mask >> 1);
	      }
	    else if constexpr (_Np <= 4 && sizeof(_Tp) == 8)
	      {
		auto __x = -(__lo128(__kk) + __hi128(__kk));
		return __x[0] + __x[1];
	      }
	    else if constexpr (_Np <= 4 && sizeof(_Tp) == 4)
	      {
		if constexpr (__have_sse2)
		  {
		    __m128i __x = __intrin_bitcast<__m128i>(__to_intrin(__kk));
		    __x = _mm_add_epi32(
		      __x, _mm_shuffle_epi32(__x, _MM_SHUFFLE(0, 1, 2, 3)));
		    __x = _mm_add_epi32(
		      __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(1, 0, 3, 2)));
		    return -_mm_cvtsi128_si32(__x);
		  }
		else
		  return __builtin_popcount(
		    _mm_movemask_ps(__auto_bitcast(__kk)));
	      }
	    else if constexpr (_Np <= 8 && sizeof(_Tp) == 2)
	      {
		auto __x = __to_intrin(__kk);
		__x = _mm_add_epi16(__x,
				    _mm_shuffle_epi32(__x,
						      _MM_SHUFFLE(0, 1, 2, 3)));
		__x = _mm_add_epi16(
		  __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(0, 1, 2, 3)));
		__x = _mm_add_epi16(
		  __x, _mm_shufflelo_epi16(__x, _MM_SHUFFLE(2, 3, 0, 1)));
		return -short(_mm_extract_epi16(__x, 0));
	      }
	    else if constexpr (_Np <= 16 && sizeof(_Tp) == 1)
	      {
		auto __x = __to_intrin(__kk);
		__x = _mm_add_epi8(__x,
				   _mm_shuffle_epi32(__x,
						     _MM_SHUFFLE(0, 1, 2, 3)));
		__x = _mm_add_epi8(__x,
				   _mm_shufflelo_epi16(__x, _MM_SHUFFLE(0, 1, 2,
									3)));
		__x = _mm_add_epi8(__x,
				   _mm_shufflelo_epi16(__x, _MM_SHUFFLE(2, 3, 0,
									1)));
		auto __y = -__vector_bitcast<_UChar>(__x);
		if constexpr (__have_sse4_1)
		  return __y[0] + __y[1];
		else
		  {
		    unsigned __z = _mm_extract_epi16(__to_intrin(__y), 0);
		    return (__z & 0xff) + (__z >> 8);
		  }
	      }
	    else if constexpr (sizeof(__kk) == 32)
	      {
		// The following works only as long as the implementations above
		// use a summation
		using _I = __int_for_sizeof_t<_Tp>;
		const auto __as_int = __vector_bitcast<_I>(__kk);
		_MaskImplX86<simd_abi::__sse>::_S_popcount(
		  simd_mask<_I, simd_abi::__sse>(__private_init,
						 __lo128(__as_int)
						   + __hi128(__as_int)));
	      }
	    else
	      __assert_unreachable<_Tp>();
	  }
      }

    // }}}
    // _S_find_first_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_first_set(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (__is_avx512_abi<_Abi>())
	  return std::__countr_zero(__k._M_data._M_data);
	else
	  return _Base::_S_find_first_set(__k);
      }

    // }}}
    // _S_find_last_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_last_set(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (__is_avx512_abi<_Abi>())
	  return std::__bit_width(__k._M_data._M_data) - 1;
	else
	  return _Base::_S_find_last_set(__k);
      }

    // }}}
  };

// }}}

_GLIBCXX_SIMD_END_NAMESPACE
#endif // __cplusplus >= 201703L
#endif // _GLIBCXX_EXPERIMENTAL_SIMD_X86_H_

// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              m    __fentry__                                              9[    __x86_return_thunk                                      ׂ    __pci_register_driver                                   S?EJ    iowrite32                                                   ioread32                                                k    iowrite16                                               ,    ioread16                                                    msleep                                                  n    __dynamic_dev_dbg                                           __const_udelay                                          V
    __stack_chk_fail                                        4    _raw_spin_lock_irqsave                                  p\    _raw_spin_unlock_irqrestore                             #    __netdev_alloc_skb                                      ^|    page_offset_base                                        le    vmemmap_base                                            ٣    dma_map_page_attrs                                      (L    phys_base                                               #WK    consume_skb                                             o4    dma_unmap_page_attrs                                    ŏW    memset                                                  @    __dev_kfree_skb_any                                     @    unregister_netdev                                           pci_iounmap                                             b    pci_release_regions                                     m    free_netdev                                             C6    pci_disable_device                                      z    kfree                                                   Tn    dma_free_attrs                                          d/    device_set_wakeup_enable                                9d    strscpy                                                     strnlen                                                     fortify_panic                                           J    generic_mii_ioctl                                       (
    dev_addr_mod                                            GV    __warn_printk                                           .7    iowrite8                                                fo    _dev_warn                                               *v    pci_unregister_driver                                   [;i    crc32_le                                                E:#    __kmalloc                                               MxN    dma_alloc_attrs                                         9I    _dev_err                                                R    pci_enable_device                                       G}    dma_set_mask                                            ;䒲    pci_request_regions                                     U<!)    pci_set_master                                          .    alloc_etherdev_mqs                                      3    pci_iomap                                               ,X    netif_napi_add_weight                                       netif_carrier_off                                       9c    init_timer_key                                          xB<    register_netdev                                         	7A    get_random_bytes                                        o2N    _dev_notice                                             ]    _dev_info                                               Ӆ3-    system_wq                                               6    queue_work_on                                           ұ    napi_disable                                            P    jiffies                                                 ܐ    timer_delete_sync                                       u#    synchronize_irq                                         ;JQ    free_irq                                                "T͎    pci_disable_msi                                         BW\    netif_device_detach                                     Y    _dev_printk                                             %Q    pci_wake_from_d3                                        4~    pci_set_power_state                                     0j    ethtool_convert_legacy_u32_to_link_mode                 d    _raw_spin_lock                                          4K    _raw_spin_unlock                                            napi_schedule_prep                                      a*    __napi_schedule                                         o<    disable_irq                                             	    enable_irq                                              y_,    pskb_expand_head                                        >    ___pskb_trim                                            S>    skb_put                                                 y6    eth_type_trans                                               netif_receive_skb                                       \g    __dev_kfree_skb_irq                                     "    napi_complete_done                                      a    netif_tx_wake_queue                                     	    round_jiffies                                               mod_timer                                               I    netif_carrier_on                                        -~    pci_enable_msi                                          Ւ    request_threaded_irq                                    p*    napi_enable                                             "*    netif_device_attach                                     X    eth_validate_addr                                            param_ops_int                                           j    param_array_ops                                         ^    ethtool_op_get_link                                     zR    module_layout                                                   I 		                                                                                                                                                                                rx_packets                            tx_packets                            rx_bytes                               tx_bytes                           (   rx_errors                          @   tx_errors                          x   multicast                          0   collisions                         8   rx_length_errors                   H   rx_over_errors                     p   rx_crc_errors                      P   rx_frame_errors                    `   rx_fifo_errors                     h   rx_missed_errors                   p   tx_aborted_errors                     tx_carrier_errors                     tx_fifo_errors                        tx_window_errors                      tx_abort_exce_coll                    tx_abort_late_coll                    tx_deferred_ok                        tx_single_coll_ok                     tx_multi_coll_ok                      tx_underrun                           tx_trunc                              tx_pause                              rx_pause                              rx_rrd_ov                             rx_trunc                                                                                                                                                                                                                                                                                                                                                                                                                      
                                                     
                                                                                                          |                                                                                                                                                                                                                                                                                                         atl1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     GCC: (Debian 12.2.0-14+deb12u1) 12.2.0  GCC: (Debian 12.2.0-14+deb12u1) 12.2.0                      
9             X    _          
Y A=      h        #       P=        W=    0   _= 
  0   h        %{            @   k=    `   x=      =      =      :"  %     =  
     = ^ @        
        %                              ] = 
     =        = $   @   = $   H   = $   P   = $   X   = $   `    > $   h   	> $   p   > $   x   #> $      2> 2     B> *       H> *       Q> *   @   7` *   `   Z> *      b> *      m> *      x> *      > *      > *      > *   @  > *   `  > *     > *     > *     > *     > *      > *      > *   @  ? *   `  ? *     ? *     0? *     B? *     N? *      T? *      ]? *   @  @` *   `  f? *     s? *     {? *     ? *     ? *      ? *      ? *   @  ? *   `  ? *     ? *     ? *     ? *     ? *      @ *      @ *   @  @ *   `  '@ *     1@ *     <@ *     E@ *     S@ *      a@ *      m@      }@ *       @ &       @ &   0   O &   @   @ &   P            @ &       @ &               v *       @ b     @      @ $       @ $      @ &      @ c     @ &   @   @ &   P   A &   `   EL &   p   A      A +      { +  @   %A &   P   -A      A +      <A +  @   BA +  `   HA      WA    gA    xA    A      A     A    A      A     A    A    A    B    B    +B      s  k       i  W   @           <B             ơ  &   @   ~v &   P   i  W      HB   (   s  k       i  W   @     &        &      S &      VB ]      dB ]      pB m           k |B   (   s  k       i  W   @     &        &      dB ]      VB &      pB m    B       s  k       i  W   @             &      dB &      VB ]      B      B q     i  W   @          a B      B s     i  W   @          ` B      O -       Z -   @   e -      n -       -       -   @  w -      -      -       -   @   -      -      -       -   @  8 -      -     [ -      & -   @  @` -     B -     B -      B -   @  B -     B -     @ -      <@ -   @  7` -     ? -     B -      B )  h   B        FO  w @   B g    C h    C i    C i    !C $       $     .C $     6C $     ;C $      CC $   (  IC $   0  OC $   8  YC $   @  cC $   H  qC $   P  {C $   X  C $   `  C &   p  C &     C &     C &     C &     C &     C &     C &     C &     C &      D *      D &   @  D &   P  0D &   `  DD &   p  WD *     fD *     uD &     }D $     z 	    D 	    D K   8  D   x   %      Z   @   D t    D *     ^ *     D &      D &     4  *     B  f  @  D Q    D Q    E I    E K      $E j @  0E l    9E *  @  AE n   JE o   SE -     _E -      lE *   @  wE &   `  {E &   p  E \   U K      E *      E K   @    u   B r "  B p @#         v                  p          
x            _                E     E    E             ]         b         E                 ?   @            B            @          }          T |       ~     E   0   c  {     V  ?   @    ?             b      E   (   E        E       E                                \        [        u        l        o        n        f        d        e        j         
F       F    p         
    %  ,F           
     %  "
 *        <F           
    %  MF =?  TF           
     %  MF =?  hF           
    %  9.  4?  |F 5?   s	  F           
     %  9.  4?  |F 5?   s	  F           
     %    ?    k   F     F           
     %  ^  *   F           
*    %  F           
    %  ^ !?  F           
     %  ^ !?  F           
     %  G ?  G           
    %  : }?  %G           
    %  : z?  =G           
    %  2"
    UG           
     %  0  I?      iG     G    R  G            
     %  G     G    R  G      G      G     G           
    %  SP    G     G            
    W w H           
(   W w H     H    }Y  .H    N  8H    #       
}	      %  IH    ň       
&   W w YH    ǈ oH     H     H     H           
     %  h    #         =    ͈       
    %  h    #    =    ψ       
       H    ш       
(      \y '  by '  H    ӈ       
(      H    Ո I    Ո       
(      I *   %I &   .I    ؈       
(      I &   %I '  AI    ڈ SI    Ո       
    %      aI    ݈       
   %      sI    ߈       
     %      I     I            
     %  I    I     I           
    %    k   I           
    %  
   :    I     mii_ioctl_data val_in val_out mii_if_info reg_num_mask full_duplex force_media supports_gmii mdio_read mdio_write atlx_spi_flash_dev manu_name cmd_wrsr cmd_read cmd_program cmd_wren cmd_wrdi cmd_rdsr cmd_rdid cmd_sector_erase cmd_chip_erase stats_msg_block rx_ok rx_bcast rx_mcast rx_ctrl rx_fcs_err rx_len_err rx_byte_cnt rx_runt rx_frag rx_sz_64 rx_sz_65_127 rx_sz_128_255 rx_sz_256_511 rx_sz_512_1023 rx_sz_1024_1518 rx_sz_1519_max rx_sz_ov rx_rxf_ov rx_rrd_ov rx_align_err rx_bcast_byte_cnt rx_mcast_byte_cnt rx_err_addr tx_ok tx_bcast tx_mcast tx_exc_defer tx_ctrl tx_defer tx_byte_cnt tx_sz_64 tx_sz_65_127 tx_sz_128_255 tx_sz_256_511 tx_sz_512_1023 tx_sz_1024_1518 tx_sz_1519_max tx_1_col tx_2_col tx_late_col tx_abort_col tx_underrun tx_rd_eop tx_len_err tx_trunc tx_bcast_byte tx_mcast_byte smb_updated coals_msg_block int_stats rrd_prod_idx rfd_cons_idx tpd_cons_idx rx_chksum pkt_size xsum_sz rx_return_desc num_buf resved buf_indx xsz pkt_flg err_flg resved2 rx_free_desc buffer_addr coalese tx_packet_desc word2 word3 atl1_dma_order atl1_dma_ord_in atl1_dma_ord_enh atl1_dma_ord_out atl1_dma_rcb atl1_rcb_64 atl1_rcb_128 atl1_dma_req_block atl1_dma_req_128 atl1_dma_req_256 atl1_dma_req_512 atl1_dma_req_1024 atl1_dma_req_2048 atl1_dma_req_4096 atl1_ring_header atl1_buffer atl1_tpd_ring next_to_clean next_to_use buffer_info atl1_rfd_ring atl1_rrd_ring atl1_cmb cmb atl1_smb smb atl1_sft_stats excecol deffer scc mcc latecol rx_trunc atl1_hw hw_addr dma_ord rcb_value dmar_block dmaw_block preamble_len jam_ipg ipgt min_ifg ipgr1 ipgr2 tpd_burst rfd_burst rfd_fetch_gap rrd_burst tpd_fetch_th tpd_fetch_gap tx_jumbo_task_th txf_burst rx_jumbo_th rx_jumbo_lkah rrd_ret_timer lcol cmb_tpd cmb_rrd cmb_rx_timer cmb_tx_timer smb_timer media_type autoneg_advertised mii_autoneg_adv_reg mii_1000t_ctrl_reg max_frame_size min_frame_size dev_rev flash_vendor perm_mac_addr phy_configured atl1_adapter soft_stats rx_buffer_len link_speed link_duplex reset_dev_task link_chg_task phy_config_timer phy_timer_pending ring_header tpd_ring mb_lock rfd_ring rrd_ring hw_csum_err hw_csum_good msg_enable imt ict mii bd_number pci_using_64 enable_option range_option list_option atl1_opt_list atl1_option atl1_stats stat_string sizeof_stat stat_offset atl1_driver_exit atl1_driver_init atl1_nway_reset atl1_get_strings epause atl1_set_pauseparam atl1_get_pauseparam kernel_ring atl1_set_ringparam atl1_get_ringparam atl1_get_regs atl1_get_regs_len atl1_set_msglevel atl1_get_msglevel atl1_set_wol atl1_get_wol drvinfo atl1_get_drvinfo atl1_set_link_ksettings atl1_get_link_ksettings atl1_get_sset_count atl1_get_ethtool_stats atl1_remove atl1_probe atl1_poll_controller atl1_shutdown atl1_resume atl1_suspend atl1_close atl1_open atl1_change_mtu atl1_reset_dev_task atl1_down atl1_up atl1_phy_config atl1_intr atl1_rings_clean atl1_xmit_frame atl1_alloc_rx_buffers atl1_free_ring_resources atl1_clean_tx_ring atl1_clean_rx_ring atl1_setup_ring_resources atl1_set_mac_addr atl1_get_speed_and_duplex atl1_init_hw atl1_phy_setup_autoneg_adv reg_addr phy_data atl1_write_phy_reg atl1_read_phy_reg atl1_reset_hw atlx_set_features atlx_fix_features atlx_vlan_mode atlx_link_chg_task txqueue atlx_tx_timeout atlx_set_multi atl1_set_mac atlx_ioctl    atl1.ko /X                                                                                                   	                      
                                                                  !                      '                      )                      ,                      <      +            g             -                   :     s             S           	       i           
       }           <                                       $                                        >           !                    `                          !                                           (                  :           ,       M          (       a    @      *       u                      !       (          p      8                 k                           '       8                                            '        8                 Y       '                 :    p      s       J          l       e           
       p           
       {    `                                 p	                 p
                      N                                             `
      P           
      {                          ,    P              8    X              D    0      z       O          P      \   )                n                 {           H          '       8                             ' P      8          '        8                            	                          ,          @      0      &    4       =       E    p      	         !              P    @5            a                 q   ,                          0         ,                   F      7           O      9       0   !                  q       ~                 n                 '                              @                   !            "           "       4     $      B       B    P$      %       M    $             e    p%            o                 ~    +      .           @+      	                T                 R          :                b      \           >      :      	                      F      r       "    F      r       2                   =    @G      g       M    G            `    PM      &      x                      '         8          ' p       8          ' 8       8                 >           O      \           (                  @              
    `                                "                  .   #                Z   %                    @      P                                    -           -                          (           8                 @       -           m       $       (    (       (       >                 X                  r                             p           
      /                                                                                                                                   	                     		                     	                     ,	    @      P       Q	                     Y	                     c	                     o	                     y	   *               	                     	                     	                     	                     	                     	                     	   	                 
                     
                      
                     4
                     A
                     G
                     R
                     \
                     n
                     
                     
                     
                     
                     
                     
                     
                     
                                                             %                     4                     G                     Z                     c                     t                                                                                                                                                                                                                  4                                                                )                     9                     E                     	                     W                     c                     m                                                                                                                                                                                             ;                     
                     
                     
                     $
                     3
                     B
                     V
                     e
                     y
                     
                     
                     
                     
                     
                     
                     
                                          (                     5                     C                     Q                     ]                     n                     u                                                                __UNIQUE_ID_alias194 __UNIQUE_ID_depends193 ____versions __UNIQUE_ID_retpoline192 __UNIQUE_ID_intree191 __UNIQUE_ID_name190 __UNIQUE_ID_vermagic189 _note_10 _note_9 atlx_fix_features atl1_get_ethtool_stats atl1_gstrings_stats atl1_get_sset_count atl1_get_wol atl1_get_msglevel atl1_set_msglevel atl1_get_regs_len atl1_get_ringparam atl1_get_pauseparam atl1_set_pauseparam atl1_driver_init atl1_driver atl1_set_mac_addr atl1_get_regs atl1_reset_hw __UNIQUE_ID_ddebug568.13 atl1_read_phy_reg atl1_get_speed_and_duplex __UNIQUE_ID_ddebug576.6 mdio_read atl1_write_phy_reg atl1_phy_config atl1_phy_setup_autoneg_adv CSWTCH.258 CSWTCH.257 mdio_write atl1_alloc_rx_buffers atl1_clean_rx_ring atl1_clean_tx_ring atl1_get_strings atl1_remove atl1_free_ring_resources atl1_set_wol atl1_get_drvinfo atl1_get_drvinfo.cold __func__.69 __func__.68 atlx_ioctl atl1_set_mac __already_done.14 atl1_init_hw flash_table __UNIQUE_ID_ddebug572.8 CSWTCH.294 __UNIQUE_ID_ddebug570.9 __UNIQUE_ID_ddebug574.7 atl1_init_hw.cold atl1_driver_exit atlx_set_multi atl1_setup_ring_resources atl1_setup_ring_resources.cold atl1_probe atl1_rings_clean atl1_netdev_ops cards_found.64 atl1_ethtool_ops num_int_mod_timer atl1_reset_dev_task atlx_link_chg_task atl1_probe.cold atlx_vlan_mode atlx_set_features atlx_tx_timeout atl1_down atl1_suspend atl1_suspend.cold atl1_shutdown atl1_close atl1_get_link_ksettings atl1_intr atl1_intr.cold atl1_poll_controller atl1_xmit_frame atl1_xmit_frame.cold atl1_rings_clean.cold atl1_check_link.isra.0 atl1_check_link.isra.0.cold atl1_up atl1_up.cold atl1_resume atl1_nway_reset CSWTCH.287 atl1_change_mtu atl1_set_ringparam atl1_set_link_ksettings CSWTCH.386 __UNIQUE_ID_ddebug582.1 __UNIQUE_ID_ddebug578.3 __UNIQUE_ID_ddebug580.2 atl1_set_link_ksettings.cold atl1_open __func__.71 __func__.70 __func__.67 __func__.63 __func__.62 __UNIQUE_ID___addressable_cleanup_module589 __UNIQUE_ID___addressable_init_module588 atl1_pci_tbl atl1_pm_ops __UNIQUE_ID_debug567 __UNIQUE_ID_debugtype566 __param_debug __param_str_debug __UNIQUE_ID_int_mod_timer565 __UNIQUE_ID_int_mod_timertype564 __param_int_mod_timer __param_str_int_mod_timer __param_arr_int_mod_timer __UNIQUE_ID_license563 __UNIQUE_ID_author562 __UNIQUE_ID_description561 .LC26 .LC6 .LC34 alloc_etherdev_mqs free_irq device_set_wakeup_enable ioread32 pci_wake_from_d3 pci_enable_device __mod_pci__atl1_pci_tbl_device_table skb_put iowrite32 consume_skb pci_iomap __this_module netif_napi_add_weight queue_work_on unregister_netdev ioread16 dma_unmap_page_attrs __pci_register_driver cleanup_module pci_disable_msi param_array_ops pci_request_regions ___pskb_trim kfree enable_irq iowrite16 eth_validate_addr timer_delete_sync pci_set_power_state _raw_spin_lock_irqsave __dynamic_dev_dbg _raw_spin_lock pci_unregister_driver fortify_panic __fentry__ init_module pskb_expand_head dev_addr_mod eth_type_trans dma_map_page_attrs napi_complete_done iowrite8 __stack_chk_fail __napi_schedule strnlen netif_device_detach _dev_info netif_device_attach page_offset_base synchronize_irq pci_enable_msi _dev_err request_threaded_irq __dev_kfree_skb_irq crc32_le dma_alloc_attrs napi_enable netif_receive_skb free_netdev phys_base ethtool_op_get_link _raw_spin_unlock_irqrestore pci_iounmap netif_tx_wake_queue memset _dev_warn pci_set_master __x86_return_thunk __netdev_alloc_skb vmemmap_base _dev_notice strscpy dma_free_attrs init_timer_key pci_release_regions __const_udelay __dev_kfree_skb_any __warn_printk netif_carrier_off generic_mii_ioctl netif_carrier_on pci_disable_device dma_set_mask get_random_bytes ethtool_convert_legacy_u32_to_link_mode napi_schedule_prep napi_disable round_jiffies param_ops_int _dev_printk _raw_spin_unlock msleep __kmalloc system_wq disable_irq                              !             (                     =                   Z             a             v                                                                                                                                                          4            A            f            q                                                U            w            !            M            ]            r                                                                                                                	                                 <            L            ]            m                                                5            w                   ~         	                                                                                  "            2            C            S            k            q                                                                                      8            a                        o                                                J	            `	            q	            	            	            
            $
            ]
            q
            
                        C            X            w                                                                                                  B            J            Y            a            i            s            z                                                Q
            a
            
            
            
            
            
                    
            
                                           '            1            d            |                                                            O                     
                                   P                
                                       #            5            O            
      b                                                                                                                                                
                  )                  .            K            	      P            n            ,                    c            s                                        	                                                    X                	                                  &                	   P                              1            _            t                                                                        A            q                        #            C            0       H            V            c            R       q                                                                                                                           +                                                      @5                                    `                         -            3            @                                                      
            H            a            q            {                                    B            m       H                                                                                p                              F      .            O      :            M            W            w                                                                        `            v                                                                        $            9                                    =            I                  d                   i                   n                                                                                                              !             (             9             A             U             c             ~                                                                              	!            !!            o!            !            !            !            "            1"            F"            Y"            l"            "            "            "            "            b#            u#            #            #            #            #                   #            $            /$            >$            Q$            q$            $            $            $            %            '%            d%            q%            %            %            
      M&            {&            (            )            .)            5      J)            ])            o)            v)            )            )            )            )            *            &*            6*            i*            |*            *            ]      *                  *            *            *            *            *            +             +            :+            A+            1-            j-            v-            d.            .            .                  .            X/            /            /            0            0            0            1            01            c3            3            3            4                  4            5            5            )5            A5            N6            k6            6            7            W7            7            7            	8            d8            8            8            Q      8                  I9            2      9            9            9            N:            :            :            :            ;                  ;            ;            R<            a<            <            <            <            =            =            ^      =            >            >            E?            `?            ?            ?            ?            ?            ?            ?            @            =@            W@            o@            @            @            9A            SA            kA            A            A            A            9B            vB            B            B            "C            nC            C            C            C            C            C            D            .D            DD            aD            p%      lD            D            D            D            D            D                  E            LE            _E            E            E            E            E            F            F            8F            AF            eF            nF            F            F            F            F            F            G                    .G            AG            G            G            G            I            QM            
N                    SN            }N                  N            N                  N         	   8       N            N                  O         	   p       O            *O            8O                  ?O         	           KO            rO                  O            O            O            O            O            P                                                                                                                P                                 X                                 :       +             0                   A             m       F             R             C      \                    a             m             C      ~                                                                              B                                                                                                                                                n                                                                        S                                      9                  
            8"                  <      $            9      0            5            %      E            8      L            9      X            ]            .)      n            S      z                                    )                  d                  9                              *                  ~                  9                              .                  `                  9                              #1                        !            9      -            2            8      @            9      G            %      L            Q            6      _                  f            9      r                                          9                                                9                                                9                                                9                                    &            9      2            F                  M            9      Y            ^            8      q            8      x            D                  ^                              =                  P                              ;                  x                              D                                                }N                                                }N                                                                                              `                                                (                    0                    8                    @                   H             @      P                     X             p      `                   h                    p                   x                                                                      p                                      `                                      p	                   p
                                                                            `
                   
                   0                                                                            @                  p                                           (                    0            @       8             !      @             $      H            P$      P            $      X            p%      `            +      h            @+      p            @5      x            :                  >                   F                  F                  F                  @G                  G                  PM                  O                  O                    n                    .                   l<                   D                   O                   P$                   @+                                                                            0      H            @G      X                                +      x                                                   !                   F                   !                   F                   !                   F                                                                           
                                                                          `
                                                         F                          H                   P            G      `                  h            @      x                                                  `       x            $                  PM                                           0                                8                                                                    (                   0                     8                     H                                                     Y                    u                                                                                                                        3      $             e      (             v      ,                   0                   4                   8                   <             4      @                   D             j      H             7      L             I	      P             \
      T             v      X                   \                   `             y      d             P
      h             
      l             
      p                   t             N      x                   |             m                                      "                   U                                                                              0"                   p$                   &%                   z&                   .                   8                   ;                   D                   E                   E                   @F                   mF                   F                   -G                   G                   G                   I                   RN                   N                   )O                   O                                                                                  ^                    `                    z                                                                    $                    (                    ,                   0                   4             8      8             @      <             j      @             p      D             v      H                   L                   P                   T                   X                   \                   `                   d                   h                   l             m      p             n      t             p      x             r      |             t                   v                   {                                                          '                   )                   +                   /                   5                                                                                                                                                                                                                                                                                                                                   (                                                                                                                                                                                                                                                                                      .      $            /      (            0      ,            2      0            4      4            9      8                  <                  @                  D                  H                  L                  P                  T                  X                   \            a      `            b      d            o      h            p      l            w      p                  t                  x                  |                                                                                                                                                            6                  7                  <                  V                  W                  \                  `                  v                                                                                                                                                                  <	                  @	                  A	                  C	                  E	                  G	                  I	                  N	                  i	                   p	                  w	                  y	                  }	                  ~	                  W
                  X
                  Z
                   \
      $            a
      (            e
      ,            p
      0            w
      4            y
      8            z
      <            ~
      @            q      D            r      H            t      L            v      P            {      T                  X                  \                  `                  d                  h                  l            q      p            r      t            w      x            x      |            y                  ~                                                                        O
                  P
                  U
                  `
                  
                  
                  %                  &                  +                  0                  =                  ?                  A                  B                  C                                                                                                                                                                                                                                                             G                  H                  J                  L                  N                  S                                                       	      $                  (            
      ,                  0                  4                  8            `      <            d      @            e      D            g      H            i      L            k      P            m      T            r      X                  \                  `                  d                  h                  l                  p                  t                  x                  |                                                                                                                        <                  @                  G                  M                  N                                                       "                  '                  R                  S                  U                  Z                  p                  w                  y                  {                  }                  ~                                                                                                                                                                                                       |                                                                                                                   $                  (                  ,                  0                  4                  8                   <                    @            =       D            @       H            F       L            G       P            !      T            !      X            !      \             !      `            '!      d            +!      h            -!      l            .!      p            2!      t            6!      x            &"      |            )"                  *"                  ,"                  ."                  0"                  5"                  #                   $                  $                  $                  <$                  =$                  B$                  P$                  V$                  p$                  u$                  $                  $                  $                  $                  $                   %                  #%                  $%                  &%                  +%                  h%                  p%                  w%                  y%                  {%                  }%                   ~%                  %                  %                  n&                  q&                  r&                  t&                  v&                   x&      $            z&      (            &      ,            +      0            +      4            +      8            9+      <            >+      @            @+      D            G+      H            L+      L            N+      P            P+      T            Q+      X            R+      \            V+      `            .      d            .      h            .      l            .      p            .      t            .      x            .      |            /                  25                  @5                  G5                  P5                  U5                  W5                  X5                  Y5                  ]5                  8                  8                  8                  8                  8                  8                  8                  8                  :                  :                  :                  :                  :                  :                  ;                  ;                  ;                  ;                  ;                  >                  >                  >                  >                   D                  D                  D                  D                  E                  E                  E                  E                   E      $            E      (            E      ,            E      0             F      4            F      8            F      <            ?F      @            @F      D            EF      H            lF      L            mF      P            rF      T            F      X            F      \            F      `            F      d            F      h            F      l            F      p            F      t            F      x            ,G      |            -G                  2G                  @G                  G                  G                  G                  G                  G                  G                  G                  G                  G                  G                  G                  I                  I                  I                  I                  I                  I                  I                  I                  L                  L                  L                  L                  L                  L                  L                  L                  PM                  WM                  YM                   bM                  jM                  nM                  IN                  LN                  NN                  PN                  RN                   WN      $            N      (            N      ,            N      0            N      4            N      8            N      <            "O      @            #O      D            %O      H            'O      L            )O      P            .O      T            vO      X            O      \            O      `            O      d            O      h            O      l            O      p            O      t            O      x            O      |            O                  O                  O                  O                  P                  P                  P                  P                                                                                                4                   q                                                                                            b                                                                                                                                                         .                                        3                                                                            	                      h                   l                	                              $                   (          	   B      0             L      4                   8          	   z      @                   D                   H          	   
      P             3N      T             .O      X          	   *       `             WN      d             N      h          	          p             N      t             N      x          	   b                                       @                  p                                     $                                                                                                       `                                             8                     @             `       H                   P                   p                     x             `                                                                                                                                                               @                                       X                                        @       (                  0                   P                    X            (       `                  h            &                                                                                   8                    P                     .symtab .strtab .shstrtab .note.gnu.build-id .note.Linux .rela.text .rela.init.text .rela.text.unlikely .rela.exit.text .rela__mcount_loc .rodata.str1.1 .rodata.str1.8 .rela.smp_locks .rela.rodata .modinfo .rela__param .rodata.cst2 .rela.return_sites .orc_unwind .rela.orc_unwind_ip __versions .rela__bug_table .rela__jump_table .rela.data .rela.exit.data .rela.init.data .rela__dyndbg .data.once .rela.gnu.linkonce.this_module .bss .comment .note.GNU-stack .BTF .gnu_debuglink                                                                                       @       $                              .                     d       <                              ?                            P                             :      @               0      ,      1                    J                     P                                    E      @                    x       1                    Z                     P                                   U      @               H     	      1                    n                     T                                    i      @                     0       1   	                 ~                     T                                   y      @               P     X      1                          2               V                                        2               Y                                                       [                                          @               !     `       1                                         [      H                                    @               "           1                                         (c                                                        d      P                                    @               %            1                                         He                                                        Je                                          @               &           1                                         6f                                   
                     r                                        @               @,     (/      1                                        y                                    ,                          $                              '     @               h[            1                    =                    ȑ                                    8     @               [     @      1                    O                    `                                    J     @               8^            1   !                 Z                    H                                    U     @               ^            1   #                 j                    P                                    e     @               ^            1   %                 z                    X                                   u     @               ^            1   '                                                                                             @                    @                    @               a     0       1   *                                                                              0                     P                                                                                                                  ="                                                  P                                                          `      8      2                    	                                                                               (b                                  0	*H
01
0	`He0	*H
1a0]080 10UDebian Secure Boot CA2(oe:B&C0	`He0
	*H
  v?T|/9L +r|,3R*fXR  >#hLu-݈Tmf.<t=M4P6aX#}t@ATJy&?w=+e6:}
ٱЗo}!@
V]+9Y˜5$3IXs7
&ĕ9~t/PKqO
ifOE5WY.U¥$!W5\,gq 	K"0w$p ڲ         ~Module signature appended~
 070701000A0465000081A4000000000000000000000001682F6DA7000131BB000000CA0000000200000000000000000000005000000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/atheros/atlx/atl2.ko   ELF          >                    #         @     @ 0 /          GNU ur9]l-        Linux                Linux   6.1.0-37-amd64      HHʀ$   HE        1    @         fD                 HF    	  F	  tF	  t
N	  t
N 	  tN    f    F^u/Ǉ	      F	  Ft		  1        ff.         H
     f
  ~H  H    1    ff.     @     ATIU   HSHH  H    
   !      HE H        tu[   ]A\    fA$1[]A\    fD      UHSH   HeH%(   HD$1HT$1fD$Iu2T$tQf tAf @u3d    f҃fU HT$eH+%(   u H[]       ޹
   ĸ       ff.          UHS   H	H     H    
  // <experimental/internet> -*- C++ -*-

// Copyright (C) 2015-2022 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/** @file experimental/internet
 *  This is a TS C++ Library header.
 *  @ingroup networking-ts
 */

#ifndef _GLIBCXX_EXPERIMENTAL_INTERNET
#define _GLIBCXX_EXPERIMENTAL_INTERNET

#pragma GCC system_header

#if __cplusplus >= 201402L

#include <experimental/netfwd>
#include <experimental/io_context>
#include <experimental/bits/net.h>
#include <array>
#include <forward_list>
#include <sstream>
#include <cstdint>
#include <experimental/string_view>
#ifdef _GLIBCXX_HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
# include <sys/socket.h>	// AF_INET, AF_INET6, SOCK_DGRAM, SOCK_STREAM
#endif
#ifdef _GLIBCXX_HAVE_ARPA_INET_H
# include <arpa/inet.h>		// inet_ntop
#endif
#ifdef _GLIBCXX_HAVE_NETINET_IN_H
# include <netinet/in.h>	// IPPROTO_IP, IPPROTO_IPV6, in_addr, in6_addr
#endif
#ifdef _GLIBCXX_HAVE_NETINET_TCP_H
# include <netinet/tcp.h>	// TCP_NODELAY
#endif
#ifdef _GLIBCXX_HAVE_NETDB_H
# include <netdb.h>		// getaddrinfo etc.
#endif

#if defined _WIN32 && __has_include(<ws2tcpip.h>)
# include <ws2tcpip.h>
#endif

namespace std _GLIBCXX_VISIBILITY(default)
{
_GLIBCXX_BEGIN_NAMESPACE_VERSION
namespace experimental
{
namespace net
{
inline namespace v1
{
namespace ip
{
  /** @addtogroup networking-ts
   *  @{
   */

  /** Error codes for resolver errors.
   * @{
   */

  enum class resolver_errc : int {
#ifdef _GLIBCXX_HAVE_NETDB_H
    host_not_found = EAI_NONAME,
    host_not_found_try_again = EAI_AGAIN,
    service_not_found = EAI_SERVICE
    // N.B. POSIX defines additional errors that have no enumerator here:
    // EAI_BADFLAGS, EAI_FAIL, EAI_FAMILY, EAI_MEMORY, EAI_SOCKTYPE, EAI_SYSTEM
    // Some C libraries define additional errors:
    // EAI_BADHINTS, EAI_OVERFLOW, EAI_PROTOCOL
    // Some C libraries define additional (obsolete?) errors:
    // EAI_ADDRFAMILY, EAI_NODATA
#endif
  };

  /// Error category for resolver errors.
  inline const error_category& resolver_category() noexcept // TODO non-inline
  {
    struct __cat : error_category
    {
      const char* name() const noexcept { return "resolver"; }
      std::string message(int __e) const {
#ifdef _GLIBCXX_HAVE_NETDB_H
	  return ::gai_strerror(__e);
#else
	  return "name resolution requires <netdb.h>";
#endif
      }
      virtual void __message(int) { } // TODO dual ABI XXX
    };
    static __cat __c;
    return __c;
  }

  inline error_code make_error_code(resolver_errc __e) noexcept
  { return error_code(static_cast<int>(__e), resolver_category()); }

  inline error_condition make_error_condition(resolver_errc __e) noexcept
  { return error_condition(static_cast<int>(__e), resolver_category()); }

  /// @cond undocumented
  inline error_code
  __make_resolver_error_code(int __ai_err,
			     [[__maybe_unused__]] int __sys_err) noexcept
  {
#ifdef EAI_SYSTEM
    if (__builtin_expect(__ai_err == EAI_SYSTEM, 0))
      return error_code(__sys_err, std::generic_category());
#endif
    return error_code(__ai_err, resolver_category());
  }
  /// @endcond

  /// @}

  using port_type = uint_least16_t;	///< Type used for port numbers.
  using scope_id_type = uint_least32_t;	///< Type used for IPv6 scope IDs.

  /// Convenience alias for constraining allocators for strings.
  template<typename _Alloc>
    using __string_with
      = enable_if_t<std::is_same<typename _Alloc::value_type, char>::value,
		    std::basic_string<char, std::char_traits<char>, _Alloc>>;

  constexpr errc
  __unsupported_err() noexcept
  {
#if defined EAFNOSUPPORT
    return std::errc::address_family_not_supported;
#else
    return std::errc::operation_not_supported;
#endif
  }

  /** Tag indicating conversion between IPv4 and IPv4-mapped IPv6 addresses.
   * @{
   */

  struct v4_mapped_t {};
  constexpr v4_mapped_t v4_mapped;

  /// @}

  /// An IPv4 address.
  class address_v4
  {
  public:
    // types:
    using uint_type = uint_least32_t;

    struct bytes_type : array<unsigned char, 4>
    {
      template<typename... _Tp>
	explicit constexpr
	bytes_type(_Tp... __tp)
	: array<unsigned char, 4>{{static_cast<unsigned char>(__tp)...}}
	{
#if UCHAR_MAX > 0xFF
	  for (auto __b : *this)
	    if (__b > 0xFF)
	      __throw_out_of_range("invalid address_v4::bytes_type value");
#endif
	}
    };

    // constructors:
    constexpr address_v4() noexcept : _M_addr(0) { }

    constexpr address_v4(const address_v4& a) noexcept = default;

    constexpr
    address_v4(const bytes_type& __b)
    : _M_addr((__b[0] << 24) | (__b[1] << 16) | (__b[2] << 8) | __b[3])
    { }

    explicit constexpr
    address_v4(uint_type __val) : _M_addr(_S_hton_32(__val))
    {
#if UINT_LEAST32_MAX > 0xFFFFFFFF
      if (__val > 0xFFFFFFFF)
	__throw_out_of_range("invalid address_v4::uint_type value");
#endif
    }

    // assignment:
    address_v4& operator=(const address_v4& a) noexcept = default;

    // members:
    constexpr bool is_unspecified() const noexcept { return to_uint() == 0; }

    constexpr bool
    is_loopback() const noexcept
    { return (to_uint() & 0xFF000000) == 0x7F000000; }

    constexpr bool
    is_multicast() const noexcept
    { return (to_uint() & 0xF0000000) == 0xE0000000; }

    constexpr bytes_type
    to_bytes() const noexcept
    {
      return bytes_type{
	  (_M_addr >> 24) & 0xFF,
	  (_M_addr >> 16) & 0xFF,
	  (_M_addr >> 8) & 0xFF,
	  _M_addr & 0xFF
      };
    }

    constexpr uint_type
    to_uint() const noexcept { return _S_ntoh_32(_M_addr); }

    template<typename _Allocator = allocator<char>>
      __string_with<_Allocator>
      to_string(const _Allocator& __a = _Allocator()) const
      {
#ifdef _GLIBCXX_HAVE_ARPA_INET_H
	__string_with<_Allocator> __str(__a);
	__str.resize(INET_ADDRSTRLEN);
	if (inet_ntop(AF_INET, &_M_addr, &__str.front(), __str.size()))
	  __str.erase(__str.find('\0'));
	else
	  __str.resize(0);
	return __str;
#else
	std::__throw_system_error((int)__unsupported_err());
#endif
      }

    // static members:
    static constexpr address_v4 any() noexcept { return address_v4{}; }

    static constexpr
    address_v4 loopback() noexcept { return address_v4{0x7F000001}; }

    static constexpr
    address_v4 broadcast() noexcept { return address_v4{0xFFFFFFFF}; }

  private:
    template<typename _InternetProtocol>
      friend class basic_endpoint;

    friend address_v4 make_address_v4(const char*, error_code&) noexcept;

#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
    static constexpr uint16_t _S_hton_16(uint16_t __h) { return __h; }
    static constexpr uint16_t _S_ntoh_16(uint16_t __n) { return __n; }
    static constexpr uint32_t _S_hton_32(uint32_t __h) { return __h; }
    static constexpr uint32_t _S_ntoh_32(uint32_t __n) { return __n; }
#else
    static constexpr uint16_t
    _S_hton_16(uint16_t __h) { return __builtin_bswap16(__h); }

    static constexpr uint16_t
    _S_ntoh_16(uint16_t __n) { return __builtin_bswap16(__n); }

    static constexpr uint32_t
    _S_hton_32(uint32_t __h) { return __builtin_bswap32(__h); }

    static constexpr uint32_t
    _S_ntoh_32(uint32_t __n) { return __builtin_bswap32(__n); }
#endif

#ifdef _GLIBCXX_HAVE_ARPA_INET_H
    in_addr_t _M_addr; // network byte order
#else
    uint32_t _M_addr;
#endif
  };

  /// An IPv6 address.
  class address_v6
  {
  public:
    // types:
    struct bytes_type : array<unsigned char, 16>
    {
      template<typename... _Tp>
	explicit constexpr
	bytes_type(_Tp... __t)
	: array<unsigned char, 16>{{static_cast<unsigned char>(__t)...}}
	{ }
    };

    // constructors:
    constexpr address_v6() noexcept : _M_bytes(), _M_scope_id() { }

    constexpr address_v6(const address_v6& __a) noexcept = default;

    constexpr
    address_v6(const bytes_type& __bytes, scope_id_type __scope = 0)
    : _M_bytes(__bytes), _M_scope_id(__scope)
    { }

    // assignment:
    address_v6& operator=(const address_v6& __a) noexcept = default;

    // members:
    void scope_id(scope_id_type __id) noexcept { _M_scope_id = __id; }

    constexpr scope_id_type scope_id() const noexcept { return _M_scope_id; }

    constexpr bool
    is_unspecified() const noexcept
    {
      for (int __i = 0; __i < 16; ++__i)
	if (_M_bytes[__i] != 0x00)
	  return false;
      return _M_scope_id == 0;
    }

    constexpr bool
    is_loopback() const noexcept
    {
      for (int __i = 0; __i < 15; ++__i)
	if (_M_bytes[__i] != 0x00)
	  return false;
      return _M_bytes[15] == 0x01 && _M_scope_id == 0;
    }

    constexpr bool
    is_multicast() const noexcept { return _M_bytes[0] == 0xFF; }

    constexpr bool
    is_link_local() const noexcept
    { return _M_bytes[0] == 0xFE && (_M_bytes[1] & 0xC0) == 0x80; }

    constexpr bool
    is_site_local() const noexcept
    { return _M_bytes[0] == 0xFE && (_M_bytes[1] & 0xC0) == 0xC0; }

    constexpr bool
    is_v4_mapped() const noexcept
    {
      const bytes_type& __b = _M_bytes;
      return __b[0] == 0 && __b[1] == 0 && __b[ 2] == 0    && __b[ 3] == 0
	  && __b[4] == 0 && __b[5] == 0 && __b[ 6] == 0    && __b[ 7] == 0
	  && __b[8] == 0 && __b[9] == 0 && __b[10] == 0xFF && __b[11] == 0xFF;
    }

    constexpr bool
    is_multicast_node_local() const noexcept
    { return is_multicast() && (_M_bytes[1] & 0x0F) == 0x01; }

    constexpr bool
    is_multicast_link_local() const noexcept
    { return is_multicast() && (_M_bytes[1] & 0x0F) == 0x02; }

    constexpr bool
    is_multicast_site_local() const noexcept
    { return is_multicast() && (_M_bytes[1] & 0x0F) == 0x05; }

    constexpr bool
    is_multicast_org_local() const noexcept
    { return is_multicast() && (_M_bytes[1] & 0x0F) == 0x08; }

    constexpr bool
    is_multicast_global() const noexcept
    { return is_multicast() && (_M_bytes[1] & 0x0F) == 0x0b; }

    constexpr bytes_type to_bytes() const noexcept { return _M_bytes; }

    template<typename _Allocator = allocator<char>>
      __string_with<_Allocator>
      to_string(const _Allocator& __a = _Allocator()) const
      {
#ifdef _GLIBCXX_HAVE_ARPA_INET_H
	__string_with<_Allocator> __str(__a);
	__str.resize(INET6_ADDRSTRLEN + (_M_scope_id ? 11 : 0));
	char* const __p = &__str.front();
	if (inet_ntop(AF_INET6, &_M_bytes, __p, __str.size()))
	  {
	    auto __end = __str.find('\0');
	    if (unsigned long __scope = _M_scope_id)
	      {
		__end +=
#if _GLIBCXX_USE_C99_STDIO
		  __builtin_snprintf(__p + __end, __str.size() - __end,
				     "%%%lu", __scope);
#else
		  __builtin_sprintf(__p + __end, "%%%lu", __scope);
#endif
	      }
	    __str.erase(__end);
	  }
	else
	  __str.resize(0);
	return __str;
#else
	std::__throw_system_error((int)__unsupported_err());
#endif
      }

    // static members:

    static constexpr address_v6
    any() noexcept
    {
      return {};
    }

    static constexpr address_v6
    loopback() noexcept
    {
      return {bytes_type{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}};
    }

  private:
    template<typename _InternetProtocol>
      friend class basic_endpoint;

    friend constexpr bool
    operator==(const address_v6&, const address_v6&) noexcept;

    friend constexpr bool
    operator< (const address_v6&, const address_v6&) noexcept;

    bytes_type _M_bytes;
    scope_id_type _M_scope_id;
  };

  /// Exception type thrown on misuse of IPv4 addresses as IPv6 or vice versa.
  class bad_address_cast : public bad_cast
  {
  public:
    bad_address_cast() { }

    const char* what() const noexcept { return "bad address cast"; }
  };

  /// An IPv4 or IPv6 address.
  class address
  {
  public:
    // constructors:
    constexpr address() noexcept : _M_v4(), _M_is_v4(true) { }

#if __cpp_constexpr_dynamic_alloc
    constexpr
#endif
    address(const address& __a) noexcept : _M_uninit(), _M_is_v4(__a._M_is_v4)
    {
      if (_M_is_v4)
	std::_Construct(std::addressof(_M_v4), __a.to_v4());
      else
	std::_Construct(std::addressof(_M_v6), __a.to_v6());
    }

    constexpr
    address(const address_v4& __a) noexcept : _M_v4(__a), _M_is_v4(true) { }

    constexpr
    address(const address_v6& __a) noexcept : _M_v6(__a), _M_is_v4(false) { }

    // assignment:
    address&
    operator=(const address& __a) noexcept
    {
      if (__a._M_is_v4)
	*this = __a.to_v4();
      else
	*this = __a.to_v6();
      return *this;
    }

    address&
    operator=(const address_v4& __a) noexcept
    {
      std::_Construct(std::addressof(_M_v4), __a);
      _M_is_v4 = true;
      return *this;
    }

    address&
    operator=(const address_v6& __a) noexcept
    {
      std::_Construct(std::addressof(_M_v6), __a);
      _M_is_v4 = false;
      return *this;
    }

    // members:

    constexpr bool is_v4() const noexcept { return _M_is_v4; }
    constexpr bool is_v6() const noexcept { return !_M_is_v4; }

    constexpr address_v4
    to_v4() const
    {
      if (!is_v4())
	_GLIBCXX_THROW_OR_ABORT(bad_address_cast());
      return _M_v4;
    }

    constexpr address_v6
    to_v6() const
    {
      if (!is_v6())
	_GLIBCXX_THROW_OR_ABORT(bad_address_cast());
      return _M_v6;
    }

    constexpr bool
    is_unspecified() const noexcept
    { return _M_is_v4 ? _M_v4.is_unspecified() : _M_v6.is_unspecified(); }

    constexpr bool
    is_loopback() const noexcept
    { return _M_is_v4 ? _M_v4.is_loopback() : _M_v6.is_loopback(); }

    constexpr bool
    is_multicast() const noexcept
    { return _M_is_v4 ? _M_v4.is_multicast() : _M_v6.is_multicast(); }

    template<typename _Allocator = allocator<char>>
      __string_with<_Allocator>
      to_string(const _Allocator& __a = _Allocator()) const
      {
	if (_M_is_v4)
	  return to_v4().to_string(__a);
	return to_v6().to_string(__a);
      }

  private:
    template<typename _InternetProtocol>
      friend class basic_endpoint;

    friend constexpr bool
    operator==(const address&, const address&) noexcept;

    friend constexpr bool
    operator<(const address&, const address&) noexcept;

    union {
      address_v4 _M_v4;
      address_v6 _M_v6;
      bool	 _M_uninit;
    };
    bool _M_is_v4;
  };

  /** ip::address_v4 comparisons
   * @{
   */

  constexpr bool
  operator==(const address_v4& __a, const address_v4& __b) noexcept
  { return __a.to_uint() == __b.to_uint(); }

  constexpr bool
  operator!=(const address_v4& __a, const address_v4& __b) noexcept
  { return !(__a == __b); }

  constexpr bool
  operator< (const address_v4& __a, const address_v4& __b) noexcept
  { return __a.to_uint() < __b.to_uint(); }

  constexpr bool
  operator> (const address_v4& __a, const address_v4& __b) noexcept
  { return __b < __a; }

  constexpr bool
  operator<=(const address_v4& __a, const address_v4& __b) noexcept
  { return !(__b < __a); }

  constexpr bool
  operator>=(const address_v4& __a, const address_v4& __b) noexcept
  { return !(__a < __b); }

  /// @}

  /** ip::address_v6 comparisons
   * @{
   */

  constexpr bool
  operator==(const address_v6& __a, const address_v6& __b) noexcept
  {
    const auto& __aa = __a._M_bytes;
    const auto& __bb = __b._M_bytes;
    int __i = 0;
    for (; __i < 16 && __aa[__i] == __bb[__i]; ++__i)
      ;
    return __i == 16 ? __a.scope_id() == __b.scope_id() : false;
  }

  constexpr bool
  operator!=(const address_v6& __a, const address_v6& __b) noexcept
  { return !(__a == __b); }

  constexpr bool
  operator< (const address_v6& __a, const address_v6& __b) noexcept
  {
    const auto& __aa = __a._M_bytes;
    const auto& __bb = __b._M_bytes;
    int __i = 0;
    for (; __i < 16 && __aa[__i] == __bb[__i]; ++__i)
      ;
    return __i == 16 ? __a.scope_id() < __b.scope_id() : __aa[__i] < __bb[__i];
  }

  constexpr bool
  operator> (const address_v6& __a, const address_v6& __b) noexcept
  { return __b < __a; }

  constexpr bool
  operator<=(const address_v6& __a, const address_v6& __b) noexcept
  { return !(__b < __a); }

  constexpr bool
  operator>=(const address_v6& __a, const address_v6& __b) noexcept
  { return !(__a < __b); }

  /// @}

  /** ip::address comparisons
   * @{
   */

  constexpr bool
  operator==(const address& __a, const address& __b) noexcept
  {
    if (__a.is_v4())
      return __b.is_v4() ? __a._M_v4 == __b._M_v4 : false;
    return __b.is_v4() ? false : __a._M_v6 == __b._M_v6;
  }

  constexpr bool
  operator!=(const address& __a, const address& __b) noexcept
  { return !(__a == __b); }

  constexpr bool
  operator< (const address& __a, const address& __b) noexcept
  {
    if (__a.is_v4())
      return __b.is_v4() ? __a._M_v4 < __b._M_v4 : true;
    return __b.is_v4() ? false : __a._M_v6 < __b._M_v6;
  }

  constexpr bool
  operator> (const address& __a, const address& __b) noexcept
  { return __b < __a; }

  constexpr bool
  operator<=(const address& __a, const address& __b) noexcept
  { return !(__b < __a); }

  constexpr bool
  operator>=(const address& __a, const address& __b) noexcept
  { return !(__a < __b); }

  /// @}

  /** ip::address_v4 creation
   * @{
   */

  constexpr address_v4
  make_address_v4(const address_v4::bytes_type& __b)
  { return address_v4{__b}; }

  constexpr address_v4
  make_address_v4(address_v4::uint_type __val)
  { return address_v4{__val}; }

  constexpr address_v4
  make_address_v4(v4_mapped_t, const address_v6& __a)
  {
    if (!__a.is_v4_mapped())
      _GLIBCXX_THROW_OR_ABORT(bad_address_cast());

    const auto __v6b = __a.to_bytes();
    return address_v4::bytes_type(__v6b[12], __v6b[13], __v6b[14], __v6b[15]);
  }

  inline address_v4
  make_address_v4(const char* __str, error_code& __ec) noexcept
  {
#ifdef _GLIBCXX_HAVE_ARPA_INET_H
    address_v4 __a;
    const int __res = ::inet_pton(AF_INET, __str, &__a._M_addr);
    if (__res == 1)
      {
	__ec.clear();
	return __a;
      }
    if (__res == 0)
      __ec = std::make_error_code(std::errc::invalid_argument);
    else
      __ec.assign(errno, generic_category());
#else
    __ec = std::make_error_code(__unsupported_err());
#endif
    return {};
  }

  inline address_v4
  make_address_v4(const char* __str)
  { return make_address_v4(__str, __throw_on_error{"make_address_v4"}); }

  inline address_v4
  make_address_v4(const string& __str, error_code& __ec) noexcept
  { return make_address_v4(__str.c_str(), __ec); }

  inline address_v4
  make_address_v4(const string& __str)
  { return make_address_v4(__str.c_str()); }

  inline address_v4
  make_address_v4(string_view __str, error_code& __ec) noexcept
  {
    char __buf[16]; // INET_ADDRSTRLEN isn't defined on Windows
    auto __len = __str.copy(__buf, sizeof(__buf));
    if (__len == sizeof(__buf))
      {
	__ec = std::make_error_code(std::errc::invalid_argument);
	return {};
      }
    __ec.clear();
    __buf[__len] = '\0';
    return make_address_v4(__buf, __ec);
  }

  inline address_v4
  make_address_v4(string_view __str)
  { return make_address_v4(__str, __throw_on_error{"make_address_v4"}); }

  /// @}

  /** ip::address_v6 creation
   * @{
   */

  constexpr address_v6
  make_address_v6(const address_v6::bytes_type& __b, scope_id_type __scope = 0)
  { return address_v6{__b, __scope}; }

  constexpr address_v6
  make_address_v6(v4_mapped_t, const address_v4& __a) noexcept
  {
    const address_v4::bytes_type __v4b = __a.to_bytes();
    address_v6::bytes_type __v6b(0, 0, 0, 0, 0, 0, 0, 0,
				 0, 0, 0xFF, 0xFF,
				 __v4b[0], __v4b[1], __v4b[2], __v4b[3]);
    return address_v6(__v6b);
  }

  inline address_v6
  __make_address_v6(const char* __addr, const char* __scope, error_code& __ec)
  {
#ifdef _GLIBCXX_HAVE_ARPA_INET_H
    address_v6::bytes_type __b;
    const int __res = ::inet_pton(AF_INET6, __addr, __b.data());
    if (__res == 1)
      {
	__ec.clear();
	if (!__scope)
	  {
	    return { __b };
	  }

	char* __eptr;
	unsigned long __val = std::strtoul(__scope, &__eptr, 10);
	if (__eptr != __scope && !*__eptr
	    && __val <= numeric_limits<scope_id_type>::max())
	  {
	    return { __b, static_cast<scope_id_type>(__val) };
	  }
	__ec = std::make_error_code(std::errc::invalid_argument);
      }
    else if (__res == 0)
      __ec = std::make_error_code(std::errc::invalid_argument);
    else
      __ec.assign(errno, generic_category());
#else
    __ec = std::make_error_code(__unsupported_err());
#endif
    return {};
  }

  inline address_v6
  make_address_v6(const char* __str, error_code& __ec) noexcept
  {
    auto __p = __builtin_strchr(__str, '%');
    if (__p == nullptr)
      return __make_address_v6(__str, nullptr, __ec);
    char __buf[64];
    char* __out = __buf;
    bool __skip_leading_zero = true;
    while (__str < __p && __out < std::end(__buf))
      {
	if (!__skip_leading_zero || *__str != '0')
	  {
	    if (*__str == ':' || *__str == '.')
	      __skip_leading_zero = true;
	    else
	      __skip_leading_zero = false;
	    *__out = *__str;
	  }
	__str++;
      }
    if (__out == std::end(__buf))
      {
	__ec = std::make_error_code(std::errc::invalid_argument);
	return {};
      }
    else
      {
	*__out = '\0';
	return __make_address_v6(__buf, __p + 1, __ec);
      }
  }

  inline address_v6
  make_address_v6(const char* __str)
  { return make_address_v6(__str, __throw_on_error{"make_address_v6"}); }

  inline address_v6
  make_address_v6(const string& __str, error_code& __ec) noexcept
  {
    auto __pos = __str.find('%');
    if (__pos == string::npos)
      return __make_address_v6(__str.c_str(), nullptr, __ec);
    char __buf[64];
    char* __out = __buf;
    bool __skip_leading_zero = true;
    size_t __n = 0;
    while (__n < __pos && __out < std::end(__buf))
      {
	if (!__skip_leading_zero || __str[__n] != '0')
	  {
	    if (__str[__n] == ':' || __str[__n] == '.')
	      __skip_leading_zero = true;
	    else
	      __skip_leading_zero = false;
	    *__out = __str[__n];
	  }
	__n++;
      }
    if (__out == std::end(__buf))
      {
	__ec = std::make_error_code(std::errc::invalid_argument);
	return {};
      }
    else
      {
	*__out = '\0';
	return __make_address_v6(__buf, __str.c_str() + __pos + 1, __ec);
      }
  }

  inline address_v6
  make_address_v6(const string& __str)
  { return make_address_v6(__str, __throw_on_error{"make_address_v6"}); }

  inline address_v6
  make_address_v6(string_view __str, error_code& __ec) noexcept
  {
    char __buf[64];
    char* __out = __buf;
    char* __scope = nullptr;
    bool __skip_leading_zero = true;
    size_t __n = 0;
    while (__n < __str.length() && __out < std::end(__buf))
      {
	if (__str[__n] == '%')
	  {
	    if (__scope)
	      __out = std::end(__buf);
	    else
	      {
		*__out = '\0';
		__scope = ++__out;
		__skip_leading_zero = true;
	      }
	  }
	else if (!__skip_leading_zero || __str[__n] != '0')
	  {
	    if (__str[__n] == ':' || __str[__n] == '.')
	      __skip_leading_zero = true;
	    else
	      __skip_leading_zero = false;
	    *__out = __str[__n];
	    __out++;
	  }
	__n++;
      }
    if (__out == std::end(__buf))
      {
	__ec = std::make_error_code(std::errc::invalid_argument);
	return {};
      }
    else
      {
	*__out = '\0';
	return __make_address_v6(__buf, __scope, __ec);
      }
  }

  inline address_v6
  make_address_v6(string_view __str)
  { return make_address_v6(__str, __throw_on_error{"make_address_v6"}); }

  /// @}

  /** ip::address creation
   * @{
   */

  inline address
  make_address(const char* __str, error_code& __ec) noexcept
  {
    address __a;
    address_v6 __v6a = make_address_v6(__str, __ec);
    if (!__ec)
      __a = __v6a;
    else
    {
      address_v4 __v4a = make_address_v4(__str, __ec);
      if (!__ec)
	__a = __v4a;
    }
    return __a;
  }

  inline address
  make_address(const char* __str)
  { return make_address(__str, __throw_on_error{"make_address"}); }

  inline address
  make_address(const string& __str, error_code& __ec) noexcept; // TODO

  inline address
  make_address(const string& __str)
  { return make_address(__str, __throw_on_error{"make_address"}); }

  inline address
  make_address(string_view __str, error_code& __ec) noexcept
  {
    if (__str.rfind('\0') != string_view::npos)
      return make_address(__str.data(), __ec);
    return make_address(__str.to_string(), __ec); // TODO don't allocate
  }

  inline address
  make_address(string_view __str)
  { return make_address(__str, __throw_on_error{"make_address"}); }

  /// @}

  /// ip::address I/O
  template<typename _CharT, typename _Traits>
    inline basic_ostream<_CharT, _Traits>&
    operator<<(basic_ostream<_CharT, _Traits>& __os, const address& __a)
    { return __os << __a.to_string(); }

  /// ip::address_v4 I/O
  template<typename _CharT, typename _Traits>
    inline basic_ostream<_CharT, _Traits>&
    operator<<(basic_ostream<_CharT, _Traits>& __os, const address_v4& __a)
    { return __os << __a.to_string(); }

  /// ip::address_v6 I/O
  template<typename _CharT, typename _Traits>
    inline basic_ostream<_CharT, _Traits>&
    operator<<(basic_ostream<_CharT, _Traits>& __os, const address_v6& __a)
    { return __os << __a.to_string(); }

  template<typename> class basic_address_iterator; // not defined

  template<> class basic_address_iterator<address_v4>
  {
  public:
    // types:
    using value_type = address_v4;
    using difference_type = ptrdiff_t;
    using pointer = const address_v4*;
    using reference = const address_v4&;
    using iterator_category = input_iterator_tag;

    // constructors:
    basic_address_iterator(const address_v4& __a) noexcept
    : _M_address(__a) { }

    // members:
    reference operator*() const noexcept { return _M_address; }
    pointer operator->() const noexcept { return &_M_address; }

    basic_address_iterator&
    operator++() noexcept
    {
      _M_address = value_type(_M_address.to_uint() + 1);
      return *this;
    }

    basic_address_iterator operator++(int) noexcept
    {
      auto __tmp = *this;
      ++*this;
      return __tmp;
    }

    basic_address_iterator& operator--() noexcept
    {
      _M_address = value_type(_M_address.to_uint() - 1);
      return *this;
    }

    basic_address_iterator
    operator--(int) noexcept
    {
      auto __tmp = *this;
      --*this;
      return __tmp;
    }

    bool
    operator==(const basic_address_iterator& __rhs) const noexcept
    { return _M_address == __rhs._M_address; }

    bool
    operator!=(const basic_address_iterator& __rhs) const noexcept
    { return _M_address != __rhs._M_address; }

  private:
    address_v4 _M_address;
  };

  using address_v4_iterator = basic_address_iterator<address_v4>;

  template<> class basic_address_iterator<address_v6>
  {
  public:
    // types:
    using value_type = address_v6;
    using difference_type = ptrdiff_t;
    using pointer = const address_v6*;
    using reference = const address_v6&;
    using iterator_category = input_iterator_tag;

    // constructors:
    basic_address_iterator(const address_v6& __a) noexcept
    : _M_address(__a) { }

    // members:
    reference operator*() const noexcept { return _M_address; }
    pointer operator->() const noexcept { return &_M_address; }

    basic_address_iterator&
    operator++() noexcept; // TODO

    basic_address_iterator
    operator++(int) noexcept
    {
      auto __tmp = *this;
      ++*this;
      return __tmp;
    }

    basic_address_iterator&
    operator--() noexcept; // TODO

    basic_address_iterator
    operator--(int) noexcept
    {
      auto __tmp = *this;
      --*this;
      return __tmp;
    }

    bool
    operator==(const basic_address_iterator& __rhs) const noexcept
    { return _M_address == __rhs._M_address; }

    bool
    operator!=(const basic_address_iterator& __rhs) const noexcept
    { return _M_address != __rhs._M_address; }

  private:
    address_v6 _M_address;
  };

  using address_v6_iterator = basic_address_iterator<address_v6>;

  template<typename> class basic_address_range; // not defined

  /** An IPv6 address range.
   * @{
   */

  template<> class basic_address_range<address_v4>
  {
  public:
    // types:

    using iterator = basic_address_iterator<address_v4>;

    // constructors:

    basic_address_range() noexcept : _M_begin({}), _M_end({}) { }

    basic_address_range(const address_v4& __first,
                        const address_v4& __last) noexcept
    : _M_begin(__first), _M_end(__last) { }

    // members:

    iterator begin() const noexcept { return _M_begin; }
    iterator end() const noexcept { return _M_end; }
    _GLIBCXX_NODISCARD bool empty() const noexcept { return _M_begin == _M_end; }

    size_t
    size() const noexcept { return _M_end->to_uint() - _M_begin->to_uint(); }

    iterator
    find(const address_v4& __addr) const noexcept
    {
      if (*_M_begin <= __addr && __addr < *_M_end)
	return iterator{__addr};
      return end();
    }

  private:
    iterator _M_begin;
    iterator _M_end;
  };

  using address_v4_range = basic_address_range<address_v4>;

  /// @}

  /** An IPv6 address range.
   * @{
   */

  template<> class basic_address_range<address_v6>
  {
  public:
    // types:

    using iterator = basic_address_iterator<address_v6>;

    // constructors:

    basic_address_range() noexcept : _M_begin({}), _M_end({}) { }
    basic_address_range(const address_v6& __first,
                        const address_v6& __last) noexcept
    : _M_begin(__first), _M_end(__last) { }

    // members:

    iterator begin() const noexcept { return _M_begin; }
    iterator end() const noexcept { return _M_end; }
    _GLIBCXX_NODISCARD bool empty() const noexcept { return _M_begin == _M_end; }

    iterator
    find(const address_v6& __addr) const noexcept
    {
      if (*_M_begin <= __addr && __addr < *_M_end)
	return iterator{__addr};
      return end();
    }

  private:
    iterator _M_begin;
    iterator _M_end;
  };

  using address_v6_range = basic_address_range<address_v6>;

  /// @}

  bool
  operator==(const network_v4& __a, const network_v4& __b) noexcept;

  bool
  operator==(const network_v6& __a, const network_v6& __b) noexcept;


  /// An IPv4 network address.
  class network_v4
  {
  public:
    // constructors:
    constexpr network_v4() noexcept : _M_addr(), _M_prefix_len(0) { }

    constexpr
    network_v4(const address_v4& __addr, int __prefix_len)
    : _M_addr(__addr), _M_prefix_len(__prefix_len)
    {
      if (_M_prefix_len < 0 || _M_prefix_len > 32)
	__throw_out_of_range("network_v4: invalid prefix length");
    }

    constexpr
    network_v4(const address_v4& __addr, const address_v4& __mask)
    : _M_addr(__addr), _M_prefix_len(__builtin_popcount(__mask.to_uint()))
    {
      if (_M_prefix_len != 0)
	{
	  address_v4::uint_type __mask_uint = __mask.to_uint();
	  if (__builtin_ctz(__mask_uint) != (32 - _M_prefix_len))
	    __throw_invalid_argument("network_v4: invalid mask");
	  if ((__mask_uint & 0x80000000) == 0)
	    __throw_invalid_argument("network_v4: invalid mask");
	}
    }

    // members:

    constexpr address_v4 address() const noexcept { return _M_addr; }
    constexpr int prefix_length() const noexcept { return _M_prefix_len; }

    constexpr address_v4
    netmask() const noexcept
    {
      address_v4::uint_type __val = address_v4::broadcast().to_uint();
      __val >>= (32 - _M_prefix_len);
      __val <<= (32 - _M_prefix_len);
      return address_v4{__val};
    }

    constexpr address_v4
    network() const noexcept
    { return address_v4{_M_addr.to_uint() & netmask().to_uint()}; }

    constexpr address_v4
    broadcast() const noexcept
    { return address_v4{_M_addr.to_uint() | ~netmask().to_uint()}; }

    address_v4_range
    hosts() const noexcept
    {
      if (is_host())
	return { address(), *++address_v4_iterator(address()) };
      return { network(), broadcast() };
    }

    constexpr network_v4
    canonical() const noexcept
    { return network_v4(network(), prefix_length()); }

    constexpr bool is_host() const noexcept { return _M_prefix_len == 32; }

    constexpr bool
    is_subnet_of(const network_v4& __other) const noexcept
    {
      if (__other.prefix_length() < prefix_length())
	{
	  network_v4 __net(address(), __other.prefix_length());
	  return __net.canonical() == __other.canonical();
	}
      return false;
    }

    template<typename _Allocator = allocator<char>>
      __string_with<_Allocator>
      to_string(const _Allocator& __a = _Allocator()) const
      {
	return address().to_string(__a) + '/'
	  + std::to_string(prefix_length());
      }

  private:
    address_v4 _M_addr;
    int _M_prefix_len;
  };

  /// An IPv6 network address.
  class network_v6
  {
  public:
    // constructors:
    constexpr network_v6() noexcept : _M_addr(), _M_prefix_len(0) { }

    constexpr
    network_v6(const address_v6& __addr, int __prefix_len)
    : _M_addr(__addr), _M_prefix_len(__prefix_len)
    {
      if (_M_prefix_len < 0 || _M_prefix_len > 128)
	__throw_out_of_range("network_v6: invalid prefix length");
    }

    // members:
    constexpr address_v6 address() const noexcept { return _M_addr; }
    constexpr int prefix_length() const noexcept { return _M_prefix_len; }

    constexpr address_v6 network() const noexcept; // TODO

    address_v6_range
    hosts() const noexcept
    {
      if (is_host())
	return { address(), *++address_v6_iterator(address()) };
      return {}; // { network(), XXX broadcast() XXX }; // TODO
    }

    constexpr network_v6
    canonical() const noexcept
    { return network_v6{network(), prefix_length()}; }

    constexpr bool is_host() const noexcept { return _M_prefix_len == 128; }

    constexpr bool
    is_subnet_of(const network_v6& __other) const noexcept
    {
      if (__other.prefix_length() < prefix_length())
	{
	  network_v6 __net(address(), __other.prefix_length());
	  return __net.canonical() == __other.canonical();
	}
      return false;
    }

    template<typename _Allocator = allocator<char>>
      __string_with<_Allocator>
      to_string(const _Allocator& __a = _Allocator()) const
      {
	return address().to_string(__a) + '/'
	  + std::to_string(prefix_length());
      }

  private:
    address_v6 _M_addr;
    int _M_prefix_len;
  };


  /** ip::network_v4 comparisons
   * @{
   */

  inline bool
  operator==(const network_v4& __a, const network_v4& __b) noexcept
  {
    return __a.address() == __b.address()
      && __a.prefix_length() == __b.prefix_length();
  }

  inline bool
  operator!=(const network_v4& __a, const network_v4& __b) noexcept
  { return !(__a == __b); }

  /// @}

  /** ip::network_v6 comparisons
   * @{
   */

  inline bool
  operator==(const network_v6& __a, const network_v6& __b) noexcept
  {
    return __a.address() == __b.address()
      && __a.prefix_length() == __b.prefix_length();
  }

  inline bool
  operator!=(const network_v6& __a, const network_v6& __b) noexcept
  { return !(__a == __b); }

  /// @}

  /** ip::network_v4 creation
   * @{
   */

  inline network_v4
  make_network_v4(const address_v4& __a, int __prefix_len)
  { return network_v4{__a, __prefix_len}; }

  inline network_v4
  make_network_v4(const address_v4& __a, const address_v4& __mask)
  { return network_v4{ __a, __mask }; }

  network_v4 make_network_v4(const char*, error_code&) noexcept; // TODO

  inline network_v4
  make_network_v4(const char* __str)
  { return make_network_v4(__str, __throw_on_error{"make_network_v4"}); }

  network_v4 make_network_v4(const string&, error_code&) noexcept; // TODO

  inline network_v4
  make_network_v4(const string& __str)
  { return make_network_v4(__str, __throw_on_error{"make_network_v4"}); }

  network_v4 make_network_v4(string_view, error_code&) noexcept; // TODO

  inline network_v4
  make_network_v4(string_view __str)
  { return make_network_v4(__str, __throw_on_error{"make_network_v4"}); }

  /// @}

  /** ip::network_v6 creation
   * @{
   */

  inline network_v6
  make_network_v6(const address_v6& __a, int __prefix_len)
  { return network_v6{__a, __prefix_len}; }

  network_v6 make_network_v6(const char*, error_code&) noexcept; // TODO

  inline network_v6
  make_network_v6(const char* __str)
  { return make_network_v6(__str, __throw_on_error{"make_network_v6"}); }

  network_v6 make_network_v6(const string&, error_code&) noexcept; // TODO

  inline network_v6
  make_network_v6(const string& __str)
  { return make_network_v6(__str, __throw_on_error{"make_network_v6"}); }

  network_v6 make_network_v6(string_view, error_code&) noexcept; // TODO

  inline network_v6
  make_network_v6(string_view __str)
  { return make_network_v6(__str, __throw_on_error{"make_network_v6"}); }

  /// @}

  /// ip::network_v4 I/O
  template<typename _CharT, typename _Traits>
    inline basic_ostream<_CharT, _Traits>&
    operator<<(basic_ostream<_CharT, _Traits>& __os, const network_v4& __net)
    { return __os << __net.to_string(); }

  /// ip::network_v6 I/O
  template<typename _CharT, typename _Traits>
    inline basic_ostream<_CharT, _Traits>&
    operator<<(basic_ostream<_CharT, _Traits>& __os, const network_v6& __net)
    { return __os << __net.to_string(); }

  /// An IP endpoint.
  template<typename _InternetProtocol>
    class basic_endpoint
    {
    public:
      // types:
      using protocol_type = _InternetProtocol;

      // constructors:

      constexpr
      basic_endpoint() noexcept : _M_data()
      { _M_data._M_v4.sin_family = protocol_type::v4().family(); }

      constexpr
      basic_endpoint(const protocol_type& __proto,
		     port_type __port_num) noexcept
      : _M_data()
      {
	__glibcxx_assert(__proto == protocol_type::v4()
			  || __proto == protocol_type::v6());

	_M_data._M_v4.sin_family = __proto.family();
	_M_data._M_v4.sin_port = address_v4::_S_hton_16(__port_num);
      }

      constexpr
      basic_endpoint(const ip::address& __addr,
		     port_type __port_num) noexcept
      : _M_data()
      {
	if (__addr.is_v4())
	  {
	    _M_data._M_v4.sin_family = protocol_type::v4().family();
	    _M_data._M_v4.sin_port = address_v4::_S_hton_16(__port_num);
	    _M_data._M_v4.sin_addr.s_addr = __addr._M_v4._M_addr;
	  }
	else
	  {
	    _M_data._M_v6 = {};
	    _M_data._M_v6.sin6_family = protocol_type::v6().family();
	    _M_data._M_v6.sin6_port = address_v4::_S_hton_16(__port_num);
	    __builtin_memcpy(_M_data._M_v6.sin6_addr.s6_addr,
			     __addr._M_v6._M_bytes.data(), 16);
	    _M_data._M_v6.sin6_scope_id = __addr._M_v6._M_scope_id;
	  }
      }

      // members:
      constexpr protocol_type protocol() const noexcept
      {
	return _M_is_v6() ? protocol_type::v6() : protocol_type::v4();
      }

      constexpr ip::address
      address() const noexcept
      {
	ip::address __addr;
	if (_M_is_v6())
	  {
	    __builtin_memcpy(&__addr._M_v6._M_bytes,
			     _M_data._M_v6.sin6_addr.s6_addr, 16);
	    __addr._M_is_v4 = false;
	  }
	else
	  {
	    __builtin_memcpy(&__addr._M_v4._M_addr,
			     &_M_data._M_v4.sin_addr.s_addr, 4);
	  }
	return __addr;
      }

      void
      address(const ip::address& __addr) noexcept
      {
	if (__addr.is_v6())
	  {
	    _M_data._M_v6 = {};
	    _M_data._M_v6.sin6_family = protocol_type::v6().family();
	    __builtin_memcpy(_M_data._M_v6.sin6_addr.s6_addr,
			     __addr._M_v6._M_bytes.data(), 16);
	    _M_data._M_v6.sin6_scope_id = __addr._M_v6._M_scope_id;
	  }
	else
	  {
	    _M_data._M_v4.sin_family = protocol_type::v4().family();
	    _M_data._M_v4.sin_addr.s_addr = __addr._M_v4._M_addr;
	  }
      }

      constexpr port_type
      port() const noexcept
      { return address_v4::_S_ntoh_16(_M_data._M_v4.sin_port); }

      void
      port(port_type __port_num) noexcept
      { _M_data._M_v4.sin_port = address_v4::_S_hton_16(__port_num); }

      void* data() noexcept { return &_M_data; }

      const void* data() const noexcept { return &_M_data; }

      constexpr size_t size() const noexcept
      { return _M_is_v6() ? sizeof(sockaddr_in6) : sizeof(sockaddr_in); }

      void
      resize(size_t __s)
      {
	if (__s != size())
	  __throw_length_error("net::ip::basic_endpoint::resize");
      }

      constexpr size_t capacity() const noexcept { return sizeof(_M_data); }

    private:
      union
      {
	sockaddr_in	_M_v4;
	sockaddr_in6	_M_v6;
      } _M_data;

      constexpr bool _M_is_v6() const noexcept
      { return _M_data._M_v4.sin_family == AF_INET6; }
    };

  /** basic_endpoint comparisons
   * @{
   */

  template<typename _InternetProtocol>
    inline bool
    operator==(const basic_endpoint<_InternetProtocol>& __a,
	       const basic_endpoint<_InternetProtocol>& __b)
    { return __a.address() == __b.address() && __a.port() == __b.port(); }

  template<typename _InternetProtocol>
    inline bool
    operator!=(const basic_endpoint<_InternetProtocol>& __a,
	       const basic_endpoint<_InternetProtocol>& __b)
    { return !(__a == __b); }

  template<typename _InternetProtocol>
    inline bool
    operator< (const basic_endpoint<_InternetProtocol>& __a,
	       const basic_endpoint<_InternetProtocol>& __b)
    {
      return __a.address() < __b.address()
	|| (!(__b.address() < __a.address()) && __a.port() < __b.port());
    }

  template<typename _InternetProtocol>
    inline bool
    operator> (const basic_endpoint<_InternetProtocol>& __a,
	       const basic_endpoint<_InternetProtocol>& __b)
    { return __b < __a; }

  template<typename _InternetProtocol>
    inline bool
    operator<=(const basic_endpoint<_InternetProtocol>& __a,
	       const basic_endpoint<_InternetProtocol>& __b)
    { return !(__b < __a); }

  template<typename _InternetProtocol>
    inline bool
    operator>=(const basic_endpoint<_InternetProtocol>& __a,
	       const basic_endpoint<_InternetProtocol>& __b)
    { return !(__a < __b); }

  /// @}

  /// basic_endpoint I/O
  template<typename _CharT, typename _Traits, typename _InternetProtocol>
    inline basic_ostream<_CharT, _Traits>&
    operator<<(basic_ostream<_CharT, _Traits>& __os,
	       const basic_endpoint<_InternetProtocol>& __ep)
    {
      basic_ostringstream<_CharT, _Traits> __ss;
      if (__ep.protocol()
	  == basic_endpoint<_InternetProtocol>::protocol_type::v6())
	__ss << '[' << __ep.address() << ']';
      else
	__ss << __ep.address();
      __ss << ':' << __ep.port();
      __os << __ss.str();
      return __os;
    }

  /** Type representing a single result of name/address resolution.
   * @{
   */

  template<typename _InternetProtocol>
    class basic_resolver_entry
    {
    public:
      // types:
      using protocol_type = _InternetProtocol;
      using endpoint_type = typename _InternetProtocol::endpoint;

      // constructors:
      basic_resolver_entry() { }

      basic_resolver_entry(const endpoint_type& __ep,
			   string_view __h, string_view __s)
      : _M_ep(__ep), _M_host(__h), _M_svc(__s) { }

      // members:
      endpoint_type endpoint() const { return _M_ep; }
      operator endpoint_type() const { return _M_ep; }

      template<typename _Allocator = allocator<char>>
	__string_with<_Allocator>
	host_name(const _Allocator& __a = _Allocator()) const
	{ return { _M_host, __a }; }

      template<typename _Allocator = allocator<char>>
	__string_with<_Allocator>
	service_name(const _Allocator& __a = _Allocator()) const
	{ return { _M_svc, __a }; }

    private:
      basic_endpoint<_InternetProtocol> _M_ep;
      string _M_host;
      string _M_svc;
    };

  template<typename _InternetProtocol>
    inline bool
    operator==(const basic_resolver_entry<_InternetProtocol>& __a,
	       const basic_resolver_entry<_InternetProtocol>& __b)
    {
      return __a.endpoint() == __b.endpoint()
	&& __a.host_name() == __b.host_name()
	&& __a.service_name() == __b.service_name();
    }

  template<typename _InternetProtocol>
    inline bool
    operator!=(const basic_resolver_entry<_InternetProtocol>& __a,
	       const basic_resolver_entry<_InternetProtocol>& __b)
    { return !(__a == __b); }

  /// @}

  /** Base class defining flags for name/address resolution.
   * @{
   */

  class resolver_base
  {
  public:
    enum flags : int { };
    static constexpr flags passive		= (flags)AI_PASSIVE;
    static constexpr flags canonical_name	= (flags)AI_CANONNAME;
    static constexpr flags numeric_host		= (flags)AI_NUMERICHOST;
#ifdef AI_NUMERICSERV
    static constexpr flags numeric_service	= (flags)AI_NUMERICSERV;
#endif
#ifdef AI_V4MAPPED
    static constexpr flags v4_mapped		= (flags)AI_V4MAPPED;
#endif
#ifdef AI_ALL
    static constexpr flags all_matching		= (flags)AI_ALL;
#endif
#ifdef AI_ADDRCONFIG
    static constexpr flags address_configured	= (flags)AI_ADDRCONFIG;
#endif

    friend constexpr flags
    operator&(flags __f1, flags __f2) noexcept
    { return flags( int(__f1) & int(__f2) ); }

    friend constexpr flags
    operator|(flags __f1, flags __f2) noexcept
    { return flags( int(__f1) | int(__f2) ); }

    friend constexpr flags
    operator^(flags __f1, flags __f2) noexcept
    { return flags( int(__f1) ^ int(__f2) ); }

    friend constexpr flags
    operator~(flags __f) noexcept
    { return flags( ~int(__f) ); }

    friend constexpr flags&
    operator&=(flags& __f1, flags __f2) noexcept
    { return __f1 = (__f1 & __f2); }

    friend constexpr flags&
    operator|=(flags& __f1, flags __f2) noexcept
    { return __f1 = (__f1 | __f2); }

    friend constexpr flags&
    operator^=(flags& __f1, flags __f2) noexcept
    { return __f1 = (__f1 ^ __f2); }

  protected:
    resolver_base() = default;
    ~resolver_base() = default;
  };

  // TODO define resolver_base::flags static constants in .so for C++14 mode

  /// @}

  /** Container for results of name/address resolution.
   * @{
   */

  template<typename _InternetProtocol>
    class basic_resolver_results
    {
    public:
      // types:
      using protocol_type = _InternetProtocol;
      using endpoint_type = typename protocol_type::endpoint;
      using value_type = basic_resolver_entry<protocol_type>;
      using const_reference = const value_type&;
      using reference = value_type&;
      using const_iterator = typename forward_list<value_type>::const_iterator;
      using iterator = const_iterator;
      using difference_type = ptrdiff_t;
      using size_type = size_t;

      // construct / copy / destroy:

      basic_resolver_results() = default;

      basic_resolver_results(const basic_resolver_results&) = default;

      basic_resolver_results(basic_resolver_results&&) noexcept = default;

      basic_resolver_results&
      operator=(const basic_resolver_results&) = default;

      basic_resolver_results&
      operator=(basic_resolver_results&&) = default;

      ~basic_resolver_results() = default;

      // size:
      size_type size() const noexcept { return _M_size; }
      size_type max_size() const noexcept { return _M_results.max_size(); }

      _GLIBCXX_NODISCARD bool
      empty() const noexcept { return _M_results.empty(); }

      // element access:
      const_iterator begin() const { return _M_results.begin(); }
      const_iterator end() const { return _M_results.end(); }
      const_iterator cbegin() const { return _M_results.begin(); }
      const_iterator cend() const { return _M_results.end(); }

      // swap:
      void
      swap(basic_resolver_results& __that) noexcept
      { _M_results.swap(__that._M_results); }

    private:
      friend class basic_resolver<protocol_type>;

      basic_resolver_results(string_view, string_view, resolver_base::flags,
			     error_code&, protocol_type* = nullptr);

      basic_resolver_results(const endpoint_type&, error_code&);

      forward_list<value_type> _M_results;
      size_t _M_size = 0;
    };

  template<typename _InternetProtocol>
    inline bool
    operator==(const basic_resolver_results<_InternetProtocol>& __a,
	       const basic_resolver_results<_InternetProtocol>& __b)
    {
      return __a.size() == __b.size()
	&& std::equal(__a.begin(), __a.end(), __b.begin());
    }

  template<typename _InternetProtocol>
    inline bool
    operator!=(const basic_resolver_results<_InternetProtocol>& __a,
	       const basic_resolver_results<_InternetProtocol>& __b)
    { return !(__a == __b); }

  /// @}

  /// Perform name/address resolution.
  template<typename _InternetProtocol>
    class basic_resolver : public resolver_base
    {
    public:
      // types:

      using executor_type = io_context::executor_type;
      using protocol_type = _InternetProtocol;
      using endpoint_type = typename _InternetProtocol::endpoint;
      using results_type = basic_resolver_results<_InternetProtocol>;

      // construct / copy / destroy:

      explicit basic_resolver(io_context& __ctx) : _M_ctx(&__ctx) { }

      basic_resolver(const basic_resolver&) = delete;

      basic_resolver(basic_resolver&& __rhs) noexcept
      : _M_ctx(__rhs._M_ctx)
      { } // TODO move state/tasks etc.

      ~basic_resolver() { cancel(); }

      basic_resolver& operator=(const basic_resolver&) = delete;

      basic_resolver& operator=(basic_resolver&& __rhs)
      {
	cancel();
	_M_ctx = __rhs._M_ctx;
	// TODO move state/tasks etc.
	return *this;
      }

      // basic_resolver operations:

      executor_type get_executor() noexcept { return _M_ctx->get_executor(); }

      void cancel() { } // TODO

      results_type
      resolve(string_view __host_name, string_view __service_name)
      {
	return resolve(__host_name, __service_name, resolver_base::flags(),
		       __throw_on_error{"basic_resolver::resolve"});
      }

      results_type
      resolve(string_view __host_name, string_view __service_name,
	      error_code& __ec)
      {
	return resolve(__host_name, __service_name, resolver_base::flags(),
		       __ec);
      }

      results_type
      resolve(string_view __host_name, string_view __service_name, flags __f)
      {
	return resolve(__host_name, __service_name, __f,
		       __throw_on_error{"basic_resolver::resolve"});
      }

      results_type
      resolve(string_view __host_name, string_view __service_name, flags __f,
	      error_code& __ec)
      { return {__host_name, __service_name, __f, __ec}; }

      template<typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, results_type)>
	async_resolve(string_view __host_name, string_view __service_name,
		      _CompletionToken&& __token)
	{
	  return async_resolve(__host_name, __service_name,
			       resolver_base::flags(),
			       forward<_CompletionToken>(__token));
	}

      template<typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, results_type)>
	async_resolve(string_view __host_name, string_view __service_name,
		      flags __f, _CompletionToken&& __token); // TODO

      results_type
      resolve(const protocol_type& __protocol,
	      string_view __host_name, string_view __service_name)
      {
	return resolve(__protocol, __host_name, __service_name,
		       resolver_base::flags(),
		       __throw_on_error{"basic_resolver::resolve"});
      }

      results_type
      resolve(const protocol_type& __protocol,
	      string_view __host_name, string_view __service_name,
	      error_code& __ec)
      {
	return resolve(__protocol, __host_name, __service_name,
		       resolver_base::flags(), __ec);
      }

      results_type
      resolve(const protocol_type& __protocol,
	      string_view __host_name, string_view __service_name, flags __f)
      {
	return resolve(__protocol, __host_name, __service_name, __f,
		       __throw_on_error{"basic_resolver::resolve"});
      }

      results_type
      resolve(const protocol_type& __protocol,
	      string_view __host_name, string_view __service_name,
	      flags __f, error_code& __ec)
      { return {__host_name, __service_name, __f, __ec, &__protocol}; }

      template<typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, results_type)>
	async_resolve(const protocol_type& __protocol,
		      string_view __host_name, string_view __service_name,
		      _CompletionToken&& __token)
	{
	  return async_resolve(__protocol, __host_name, __service_name,
			       resolver_base::flags(),
			       forward<_CompletionToken>(__token));
	}

      template<typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, results_type)>
	async_resolve(const protocol_type& __protocol,
		      string_view __host_name, string_view __service_name,
		      flags __f, _CompletionToken&& __token); // TODO

      results_type
      resolve(const endpoint_type& __ep)
      { return resolve(__ep, __throw_on_error{"basic_resolver::resolve"}); }

      results_type
      resolve(const endpoint_type& __ep, error_code& __ec)
      { return { __ep, __ec }; }

      template<typename _CompletionToken> // TODO
	__deduced_t<_CompletionToken, void(error_code, results_type)>
	async_resolve(const endpoint_type& __ep, _CompletionToken&& __token);

    private:
      io_context* _M_ctx;
    };

  /// Private constructor to synchronously resolve host and service names.
  template<typename _InternetProtocol>
    basic_resolver_results<_InternetProtocol>::
    basic_resolver_results(string_view __host_name, string_view __service_name,
			   resolver_base::flags __f, error_code& __ec,
			   protocol_type* __protocol)
    {
#ifdef _GLIBCXX_HAVE_NETDB_H
      string __host;
      const char* __h = __host_name.data()
	? (__host = __host_name.to_string()).c_str()
	: nullptr;
      string __svc;
      const char* __s = __service_name.data()
	? (__svc = __service_name.to_string()).c_str()
	: nullptr;

      ::addrinfo __hints{ };
      __hints.ai_flags = static_cast<int>(__f);
      if (__protocol)
	{
	  __hints.ai_family = __protocol->family();
	  __hints.ai_socktype = __protocol->type();
	  __hints.ai_protocol = __protocol->protocol();
	}
      else
	{
	  auto __p = endpoint_type{}.protocol();
	  __hints.ai_family = AF_UNSPEC;
	  __hints.ai_socktype = __p.type();
	  __hints.ai_protocol = __p.protocol();
	}

      struct __scoped_addrinfo
      {
	~__scoped_addrinfo() { if (_M_p) ::freeaddrinfo(_M_p); }
	::addrinfo* _M_p = nullptr;
      } __sai;

      if (int __err = ::getaddrinfo(__h, __s, &__hints, &__sai._M_p))
	{
	  __ec = ip::__make_resolver_error_code(__err, errno);
	  return;
	}
      __ec.clear();

      endpoint_type __ep;
      auto __tail = _M_results.before_begin();
      for (auto __ai = __sai._M_p; __ai != nullptr; __ai = __ai->ai_next)
	{
	  if (__ai->ai_family == AF_INET || __ai->ai_family == AF_INET6)
	    {
	      if (__ai->ai_addrlen <= __ep.capacity())
		__builtin_memcpy(__ep.data(), __ai->ai_addr, __ai->ai_addrlen);
	      __ep.resize(__ai->ai_addrlen);
	      __tail = _M_results.emplace_after(__tail, __ep, __host, __svc);
	      _M_size++;
	    }
	}
#else
      __ec = std::make_error_code(errc::operation_not_supported);
#endif
    }

  /// Private constructor to synchronously resolve an endpoint.
  template<typename _InternetProtocol>
    basic_resolver_results<_InternetProtocol>::
    basic_resolver_results(const endpoint_type& __ep, error_code& __ec)
    {
#ifdef _GLIBCXX_HAVE_NETDB_H
      char __host_name[1025];	// glibc NI_MAXHOST
      char __service_name[32];  // glibc NI_MAXSERV
      int __flags = 0;
      if (__ep.protocol().type() == SOCK_DGRAM)
	__flags |= NI_DGRAM;
      auto __sa = static_cast<const sockaddr*>(__ep.data());
      int __err = ::getnameinfo(__sa, __ep.size(),
				__host_name, sizeof(__host_name),
				__service_name, sizeof(__service_name),
				__flags);
      if (__err)
	{
	  __flags |= NI_NUMERICSERV;
	  __err = ::getnameinfo(__sa, __ep.size(),
				__host_name, sizeof(__host_name),
				__service_name, sizeof(__service_name),
				__flags);
	}
      if (__err)
	__ec = ip::__make_resolver_error_code(__err, errno);
      else
	{
	  __ec.clear();
	  _M_results.emplace_front(__ep, __host_name, __service_name);
	  _M_size = 1;
	}
#else
      __ec = std::make_error_code(errc::operation_not_supported);
#endif
    }

  /** The name of the local host.
   * @{
   */

  template<typename _Allocator>
    __string_with<_Allocator>
    host_name(const _Allocator& __a, error_code& __ec)
    {
#ifdef HOST_NAME_MAX
      constexpr size_t __maxlen = HOST_NAME_MAX;
#else
      constexpr size_t __maxlen = 256;
#endif
      char __buf[__maxlen + 1];
      if (::gethostname(__buf, __maxlen) == -1)
	__ec.assign(errno, generic_category());
      __buf[__maxlen] = '\0';
      return { __buf, __a };
    }

  template<typename _Allocator>
    inline __string_with<_Allocator>
    host_name(const _Allocator& __a)
    { return host_name(__a, __throw_on_error{"host_name"}); }

  inline string
  host_name(error_code& __ec)
  { return host_name(std::allocator<char>{}, __ec); }

  inline string
  host_name()
  { return host_name(std::allocator<char>{}, __throw_on_error{"host_name"}); }

  /// @}

#ifdef IPPROTO_TCP
  /// The TCP byte-stream protocol.
  class tcp
  {
  public:
    // types:
    using endpoint = basic_endpoint<tcp>;	 ///< A TCP endpoint.
    using resolver = basic_resolver<tcp>;	 ///< A TCP resolver.
    using socket = basic_stream_socket<tcp>;	 ///< A TCP socket.
    using acceptor = basic_socket_acceptor<tcp>; ///< A TCP acceptor.
    using iostream = basic_socket_iostream<tcp>; /// A TCP iostream.

#ifdef TCP_NODELAY
    /// Disable coalescing of small segments (i.e. the Nagle algorithm).
    struct no_delay : __sockopt_crtp<no_delay, bool>
    {
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

      static const int _S_level = IPPROTO_TCP;
      static const int _S_name = TCP_NODELAY;
    };
#endif

    // static members:

    /// A protocol object representing IPv4 TCP.
    static constexpr tcp v4() noexcept { return tcp(AF_INET); }
    /// A protocol object representing IPv6 TCP.
    static constexpr tcp v6() noexcept { return tcp(AF_INET6); }

    tcp() = delete;

    constexpr int family() const noexcept { return _M_family; }
    constexpr int type() const noexcept { return SOCK_STREAM; }
    constexpr int protocol() const noexcept { return IPPROTO_TCP; }

  private:
    constexpr explicit tcp(int __family) : _M_family(__family) { }

    int _M_family;
  };

  /** tcp comparisons
   * @{
   */

  constexpr bool
  operator==(const tcp& __a, const tcp& __b) noexcept
  { return __a.family() == __b.family(); }

  constexpr bool
  operator!=(const tcp& __a, const tcp& __b) noexcept
  { return !(__a == __b); }

  /// @}
#endif // IPPROTO_TCP

#ifdef IPPROTO_UDP
  /// The UDP datagram protocol.
  class udp
  {
  public:
    // types:
    using endpoint = basic_endpoint<udp>;
    using resolver = basic_resolver<udp>;
    using socket = basic_datagram_socket<udp>;

    // static members:
    static constexpr udp v4() noexcept { return udp(AF_INET); }
    static constexpr udp v6() noexcept { return udp(AF_INET6); }

    udp() = delete;

    constexpr int family() const noexcept { return _M_family; }
    constexpr int type() const noexcept { return SOCK_DGRAM; }
    constexpr int protocol() const noexcept { return IPPROTO_UDP; }

  private:
    constexpr explicit udp(int __family) : _M_family(__family) { }

    int _M_family;
  };

  /** udp comparisons
   * @{
   */

  constexpr bool
  operator==(const udp& __a, const udp& __b) noexcept
  { return __a.family() == __b.family(); }

  constexpr bool
  operator!=(const udp& __a, const udp& __b) noexcept
  { return !(__a == __b); }

  /// @}
#endif // IPPROTO_UDP

#if defined IPPROTO_IP && defined IPPROTO_IPV6

  /// Restrict a socket created for an IPv6 protocol to IPv6 only.
  class v6_only : public __sockopt_crtp<v6_only, bool>
  {
  public:
    using __sockopt_crtp::__sockopt_crtp;
    using __sockopt_crtp::operator=;

  private:
    friend __sockopt_crtp<v6_only, bool>;
    static const int _S_level = IPPROTO_IPV6;
    static const int _S_name = IPV6_V6ONLY;
  };

  namespace unicast
  {
    /// Set the default number of hops (TTL) for outbound datagrams.
    class hops : public __sockopt_crtp<hops>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

      template<typename _Protocol>
	int
	level(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? IPPROTO_IPV6 : IPPROTO_IP; }

      template<typename _Protocol>
	int
	name(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? IPV6_UNICAST_HOPS : IP_TTL; }
    };
  } // namespace unicast

  namespace multicast
  {
    class __mcastopt
    {
    public:
      explicit
      __mcastopt(const address& __grp) noexcept
      : __mcastopt(__grp.is_v4() ? __mcastopt(__grp.to_v4()) : __mcastopt(__grp.to_v6()))
      { }

      explicit
      __mcastopt(const address_v4& __grp,
		 const address_v4& __iface = address_v4::any()) noexcept
      {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	_M_v4.imr_multiaddr.s_addr = __grp.to_uint();
	_M_v4.imr_interface.s_addr = __iface.to_uint();
#else
	_M_v4.imr_multiaddr.s_addr = __builtin_bswap32(__grp.to_uint());
	_M_v4.imr_interface.s_addr = __builtin_bswap32(__iface.to_uint());
#endif
      }

      explicit
      __mcastopt(const address_v6& __grp, unsigned int __iface = 0) noexcept
      {
	const auto __addr = __grp.to_bytes();
	__builtin_memcpy(_M_v6.ipv6mr_multiaddr.s6_addr, __addr.data(), 16);
	_M_v6.ipv6mr_interface = __iface;
      }

      template<typename _Protocol>
	int
	level(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? IPPROTO_IPV6 : IPPROTO_IP; }

      template<typename _Protocol>
	const void*
	data(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? &_M_v6 : &_M_v4; }

      template<typename _Protocol>
	size_t
	size(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? sizeof(_M_v6) : sizeof(_M_v4); }

    private:
      ipv6_mreq _M_v6 = {};
      ip_mreq _M_v4 = {};
    };

    /// Request that a socket joins a multicast group.
    class join_group : private __mcastopt
    {
    public:
      using __mcastopt::__mcastopt;
      using __mcastopt::level;
      using __mcastopt::data;
      using __mcastopt::size;

      template<typename _Protocol>
	int
	name(const _Protocol& __p) const noexcept
	{
	  if (__p.family() == AF_INET6)
	    return IPV6_JOIN_GROUP;
	  return IP_ADD_MEMBERSHIP;
	}
    };

    /// Request that a socket leaves a multicast group.
    class leave_group : private __mcastopt
    {
    public:
      using __mcastopt::__mcastopt;
      using __mcastopt::level;
      using __mcastopt::data;
      using __mcastopt::size;

      template<typename _Protocol>
	int
	name(const _Protocol& __p) const noexcept
	{
	  if (__p.family() == AF_INET6)
	    return IPV6_LEAVE_GROUP;
	  return IP_DROP_MEMBERSHIP;
	}
    };

    /// Specify the network interface for outgoing multicast datagrams.
    class outbound_interface
    {
    public:
      explicit
      outbound_interface(const address_v4& __v4) noexcept
      {
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	_M_v4.s_addr = __v4.to_uint();
#else
	_M_v4.s_addr = __builtin_bswap32(__v4.to_uint());
#endif
      }

      explicit
      outbound_interface(unsigned int __v6) noexcept
      : _M_v4(), _M_v6(__v6)
      { }

      template<typename _Protocol>
	int
	level(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? IPPROTO_IPV6 : IPPROTO_IP; }

      template<typename _Protocol>
	int
	name(const _Protocol& __p) const noexcept
	{
	  return __p.family() == AF_INET6
	    ? IPV6_MULTICAST_IF : IP_MULTICAST_IF;
	}

      template<typename _Protocol>
	const void*
	data(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? &_M_v6 : &_M_v4; }

      template<typename _Protocol>
	size_t
	size(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? sizeof(_M_v6) : sizeof(_M_v4); }

    private:
      in_addr _M_v4;
      unsigned _M_v6 = 0;
    };

    /// Set the default number of hops (TTL) for outbound datagrams.
    class hops : public __sockopt_crtp<hops>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

      template<typename _Protocol>
	int
	level(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? IPPROTO_IPV6 : IPPROTO_IP; }

      template<typename _Protocol>
	int
	name(const _Protocol& __p) const noexcept
	{
	  return __p.family() == AF_INET6
	    ? IPV6_MULTICAST_HOPS : IP_MULTICAST_TTL;
	}
    };

    /// Set whether datagrams are delivered back to the local application.
    class enable_loopback : public __sockopt_crtp<enable_loopback, bool>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

      template<typename _Protocol>
	int
	level(const _Protocol& __p) const noexcept
	{ return __p.family() == AF_INET6 ? IPPROTO_IPV6 : IPPROTO_IP; }

      template<typename _Protocol>
	int
	name(const _Protocol& __p) const noexcept
	{
	  return __p.family() == AF_INET6
	    ? IPV6_MULTICAST_LOOP : IP_MULTICAST_LOOP;
	}
    };

  } // namespace multicast

#endif // IPPROTO_IP && IPPROTO_IPV6

  /// @}

} // namespace ip
} // namespace v1
} // namespace net
} // namespace experimental

  template<>
    struct is_error_condition_enum<experimental::net::v1::ip::resolver_errc>
    : public true_type {};

  // hash support
  template<typename _Tp> struct hash;
  template<>
    struct hash<experimental::net::v1::ip::address>
    : __hash_base<size_t, experimental::net::v1::ip::address>
    {
      size_t
      operator()(const experimental::net::v1::ip::address& __a) const
      {
	if (__a.is_v4())
	  return _Hash_impl::hash(__a.to_v4());
	else
	  return _Hash_impl::hash(__a.to_v6());
      }
    };

  template<>
    struct hash<experimental::net::v1::ip::address_v4>
    : __hash_base<size_t, experimental::net::v1::ip::address_v4>
    {
      size_t
      operator()(const experimental::net::v1::ip::address_v4& __a) const
      { return _Hash_impl::hash(__a.to_bytes()); }
    };

  template<> struct hash<experimental::net::v1::ip::address_v6>
    : __hash_base<size_t, experimental::net::v1::ip::address_v6>
    {
      size_t
      operator()(const experimental::net::v1::ip::address_v6& __a) const
      { return _Hash_impl::hash(__a.to_bytes()); }
    };

_GLIBCXX_END_NAMESPACE_VERSION
} // namespace std

#endif // C++14

#endif // _GLIBCXX_EXPERIMENTAL_INTERNET
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          '       @            0       D            7       H            8       L            9       P            "      T            "      X            
"      \            "      `            "      d            "      h            "      l            "      p            #      t            #      x            #      |            #                  #                  #                  #                  #                  #                  $                  $                  $                  $                  $                  A%                  P%                  V%                  Z%                  %                  %                  %                  %                  %                  %                  %                  %                  %                  %                  %                  %                  &                  &                  &                  &                  &                  &                   &                  &                  &                  &                  &                  &                  @'                  E'                   W'      $            `'      (            g'      ,            i'      0            k'      4            m'      8            n'      <            r'      @            v'      D            J(      H            N(      L            O(      P            Q(      T            S(      X            U(      \            W(      `            \(      d            ,      h            ,      l            ,      p            ,      t            ,      x            ,      |            ,                  ,                  ,                  ,                  ,                  ,                  e.                  h.                  j.                  l.                  n.                  p.                  u.                   /                  /                  /                  /                  /                  	/                  /                  /                  /                  /                  /                   /                  !/                  2/                  50                  <0                  =0                  ?0                  A0                  C0                   H0                  2                  2                  2                  2                  2                  2                  2                   2      $            2      (            2      ,            2      0            2      4            2      8            3      <            3      @            3      D            
3      H            v;      L                    P                   T                    X                   \            $       `            C       d            T       h            ,      l                  p                    t                                                                         o                                        S'                                                           `                   2                                                         %                   0                                                  8                    P                     .symtab .strtab .shstrtab .note.gnu.build-id .note.Linux .rela.text .rela.init.text .rela.text.unlikely .rela.exit.text .rela__mcount_loc .rodata.str1.1 .rela.smp_locks .rodata.str1.8 .rela.rodata .modinfo .rela__param .rodata.cst2 .rela.return_sites .orc_unwind .rela.orc_unwind_ip __versions .rela__bug_table .rela.data .rela.exit.data .rela.init.data .data.once .rela.gnu.linkonce.this_module .bss .comment .note.GNU-stack .BTF .gnu_debuglink                                                                                       @       $                              .                     d       <                              ?                            v;                             :      @                     ,      -                    J                     <                                    E      @                     x       -                    Z                     5<                                   U      @                           -                    n                     /?                                    i      @                     0       -   	                 ~                     ;?                                   y      @                           -                          2               @      H                                                 B      T                                    @                           -                          2               hB                                                        E                                          @                            -                                         L                                                        N                                          @                           -                                         hO                                                        lO                                          @                     P      -                                         $P      	                             
                    Y      x                                  @                     &      -                                        ``                                    ,                     v      $                              '     @                            -                    =                    `v                                    8     @               0!            -                    H                    (x                                    C     @               !            -   !                 X                    0x                                    S     @               !            -   #                 c                    8x                                    s                    @x                    @               n     @               "     0       -   &                                     {                                         0               {      P                                                  |                                                          |      K                                                  \                                                          h            .                    	                            
                                                   8"                                  0	*H
01
0	`He0	*H
1a0]080 10UDebian Secure Boot CA2(oe:B&C0	`He0
	*H
  .iLUtbǶb3Z(v|^r
*= T 0+B|p>H:c.ds:aJIL^$=/TjtѿJsmXnnҋGIud 7wSߌ#
Aq'mÙJo;͂^q	iőF_>O1tjFhH~i2vj         ~Module signature appended~
 070701000A03EB000041ED000000000000000000000004688C084600000000000000CA0000000200000000000000000000004400000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/broadcom   070701000A03F6000081A4000000000000000000000001682F6DA700018EDB000000CA0000000200000000000000000000004B00000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/broadcom/b44.ko    ELF          >                    ؀         @     @ 1 0          GNU %(Z uRA*        Linux                Linux   6.1.0-37-amd64      US  H%	     H  HH@    H     ߃    DHH@(    H     HH@    H     [     ]DHH@(    f         UHS   H  HH@    H]HH`  uԻ  H  HH@    H]HH  u[]    ff.     f    AVIAUIATAUS1I  T    HH@(    I  ؃   AT HH@(    D9r[]A\A]A^         H`  HH  HVHX  HVH  HVH  H  H   H   H(  HV(Hp  HV@H  HVHH  H  HVPH  HVXH  HVhH  HV`H  H  H  H  H  H  H  HV H   H                X          X          F   
  F    fD      T  FT  FT  F    f         /   D    fD      ATL	  UHSHL    LZL    1H  HL HH=x  u[]A\    ff.          HcAWIHAVA   AUIATIHUSfPD AD$ŉ։LÃHcH79uA`   A   A   LcLc1MA4MA4HIHcD<7L9uD9tD   )9Nڃ[]A\A]A^A_    f.         F    T  F    F1fF         t    H    HzH    HHH    H  H)H)ց  H        ATIU1SI\$0HH; t6I$  HsE1     Hx    H;       H    HH    u1I\$8HHHt4PpHsE1   I$  Hx    H;       H    HH    u[]A\    D      ATIUHST     t H  H       [1]A\    ȃ   D   E   E   ҉⦃dVV  F	 V(
  V
T  H~0   F`
@`  ҉˾O  D    H}@    ID$8ME    1E[]A\         ATL	  UHSHL    E T  t7
   T  L    uH 
  Hx    1[]A\    %         U   SH 
  HH~H    Hh    E t*vw[]    H{d   [H    ]    HE(H   Ht?    HHd    HH    H tHH     HH[]    H   ff.         AW  HcAVH4    HAUATUSL<    IHHHLg0LoLIOt= II$HE H        ID$HE   ucAE %   A  DAAEAFI$       u[H  Hu     HxH[]A\A]A^A_    H  H  1DD$Hc    Hx    DD$nH  H  1Hc    HxL        AUATUSHG8tjH	  HIAH    W  t.H  Ht;DL    H    []A\A]    H@
  Iu1D    ff.         SHH0    HC0    H{8    HSHC8    HtAH  H  E1Hx      H         HC    HSHt=H  H  E1Hx  @u%H         HC    [              H{    1ɺ       H{    nff.         USH  H     H98
     8
     t+     uT     u
   T  H 
  0  HH@    H 
  0  V  DHH@(    H[]     t  u3T     u[]    T     t%T  tuՋT     uPff.         AUATL	  ULSH    H 
      HH@    H 
  $   HH@    !tVHC8    L	  L    uLH 
         HH@(    H 
      HH@    L    []A\A]    H 
  	  1Ҿ$   HH@(    L             SHl      l  Hl  [    f    ATUSHo0HH    W  uG1H    H{    H	          H    HCx t=HC0    []A\    H
  L0
  H      L    L    Hx(       HC0    []A\    ff.     f    AV      AUATUSHH H  eH%(   HD$1HH@(      V  H     HH@(    H  H     H0  PHH@(    H  H     H0  HfH@(    H  1HH@    H  H1H@(    H  1fD$HGP trH  H@    H    
   @HH@(    H  D   HT$H@Hx(    T$D   H  H@Hx(    HD$eH+%(     H []A\A]A^    H=       
  HD$    HD$        HH*   HT$HE1HD$    H  HD$    H0     HHߺ   Ak      HHt$TL   HHH     HT$HD$    HHD$    H0  t  HHߺ   A     HHt$L   HHH  >   HT$HD$    HHD$    H0     HHߺ   A   HA   Ht$rH    H  D   HD	D	H@(       H  1HH@    H  q             AUAպ   ATAAUHESH    HH@(    H  ھ  1HD	D	H@(  P    Ƨ  H    Hdt)H    HH@    t1[]A\A]        u    fD      Hǀ	  /ff.     @     HPff.     @     AUAպ   ATIAUHSH    HH@(    H  ھ  1HD	  `H@(    Ƨ  H    HdtJH    HH@    t1H    HH@    A$[]A\A]        u    ff.         HHǀ	  eH%(   HD$1HL$uD$HT$eH+%(   uH        f         HHPeH%(   HD$1HL$uD$HT$eH+%(   uH        ff.         ATE1UH	  SHHHeH%(   HD$1D$        W  tXA      W  t)1H    HD$eH+%(   uqH[]A\    D(
  1H(
  1HL$HW  u(
  HL$1HDd$iw            USHeH%(   HD$1D$      t1HT$eH+%(   ugH[]      1ҹ   Hu˿ 1      t
   t      1HL$Hul$    fD      AUAATUHSL  L    H  1    ID$LH         H  1Ҿ   1HH@(    H     ,  HH@(    Ƨ  H    H     H  ,  HH@    uH  1Ҿ   HH@(    H  HE       HH@         H  1Ҿ  HH@(    HE(       8     H  HH@(    H  HHH@    H`  u޻  H  HHH@    H  uA   I|$G ]     wH    HH@    H  1HH@    H  Hu@H@(   ,      H  ,  HH@         []A\A]    1H@    Āt9H  1HH@(    H  1HH@           []A\A]        H    %& Hik_kHH@(H5Ҁʀ    1Ƨ  H    HdtVH    HH@     tH       HH@(                    f    SHH  1Ҿ$   HH@(    H  $   HH@    HH  H              H  HH@(      tH߾   [&H߾   [         AW      AVAUATUH  SHheH%(   HD$`1        HI  IƋ
    HEM	  I 
  Ip     M
  AƆ)
   w1t
   AX  @   H    LH     Aǆ	      I 
  I	  Iǆ       HH$    1H<     Aǆ    I   E(Iǆ      Al  H}    Å    Lm?L    Å    ?L    I 
  H    AP  HCP u=    ~H    H    I
     1A(
      I
  H0      R	    I 
  Aǆ	  x$vAT  A(
  <    A@
  LH      AT    MX
  Iǆ`
      Iǆh
      IH
      Å    L    Lu0L   L    AW    1H|$    M 
  H1HD$    HD$    IGHD$    IH    L`PMe=   H    H@X    LH@`    H@    IGI  A(
  
    A      M0
  H    L        A(
  I0
      uHD$<      E(
  L@   H    H|$     I
     Ht$ H        IH=     Hl$Hl$Hl$Hl$	L  ]   HT$LL    I$  LI$  I$  I$   Aǆ8
      A$(  A(
      H}       I0  H    LH        HD$`eH+%(   uHh[]A\A]A^A_            SHeH%(   HD$1$      tHD$eH+%(   4  H[      H   Huǋ  $              Ƀ!΃@   EΉ@΀   EΉ΁      EΉ΁   ĀE΅Q     HC5  (HD$eH+%(   \    HH߹   1[       H       H   H  $@        H|n        D$    1Ʌu.  HL$H9+  L$   Ή΁       EΩ   tL$u  1HHD$eH+%(   uHH1[1    f         AUATL	  UHSHHeH%(   HD$1D$    W  t`H      L    HC8   H  H    L    HD$eH+%(   '  H[]A\A]    DnH|$Hv@    }t{AdtA
uv}wpL    }T  tf%
      AdET  }t$HC8uYL1    aLa;T     D$0t7T$%T  u
   T  Lt
   T  t
T     t
T     \T     M             UH1Ҿ(  SH  HH@(    H     t[]    H0  H     SHH@(    H  $  1HfH@(       H     (  HH@(    Ƨ  H    HdtTH  (  HH@    xH  (  HH@    H  [(  ]HH@(        t    f    AW   AVAUATUSHHH 
  HH@    A   At-H 
  D   HH@(H[]A\A]A^A_    H	  g   uD  A   AA   H 
  1Ҿ   1HH@(    H 
     $  HH@(    H 
  D(  HH@(    Ƨ  H    Hd  H 
  (  HH@    xAA@bH 
  D   HH@(    H 
  (  HH@    H 
  (  HH@(H[]A\A]A^A_    L  H  HD$I9   E   E1AV*Dl$   1H 
  AHH@(    H 
  AV($  HfH@(       H 
  D(  HH@(    Ƨ  H    HdtgH 
  (  HH@    xM6E9tHD$I9DDl$AA?A               t    ff.     @     UH	  SHH    HH[]    D      U   SH  H        HH@(    H        HH@(    H  H    H     H2H@(    H  H       H2H@(    H  8   4  HH@(    H  HH@(            H        HH@(    H  =     HH@(    H        HH@(    H      HH@(      H     8  C(HH@(    H  ,  HH@    H  ,  HH@(    H  H     [H   ]    =         HMH    ATUSH_0HC8u[1]A\    L	  HH`      L    LH
      H
      LL    l  H    T  xHEx uHx(       rL   0LxfD      UH    H	  SH  HH      W  uiH	      H`      H    H!HYH    H    l  H    T  xH1[]    H      뉾   HeHH1[]    ff.         HG8   VuZN	tRUHV   1SHH	      H    H 
     HH@    tH    1[]        HH    1[]            AVAUATUSHHeH%(   HD$1  $    D$       B    H   t%HD$eH+%(     H[]A\A]A^        ,$  tË  T$H  %   E%   E  HG8^  @|  H  0  E1E1HH@    H  0  D$      D$    DHH@(      AA     H      H    @        
     H  H@8H  0  HH@    H  0    DHH@(    H      HD$eH+%(     HH[]A\A]A^      HL$   H=!NHG8"@    H    HG8   t9A   A   uA   E1A   AADH  HL$   H  u&  HL$   HwYDl$Dd$m        fD      UH SHH    HHH    H=    H       H[]H    ff.     @     AWAVE1AUATUSHHLo0xHcHMt5    
    H  HD$HI    IH  H   H  HH$Lx    H$  Hй   %  H  H   H+
    H4
LE1A   H  HHH5        HH  HxHt%H  H=?   E1         L       1Ҿ!
        IH  H   H  HH$Lx    H$  Hй   %  H  H   H+
    H4
LE1A   H  HHH5        HH  HxH  H  H=?   ƃ  I$        Me IuMtI       Ld$  DHSIL  r   u  H[]A\A]A^A_    H  H  1Hc    HxL    À=     IWP    HuILH$    H$H    H    IE1            L    Y=     uI_P    HtL    HH    H    IH
    sH
    I@     UHSaHU1HzH    Hǂ      HH)   HHUHzH    Hǂ      HH)   H      u:   @u\  1ۅu
;  sھH	y[]    H  H     Hx         @tH        H  Hx    f    ATL	  UHSHL    uT     ̀T  Mt
   T  Eցt   HC8T  t%L   tSLo   L2L    	  $   H 
  HH@(    1[]A\    [f    ATUSHG8Hu   1[]A\    L	  L    L   L   LL    	  $   H 
  HH@(    1[]A\    fD      ATI1USHH_0H    Ņ    HC8uH[]A\    L	  L    L9   LL    l  1IIع   H            H
      	  $   H 
  HH@(    H      H5    H`  H    H[]A\        ~     ATUSVH   F   ~      L	  HL    CL牅 
  C $
  hL0   LH
  H      L    	  $   H 
  HH@(    1[]A\        ff.          AWHGAVAUATUHHSH t$H$    HD$Eȩ  uT@5  Ht$H<$E1    Eȩ   =     D$A9   H D[]A\A]A^A_    H@    1HH@    DmAA  E9uO  H@  AWpE1IvHy       I    1LA_p    AE%  AA92  EILuM>MuDH    H@  Uľ$   HH@(    +L<$E1L    LHLR   LHH  H      HL    H@    HH@    ]%  D$9}  D|$E   j  f0  i   UfD$H<$u
f=   t$ 
  Ht$    IH   t$H    I$   T$I   H    HH  LAA    LfA       E  %  E9\$   E   غ     HHEL LhH@  LHx    I$   f= w
BUH<$HH  H`  kD$   !  HT$    HT$fl$uE1H@  ]      HH@(    EH@  1Ҿ$   EHH@(    H@  $   HH@    H@  1    H$HH߾   HH  H      T}HcH@  E1L   MHx    t$L       L    HH  L  t=I$       D$ A$   A$   9x
I$   rfHH  DmH  H   t&Md  D9sDD)ȍ =   v    H@  1Ҿ(   HH@(    EML    )B*=   vff.         AU    
  ATUH	  SHH=        H	  H?  H=        
      H	  H  H 
  H
  E1     Hx    H	  H  H 
  H
  E1     Hx    H	  H  L	  L    H   HHl  I1Iع   H        Aąn  H`  1E11H    H    H    HH   Hp      	  $   H 
  HH@(    W    H     [D]A\A]    H=       
      IH  H 
  LL`    [  L     L  H   H+    H4E1E1HH   LHH5        H=  H   H?)  T      L	  H
  /H=       
      IH   H 
  LL`       L     L	  H   H+    H4E1H   HA   LHH5        HtxH   H?whT     @L	  H
  H      K=     u1    I\$PHuI$L    HH    H    L    HAHL       HHvHH    ?H        AWL	  AVAUIATUHSH_pL    A	  H$A	  90  A$
  9H  H   I 
  AHHT$Lp    HT$  Hй   %  H  H   H+
    H4
E1LA   HLHHH5        II 
  HxIt#KH=?  E1   LL    1Ҿ!
      IH  H   I 
  H@HHT$HD$    HT$|  Hй   %  H  H   H+
    H4
E1LHH|$HA   HH5        II 
  HxIF  KH=?#  LLL$    H   LH    H1L    LL$A	  I	    HH4    IHH,I	  LLځ      =  EI	  I	  EP  DL0AW  @/  AA  E	  I 
  B      HH@(    AT  t!I 
  ھ  HH@(    AT     UpI                9w  A	  A$
  A	  9sl)с      1H4$L    H[]A\A]A^A_    A$
  )=   I 
    HH@    C9I 
  Hc       I
  Hx    I  H    H      VE1   LL       L       .I  H    H      H    L    Ȁ=     IVP    HtgLHT$    HT$H    H    xH    H      D$       9Y   LI딀=     %HD$    HXPHuHH|$    HH    H    H
    H
    HH        H        H    H            SHH  HG8t
H    [      H    H    H       HD%   ҃⦃d      H    H    HH  [   HE   HEHH        H    I       H            H    I       H            H  H            H  ,  I       H            H    I        H            H}H        H}    H<$        L        H        H}H        H}H        A(
  IH        LDd$    L    ED$    H}H    D$    D$LD$    \$@IH    D$    L    D$A(
  IH        AƆ(
   E1H    H    @   H|$         H}H        H}H    D$    D$MIH        H}H        sH  (  I       H            H
  (  I       H            H
  (  I       H            HD$eH+%(   u7H  HH    []A\A]A^    H  H                H    H߉D$    L    L    L    L    l$    I|$H                UH    H	  SH    H    H    H       H    H    	  $   H 
  HH@(    H  []        SH               Åu%H    H        ÅtH        [    H        H                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   drivers/net/ethernet/broadcom/b44.c     Link is up at %d Mbps, %s duplex
       Flow control is %s for TX and %s for RX
        BUG!  Timeout waiting for bit %08x of register %lx to %s
       Required 30BIT DMA mask unsupported by the system
      3b44: Invalid MAC address found in EEPROM
     Problem fetching invariants of chip, aborting
  No PHY present on this MAC, aborting
   Cannot register net device, aborting
   could not find PHY at %i, use fixed one
        Cannot register PHY, aborting
  Broadcom 44xx/47xx 10/100 PCI ethernet driver   %s %s: rejecting DMA map of vmalloc memory
     transmit timed out, resetting
  BUG! Tx Ring full when queue awake!
    include/linux/dynamic_queue_limits.h b44 SSB late interrupt
 full half on off Link is down
 set PHY Reset would not complete
 clear powering down PHY
 Failed to powerup the bus
 phy reset failed
 mdiobus_alloc() failed
 b44_eth_mii %x failed to register MII bus
 fixed-0 %s:%02x could not attach PHY at %i
 %s %pM
 Remote fault detected in PHY
 Jabber detected in PHY
 include/linux/dma-mapping.h request_irq failed
                                        strnlen strscpy                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 tx_good_octets                  tx_good_pkts                    tx_octets                       tx_pkts                         tx_broadcast_pkts               tx_multicast_pkts               tx_len_64                       tx_len_65_to_127                tx_len_128_to_255               tx_len_256_to_511               tx_len_512_to_1023              tx_len_1024_to_max              tx_jabber_pkts                  tx_oversize_pkts                tx_fragment_pkts                tx_underruns                    tx_total_cols                   tx_single_cols                  tx_multiple_cols                tx_excessive_cols               tx_late_cols                    tx_defered                      tx_carrier_lost                 tx_pause_pkts                   rx_good_octets                  rx_good_pkts                    rx_octets                       rx_pkts                         rx_broadcast_pkts               rx_multicast_pkts               rx_len_64                       rx_len_65_to_127                rx_len_128_to_255               rx_len_256_to_511               rx_len_512_to_1023              rx_len_1024_to_max              rx_jabber_pkts                  rx_oversize_pkts                rx_fragment_pkts                rx_missed_pkts                  rx_crc_align_errs               rx_undersize                    rx_crc_errs                     rx_align_errs                   rx_symbol_errs                  rx_pause_pkts                   rx_nonpause_pkts                CB                             D                            D                                                                                              b44_debug parm=b44_debug:B44 bitmapped debugging message enable value parmtype=b44_debug:int license=GPL description=Broadcom 44xx/47xx 10/100 PCI ethernet driver author=Felix Fietkau, Florian Schirmer, Pekka Pietikainen, David S. Miller alias=pci:v000014E4d0000170Csv*sd*bc*sc*i* alias=pci:v000014E4d00004402sv*sd*bc*sc*i* alias=pci:v000014E4d00004401sv*sd*bc*sc*i* alias=ssb:v4243id0806rev* depends=libphy,mii,ssb retpoline=Y intree=Y name=b44 vermagic=6.1.0-37-amd64 SMP preempt mod_unload modversions                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    (  0  (                                                                                                               (    0  8  0  (                                                                                                                                                                                                                (    0  8  @  8  0  (                     @                 (                 (                                                                           (                 (                                                                                     (  0  P  0  (                   P                       (                 (                                           (                 (                                                                 0               0                (          (                       (                 (                 (                                                   (    0  8    8  0  (                                                                                                      (  8  (                 8                                                             (    0  8  H  8  0  (                           H  8  0  (                           H                                                                                                                                           (  0  H  0  (                   H  0  (                   H                                             (    0  8  H  8  0  (                     H                                                                                                                     (               (                                                                          (    0  8  X  8  0  (                     X                       (                 (                         (    0  8  P  8  0  (                     P          (                      (  (  (      H  H  0  (                   H  (                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          m    __fentry__                                              pHe    __x86_indirect_thunk_rax                                9[    __x86_return_thunk                                      {'    _raw_spin_lock_irq                                      SuK    _raw_spin_unlock_irq                                    o4    dma_unmap_page_attrs                                    @    __dev_kfree_skb_any                                     L    phy_ethtool_ksettings_get                               0j    ethtool_convert_legacy_u32_to_link_mode                 d/    device_set_wakeup_enable                                9d    strscpy                                                     strnlen                                                     fortify_panic                                           !1    dma_sync_single_for_device                              A
    dma_sync_single_for_cpu                                 lΥ    phy_mii_ioctl                                           J    generic_mii_ioctl                                       z    kfree                                                   Tn    dma_free_attrs                                          6S    phy_print_status                                        ,    ssb_pcihost_register                                    i    __ssb_driver_register                                   *v    pci_unregister_driver                                   (e    ssb_driver_unregister                                   d    _raw_spin_lock                                              napi_schedule_prep                                      4K    _raw_spin_unlock                                        a*    __napi_schedule                                         _5    netdev_info                                             o<    disable_irq                                             	    enable_irq                                              @    unregister_netdev                                       H7˘    ssb_device_disable                                      ؝    ssb_bus_may_powerdown                                   Aw    __netif_napi_del                                        ~`    synchronize_net                                         m    free_netdev                                             '    phy_disconnect                                          c    mdiobus_unregister                                      e>    mdiobus_free                                            4~    pci_set_power_state                                     Q|K    pci_read_config_word                                    g    pci_write_config_word                                   -    kmalloc_caches                                          wX    kmalloc_trace                                           V
    __stack_chk_fail                                            __const_udelay                                          ;    net_ratelimit                                           ,    netdev_err                                                  ssb_device_is_enabled                                   zF    ssb_device_enable                                       16    ssb_pcicore_dev_irqvecs_enable                          qqZ    ssb_clockspeed                                          .    alloc_etherdev_mqs                                      ,X    netif_napi_add_weight                                   ?    ssb_bus_powerup                                         G}    dma_set_mask                                                dma_set_coherent_mask                                   
(    ssb_dma_translation                                     (
    dev_addr_mod                                            xB<    register_netdev                                             netif_carrier_off                                       =    mdiobus_alloc_size                                      nJne    snprintf                                                "    __mdiobus_register                                      zaK    mdiobus_is_registered_device                            '    phy_connect                                             W    __bitmap_and                                            Vv
    phy_attached_info                                       d*    device_set_wakeup_capable                               9I    _dev_err                                                ~    _printk                                                 ]    _dev_info                                               
w    phy_ethtool_ksettings_set                               U    ethtool_convert_link_mode_to_legacy_u32                 	WN    dql_reset                                               ܐ    timer_delete_sync                                       BW\    netif_device_detach                                     ;JQ    free_irq                                                P    jiffies                                                 ұ    napi_disable                                            :C    phy_stop                                                I    netif_carrier_on                                        j    netdev_warn                                             	    round_jiffies                                               mod_timer                                               #    __netdev_alloc_skb                                      ΰ    is_vmalloc_addr                                         ^|    page_offset_base                                        le    vmemmap_base                                            ٣    dma_map_page_attrs                                      5    __alloc_skb                                             ם    dev_driver_string                                       GV    __warn_printk                                           (L    phys_base                                               Ւ    request_threaded_irq                                    "*    netif_device_attach                                     a    netif_tx_wake_queue                                     4    _raw_spin_lock_irqsave                                  p\    _raw_spin_unlock_irqrestore                             \g    __dev_kfree_skb_irq                                     "    napi_complete_done                                      V_    __napi_alloc_skb                                        S>    skb_put                                                 8߬i    memcpy                                                  y6    eth_type_trans                                               netif_receive_skb                                       !`    skb_pull                                                *
    dql_completed                                           h>y    netif_schedule_queue                                    MxN    dma_alloc_attrs                                         p*    napi_enable                                             9c    init_timer_key                                          s$    add_timer                                               n")    phy_start                                               X    eth_validate_addr                                       ^    ethtool_op_get_link                                          param_ops_int                                           zR    module_layout                                                   +          	        g                    +	        c          L		        L		        r          L		        L		        M           L		                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        b44                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      GCC: (Debian 12.2.0-14+deb12u1) 12.2.0  GCC: (Debian 12.2.0-14+deb12u1) 12.2.0             O         
9             X    w          
Y A=      h        #       P=        W=    0   _= 
  0   h        %{            @   k=    `   x=      =      =      :"  %     =  
     = ^ @        
        %                              ] =      b        =       @             (          
_ = 
  &   = $       = $      = $      = $      > $       > $   (   > vO  0   "> vO  p   )> vO     /> vO              6> #       MY
 #      PY
 #      j #               9> $       A> $      L> $      W> $      ^> $             
   g> c     l> c (   q>     @ $       {> 	     > 	  @   > 	  p   > 	     > $      > $      > $      > $      > $      > $      `  &      > &     > &      > &   0  > $   @   r  H  ? $   X  ? $   `  "? $   h  2? $   p  C? &     I? &     O? &     U? &     [? &     a? &     g? &     o? &     w? &      ? &     ? &      ? &   0  ? $   @  ? $   H  ? $   P  ? $   X  ? $   `  ? $   h  ? $   p  ? $   x  ? $     ? $     ? $     ? $     ? $     ? $     @ )    	@ )    @ )    @ )    #@ #   0  *@ #   8  1@ $   @  ;@ $   H  E@ $   P  O@ $   X  U@ $   `  _@ $   h  i@ $   p  s@ $   x  y@ &     @ *     @ *     @ *     @ *      @ *      @ *   @  @ *   `  @ &     @ &     @ &     @ &     A f   A b 	  A d 	  #A cQ  	  +A cQ  p
  3A cQ  
  <A cQ  p  EA $     IA ~     VA ~     dA ~   (  rA ~   @  A ~   X  A ~   p  A ~     A ~     A ~     A ~     A $     A $     A $     A $    
  A &   
  A &    
  A &   0
  B &   @
  B $   P
  B $   X
  !B &   `
  .B $   p
  8B $   x
  HB $   
  RB $   
  cB $   
  sB $   
  B $   
  B $   
  B $   
  B $   
  B $   
  B $   
  B $   
  B $   
  B $   
  B &   
  B &      
C *      C *   @  ,C *   `  =C *     PC *     aC *     tC *     C *      C *      C *   @  C *   `  C *     C *     C *     C *     C *      D *      D *   @  D *   `  ,D *     9D &     AD &     PD $     VD $                a       \D      b &       c  &      jD   @   vD m     |D o @   D q    D s    D u    D w @  D y   D {          
h       
$       k     &          l D 	  @   )  |     :"    @           ~    x   _    D $   0  i     @  = k     D k            j       
&       k     &          n       
*       k     &          p       
        k     &       $          r       
        k     &       &          t       
        k     &       *          v       
        k     k       Q       &       $          x       
        k     l       Q       &       $          z        i D     i	 k       )  | @   D k            D *     x          @  E      D G    M &     E $     E &     E &     &E $        @  3E $   @#  >E    `#  HE  #  OE  $  WE  @%  `E  @%  R g @%  fE e `%  lE K   @6    `   6  }E K    7  E     7         } E      V           @               !      )   @  Ŗ     E            `       
       k                    
        k               
       k                     
       k         E      E     E    E    E    E      Z  $       F *       F   (   :"  k     Q  *   @     *   `   "F &      3F *     p      =F *      JF *      WF        dF        nF      :"  k     zF $   @  F $   A  F $   B           F $       F *                F       F      F      F                F                           l       F      3v  +        +      F      4&  +      ]  +     hS                 +        G             ,  W   @   G 0  x  G -       (G -   @   5G -      ?G -      GG -      YG -   @  kG -     uG -     G -      G -   @  G -     G -     G -      G -   @  G -     H -     H -      H -   @  +H -     <H -     NH -      [H -   @  fH -     vH -     H -      H -   @  H -     H -     H -      H -   @  H -     H -     H -      I -   @  I -     (I -     ;I -    	  JI -   @	  [I -   	  lI -   	  {I -    
  I -   @
  I -   
  I -   
  I -      I -   @  I -           I     4  *      I *       I *   @   W     _     I *      I *      J *   @  
J *   `  J    J    B  f     (J *     ]  *     3J *     %  I     7
  @   k    :"  %  @  >J W     JJ W     _ *      _ *      VJ $   @  _J $   H  ! _    oJ      xJ \           
                            `              
                           
                \        [                        e                         J       J    p         
    k J           
    k     J           
     k J           
    k X  J     J    j        
   :"  %  
   :    J           
   :"  %  ^ !?  J           
    :"  %  ^ !?  J           
    :"  %  0  I?      J     K      "K            
   :"  %  2K =?  9K           
    :"  %  2K =?  LK    È       
   :"  %  _K 4?  eK 5?   s	  rK    ň       
    :"  %  _K 4?  eK 5?   s	  K    ǈ       
   :"  %  : }?  K    Ɉ       
   :"  %  : z?  K    ˈ K    j        
    :"  %  P  ?  K    Έ       
    :"  %  ^  *   K    Ј K      L    j  L    j        
    :"  %  'L 
  -L    Ո =L    j        
    @   GL    ؈       
   UL O-  ]L    q
    "     fL    ڈ       
    @   -     * *   xL *   L    ܈ L    j  L    j        
    @   L    L     L    j  L    ؈ L    ؈ L     M    ؈ M    ؈ 'M    ؈ 6M    ZI EM    s        
    :"  %  TM    \M     kM    P  yM    #       
    @   M    M *   M           
   @   M    M *   M     M    }Y  M    ؈ M    ؈ M    ؈       
    @   M *   N           
   @   N           
     _  h    2j      &   #N           
     _  h    2j    9N           
    :"  %  h    2j         NN           
   :"  %  h    2j    aN           
   @   VJ    S      *   sN           
   @   VJ    S         N     mii_ioctl_data val_in val_out mii_if_info reg_num_mask full_duplex force_media supports_gmii mdio_read mdio_write ssb_device_id coreid ssb_sprom_core_pwr_info itssi_2g itssi_5g maxpwr_2g maxpwr_5gl maxpwr_5g maxpwr_5gh pa_2g pa_5gl pa_5g pa_5gh a0 tssipos extpa_gain pdet_range tr_iso antswlut ghz2 ghz5 ssb_sprom il0mac et0mac et1mac et2mac et0phyaddr et1phyaddr et2phyaddr et0mdcport et1mdcport et2mdcport board_rev board_num board_type country_code leddc_on_time leddc_off_time ant_available_a ant_available_bg pa0b0 pa0b1 pa0b2 pa1b0 pa1b1 pa1b2 pa1lob0 pa1lob1 pa1lob2 pa1hib0 pa1hib1 pa1hib2 gpio0 gpio1 gpio2 gpio3 maxpwr_bg maxpwr_al maxpwr_a maxpwr_ah itssi_a itssi_bg tri2g tri5gl tri5g tri5gh txpid2g txpid5gl txpid5g txpid5gh rxpo2g rxpo5g rssisav2g rssismc2g rssismf2g bxa2g rssisav5g rssismc5g rssismf5g bxa5g cck2gpo ofdm2gpo ofdm5glpo ofdm5gpo ofdm5ghpo boardflags boardflags2 boardflags3 boardflags_lo boardflags_hi boardflags2_lo boardflags2_hi core_pwr_info antenna_gain fem mcs2gpo mcs5gpo mcs5glpo mcs5ghpo opo rxgainerr2ga rxgainerr5gla rxgainerr5gma rxgainerr5gha rxgainerr5gua noiselvl2ga noiselvl5gla noiselvl5gma noiselvl5gha noiselvl5gua regrev txchain rxchain antswitch cddpo stbcpo bw40po bwduppo tempthresh tempoffset rawtempsense measpower tempsense_slope tempcorrx tempsense_option freqoffset_corr iqcal_swp_dis hw_iqcal_en elna2g elna5g phycal_tempdelta temps_period temps_hysteresis measpower1 measpower2 pcieingress_war cckbw202gpo cckbw20ul2gpo legofdmbw202gpo legofdmbw20ul2gpo legofdmbw205glpo legofdmbw20ul5glpo legofdmbw205gmpo legofdmbw20ul5gmpo legofdmbw205ghpo legofdmbw20ul5ghpo mcsbw202gpo mcsbw20ul2gpo mcsbw402gpo mcsbw205glpo mcsbw20ul5glpo mcsbw405glpo mcsbw205gmpo mcsbw20ul5gmpo mcsbw405gmpo mcsbw205ghpo mcsbw20ul5ghpo mcsbw405ghpo mcs32po legofdm40duppo sar2g sar5g ssb_boardinfo ssb_bus_ops read8 read16 read32 write8 write16 write32 block_read block_write ssb_device core_index devtypedata ssb_bus mapped_device bar_lock sprom_mutex chip_rev sprom_offset sprom_size chip_package nr_devices busnumber chipco pcicore mipscore extif sprom has_cardbus_slot powered_up power_warn_count ssb_driver ssb_bustype SSB_BUSTYPE_SSB SSB_BUSTYPE_PCI SSB_BUSTYPE_PCMCIA SSB_BUSTYPE_SDIO ssb_chipcommon_pmu crystalfreq ssb_chipcommon fast_pwrup_delay gpio_lock ticks_per_ms max_timer_ms ssb_mipscore ssb_extif ssb_pcicore setup_done hostmode cardbusmode mapped_pcmcia_seg sdio_sbaddr host_pci host_pcmcia host_sdio pcmcia_device sdio_func dma_desc rx_header ring_info b44_hw_stats tx_good_octets tx_good_pkts tx_octets tx_pkts tx_broadcast_pkts tx_multicast_pkts tx_len_64 tx_len_65_to_127 tx_len_128_to_255 tx_len_256_to_511 tx_len_512_to_1023 tx_len_1024_to_max tx_jabber_pkts tx_oversize_pkts tx_fragment_pkts tx_underruns tx_total_cols tx_single_cols tx_multiple_cols tx_excessive_cols tx_late_cols tx_defered tx_carrier_lost tx_pause_pkts rx_good_octets rx_good_pkts rx_octets rx_pkts rx_broadcast_pkts rx_multicast_pkts rx_len_64 rx_len_65_to_127 rx_len_128_to_255 rx_len_256_to_511 rx_len_512_to_1023 rx_len_1024_to_max rx_jabber_pkts rx_oversize_pkts rx_fragment_pkts rx_missed_pkts rx_crc_align_errs rx_undersize rx_crc_errs rx_align_errs rx_symbol_errs rx_pause_pkts rx_nonpause_pkts b44 imask istat tx_prod tx_cons rx_prod rx_cons rx_buffers tx_buffers dma_offset msg_enable rx_ring_dma tx_ring_dma phy_addr force_copybreak old_link mii_if b44_cleanup b44_init b44_resume b44_suspend b44_remove_one b44_init_one b44_adjust_link b44_ioctl b44_set_wol b44_get_wol b44_get_ethtool_stats b44_get_sset_count b44_get_strings epause b44_set_pauseparam b44_get_pauseparam ering kernel_ering b44_set_ringparam b44_get_ringparam b44_set_link_ksettings b44_get_link_ksettings b44_nway_reset b44_get_drvinfo b44_set_msglevel b44_get_msglevel b44_set_rx_mode __b44_set_rx_mode nstat b44_get_stats64 b44_close b44_setup_wol macaddr ppattern b44_magic_pattern table_offset bwfilter_table b44_poll_controller b44_open reset_kind b44_init_hw b44_set_mac_addr __b44_set_mac_addr b44_halt b44_chip_reset b44_free_consistent b44_init_rings b44_free_rings b44_change_mtu b44_start_xmit txqueue b44_tx_timeout b44_interrupt b44_poll src_idx dest_idx_unmasked b44_recycle_rx b44_alloc_rx_skb b44_timer b44_check_phy b44_link_report b44_stats_update pause_flags __b44_set_flow_ctrl b44_phy_reset b44_mdio_write_phylib b44_mdio_read_phylib b44_mdio_write_mii b44_mdio_read_mii __b44_writephy __b44_readphy  b44.ko  2	                                                                                               	                      
                                                                                         &                      )                             +                  +       +     :      +       @     e             U                  l                   y                             	                  	                  <                                       $                                       s       
    0      m                        )                 :                 K                 ]          7       p                     0      R                            p      -                 O                                                                    h                                            +                  7                   C          0      R   )               e    	             o    	                 
                        O                   (                             	                                                             .                        
    
      x          ,              (                  7           )       K                 ^                  t                             )                  W           `      T                                                              `      N          ;      R                            P            %   )                                  .    5            7                  F          0      V                h          w      }    p                 0"  // <experimental/socket> -*- C++ -*-

// Copyright (C) 2015-2022 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/** @file experimental/socket
 *  This is a TS C++ Library header.
 *  @ingroup networking-ts
 */

#ifndef _GLIBCXX_EXPERIMENTAL_SOCKET
#define _GLIBCXX_EXPERIMENTAL_SOCKET

#pragma GCC system_header

#if __cplusplus >= 201402L

#include <experimental/netfwd>
#include <experimental/buffer>
#include <experimental/io_context>
#include <experimental/bits/net.h>
#include <streambuf>
#include <istream>
#include <bits/unique_ptr.h>
#if _GLIBCXX_HAVE_UNISTD_H
# include <unistd.h>
# ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
#  include <sys/socket.h>	// socket etc
# endif
# ifdef _GLIBCXX_HAVE_SYS_IOCTL_H
#  include <sys/ioctl.h>	// ioctl
# endif
# ifdef _GLIBCXX_HAVE_SYS_UIO_H
#  include <sys/uio.h>		// iovec
# endif
# ifdef _GLIBCXX_HAVE_POLL_H
#  include <poll.h>		// poll, pollfd, POLLIN, POLLOUT, POLLERR
# endif
# ifdef _GLIBCXX_HAVE_FCNTL_H
#  include <fcntl.h>		// fcntl, F_GETFL, F_SETFL
# endif
#endif

namespace std _GLIBCXX_VISIBILITY(default)
{
_GLIBCXX_BEGIN_NAMESPACE_VERSION
namespace experimental
{
namespace net
{
inline namespace v1
{

  /** @addtogroup networking-ts
   *  @{
   */

  enum class socket_errc {  // TODO decide values
    already_open = 3,
    not_found = 4
  };

} // namespace v1
} // namespace net
} // namespace experimental

  template<>
    struct is_error_code_enum<experimental::net::v1::socket_errc>
    : public true_type {};

namespace experimental
{
namespace net
{
inline namespace v1
{
  const error_category& socket_category() noexcept
  {
    struct __cat : error_category
    {
      const char* name() const noexcept { return "socket"; }

      std::string message(int __e) const
      {
	if (__e == (int)socket_errc::already_open)
	  return "already open";
	else if (__e == (int)socket_errc::not_found)
	  return "endpoint not found";
	return "socket error";
      }

      virtual void __message(int) { } // TODO dual ABI XXX
    };
    static __cat __c;
    return __c;
  }

  inline error_code
  make_error_code(socket_errc __e) noexcept
  { return error_code(static_cast<int>(__e), socket_category()); }

  inline error_condition
  make_error_condition(socket_errc __e) noexcept
  { return error_condition(static_cast<int>(__e), socket_category()); }


  // TODO GettableSocket reqs
  // TODO SettableSocket reqs
  // TODO BooleanSocketOption reqs
  // TODO IntegerSocketOption reqs
  // TODO IoControlCommand reqs
  // TODO ConnectCondition reqs

  /** @brief Sockets
   * @{
   */

  class socket_base
  {
  public:
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
    class broadcast : public __sockopt_crtp<broadcast, bool>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<broadcast, bool>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_BROADCAST;
    };

    class debug : public __sockopt_crtp<debug, bool>
    {
    public:
      friend __sockopt_crtp<debug, bool>;
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_DEBUG;
    };

    class do_not_route : public __sockopt_crtp<do_not_route, bool>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<do_not_route, bool>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_DONTROUTE;
    };

    class keep_alive : public __sockopt_crtp<keep_alive, bool>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<keep_alive, bool>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_KEEPALIVE;
    };

    class linger : public __sockopt_crtp<linger, ::linger>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

      linger() noexcept = default;

      linger(bool __e, chrono::seconds __t) noexcept
      {
	enabled(__e);
	timeout(__t);
      }

      bool
      enabled() const noexcept
      { return _M_value.l_onoff != 0; }

      void
      enabled(bool __e) noexcept
      { _M_value.l_onoff = int(__e); }

      chrono::seconds
      timeout() const noexcept
      { return chrono::seconds(_M_value.l_linger); }

      void
      timeout(chrono::seconds __t) noexcept
      { _M_value.l_linger = __t.count(); }

    private:
      friend __sockopt_crtp<linger, ::linger>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_LINGER;
    };

    class out_of_band_inline : public __sockopt_crtp<out_of_band_inline, bool>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<out_of_band_inline, bool>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_OOBINLINE;
    };

    class receive_buffer_size : public __sockopt_crtp<receive_buffer_size>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<receive_buffer_size>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_RCVBUF;
    };

    class receive_low_watermark : public __sockopt_crtp<receive_low_watermark>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<receive_low_watermark>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_RCVLOWAT;
    };

    class reuse_address : public __sockopt_crtp<reuse_address, bool>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<reuse_address, bool>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_REUSEADDR;
    };

    class send_buffer_size : public __sockopt_crtp<send_buffer_size>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<send_buffer_size>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_SNDBUF;
    };

    class send_low_watermark : public __sockopt_crtp<send_low_watermark>
    {
    public:
      using __sockopt_crtp::__sockopt_crtp;
      using __sockopt_crtp::operator=;

    private:
      friend __sockopt_crtp<send_low_watermark>;
      static const int _S_level = SOL_SOCKET;
      static const int _S_name = SO_SNDLOWAT;
    };
#endif // HAVE_SYS_SOCKET_H

    enum shutdown_type : int { };
#if defined SHUT_RD && defined SHUT_WR && defined SHUT_RDWR
    static constexpr shutdown_type shutdown_receive = (shutdown_type)SHUT_RD;
    static constexpr shutdown_type shutdown_send    = (shutdown_type)SHUT_WR;
    static constexpr shutdown_type shutdown_both    = (shutdown_type)SHUT_RDWR;
#endif

    enum wait_type : int { };
#ifdef _GLIBCXX_HAVE_POLL_H
    static constexpr wait_type wait_read  = (wait_type)POLLIN;
    static constexpr wait_type wait_write = (wait_type)POLLOUT;
    static constexpr wait_type wait_error = (wait_type)POLLERR;
#else
    static constexpr wait_type wait_read  = (wait_type)1;
    static constexpr wait_type wait_write = (wait_type)2;
    static constexpr wait_type wait_error = (wait_type)4;
#endif

    enum message_flags : int { };
#if defined MSG_PEEK && defined MSG_OOB && defined MSG_DONTROUTE
    static constexpr message_flags message_peek
      = (message_flags)MSG_PEEK;
    static constexpr message_flags message_out_of_band
      = (message_flags)MSG_OOB;
    static constexpr message_flags message_do_not_route
      = (message_flags)MSG_DONTROUTE;
#endif

#ifdef SOMAXCONN
    static constexpr int max_listen_connections = SOMAXCONN;
#else
    static constexpr int max_listen_connections = 4;
#endif

    // message_flags bitmask operations are defined as hidden friends.

    friend constexpr message_flags
    operator&(message_flags __f1, message_flags __f2) noexcept
    { return message_flags( int(__f1) & int(__f2) ); }

    friend constexpr message_flags
    operator|(message_flags __f1, message_flags __f2) noexcept
    { return message_flags( int(__f1) | int(__f2) ); }

    friend constexpr message_flags
    operator^(message_flags __f1, message_flags __f2) noexcept
    { return message_flags( int(__f1) ^ int(__f2) ); }

    friend constexpr message_flags
    operator~(message_flags __f) noexcept
    { return message_flags( ~int(__f) ); }

    friend constexpr message_flags&
    operator&=(message_flags& __f1, message_flags __f2) noexcept
    { return __f1 = (__f1 & __f2); }

    friend constexpr message_flags&
    operator|=(message_flags& __f1, message_flags __f2) noexcept
    { return __f1 = (__f1 | __f2); }

    friend constexpr message_flags&
    operator^=(message_flags& __f1, message_flags __f2) noexcept
    { return __f1 = (__f1 ^ __f2); }

#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
  protected:
    struct __msg_hdr : ::msghdr
    {
#ifdef IOV_MAX
      using __iovec_array = array<::iovec, IOV_MAX>;
#elif _GLIBCXX_HAVE_UNISTD_H
      struct __iovec_array
      {
	__iovec_array() : _M_ptr(new ::iovec[size()]) { }

	::iovec& operator[](size_t __n) noexcept { return _M_ptr[__n]; }

	::iovec* data() noexcept { return _M_ptr.get(); }

	static size_t size()
	{
	  static const size_t __iov_max = ::sysconf(_SC_IOV_MAX);
	  return __iov_max;
	}

      private:
	unique_ptr<::iovec[]> _M_ptr;
      };
#else
      using __iovec_array = array<::iovec, 16>;
#endif

      __iovec_array _M_iov;

      template<typename _BufferSequence>
	explicit
	__msg_hdr(const _BufferSequence& __buffers)
	: msghdr()
	{
	  auto __buf = net::buffer_sequence_begin(__buffers);
	  const auto __bufend = net::buffer_sequence_end(__buffers);
	  size_t __len = 0;
	  while (__buf != __bufend && __len != _M_iov.size())
	    {
	      _M_iov[__len].iov_base = (void*)__buf->data();
	      _M_iov[__len].iov_len = __buf->size();
	      ++__buf;
	      ++__len;
	    }
	  this->msg_iovlen = __len;
	  this->msg_iov = _M_iov.data();
	}

      template<typename _BufferSequence, typename _Endpoint>
	__msg_hdr(const _BufferSequence& __buffers, const _Endpoint& __ep)
	: __msg_hdr(__buffers)
	{
	  this->msg_name = __ep.data();
	  this->msg_namelen = __ep.size();
	}
    };
#endif

  protected:
    socket_base() = default;
    ~socket_base() = default;
  };

  // TODO define socket_base static constants in .so for C++14 mode

#if _GLIBCXX_HAVE_UNISTD_H

  class __socket_impl
  {
  protected:

    using executor_type = io_context::executor_type;
    using native_handle_type = int;

    explicit
    __socket_impl(io_context& __ctx) : _M_ctx(std::addressof(__ctx)) { }

    __socket_impl(__socket_impl&& __rhs)
    : _M_ctx(__rhs._M_ctx),
      _M_sockfd(std::__exchange(__rhs._M_sockfd, -1)),
      _M_bits(std::__exchange(__rhs._M_bits, {}))
    { }

    __socket_impl&
    operator=(__socket_impl&& __rhs)
    {
      _M_ctx = __rhs._M_ctx;
      _M_sockfd = std::__exchange(__rhs._M_sockfd, -1);
      _M_bits = std::__exchange(__rhs._M_bits, {});
      return *this;
    }

    ~__socket_impl() = default;

    __socket_impl(const __socket_impl&) = delete;
    __socket_impl& operator=(const __socket_impl&) = delete;

    executor_type get_executor() noexcept { return _M_ctx->get_executor(); }

    native_handle_type native_handle() noexcept { return _M_sockfd; }

    bool is_open() const noexcept { return _M_sockfd != -1; }

    void
    close(error_code& __ec)
    {
      if (is_open())
	{
	  cancel(__ec);
	  if (!__ec)
	    {
	      if (::close(_M_sockfd) == -1)
		__ec.assign(errno, generic_category());
	      else
		{
		  get_executor().context()._M_remove_fd(_M_sockfd);
		  _M_sockfd = -1;
		}
	    }
	}
    }

    void cancel(error_code& __ec) { _M_ctx->cancel(_M_sockfd, __ec); }

    void
    non_blocking(bool __mode, error_code&)
    { _M_bits.non_blocking = __mode; }

    bool non_blocking() const { return _M_bits.non_blocking; }

    void
    native_non_blocking(bool __mode, error_code& __ec)
    {
#if defined _GLIBCXX_HAVE_FCNTL_H && defined _GLIBCXX_HAVE_DECL_O_NONBLOCK
      int __flags = ::fcntl(_M_sockfd, F_GETFL, 0);
      if (__flags >= 0)
	{
	  if (__mode)
	    __flags |= O_NONBLOCK;
	  else
	    __flags &= ~O_NONBLOCK;
	  __flags = ::fcntl(_M_sockfd, F_SETFL, __flags);
	}
      if (__flags == -1)
	__ec.assign(errno, generic_category());
      else
	{
	  __ec.clear();
	  _M_bits.native_non_blocking = __mode;
	}
#else
      __ec = std::make_error_code(std::errc::not_supported);
#endif
    }

    bool
    native_non_blocking() const
    {
#if defined _GLIBCXX_HAVE_FCNTL_H && defined _GLIBCXX_HAVE_DECL_O_NONBLOCK
      if (_M_bits.native_non_blocking == -1)
	{
	  const int __flags = ::fcntl(_M_sockfd, F_GETFL, 0);
	  if (__flags == -1)
	    return 0;
	  _M_bits.native_non_blocking = __flags & O_NONBLOCK;
	}
      return _M_bits.native_non_blocking;
#else
      return false;
#endif
    }

    io_context*	_M_ctx;
    int		_M_sockfd{-1};
    struct {
      unsigned		non_blocking : 1;
      mutable signed	native_non_blocking : 2;
      unsigned		enable_connection_aborted : 1;
    } _M_bits{};
  };

  template<typename _Protocol>
    class __basic_socket_impl : public __socket_impl
    {
      using __base = __socket_impl;

    protected:
      using protocol_type = _Protocol;
      using endpoint_type = typename protocol_type::endpoint;

      explicit
      __basic_socket_impl(io_context& __ctx) : __base(__ctx) { }

      __basic_socket_impl(__basic_socket_impl&&) = default;

      template<typename _OtherProtocol>
	__basic_socket_impl(__basic_socket_impl<_OtherProtocol>&& __rhs)
	: __base(std::move(__rhs)), _M_protocol(std::move(__rhs._M_protocol))
	{ }

      __basic_socket_impl&
      operator=(__basic_socket_impl&& __rhs)
      {
	if (this == std::addressof(__rhs))
	  return *this;
	_M_close();
	__base::operator=(std::move(__rhs));
	return *this;
      }

      ~__basic_socket_impl() { _M_close(); }

      __basic_socket_impl(const __basic_socket_impl&) = delete;
      __basic_socket_impl& operator=(const __basic_socket_impl&) = delete;

      void
      open(const protocol_type& __protocol, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	if (is_open())
	  __ec = socket_errc::already_open;
	else
	  {
	    _M_protocol = __protocol;
	    _M_sockfd = ::socket(__protocol.family(), __protocol.type(),
				 __protocol.protocol());
	    if (is_open())
	      {
		get_executor().context()._M_add_fd(_M_sockfd);
	      __ec.clear();
	      }
	    else
	      __ec.assign(errno, std::generic_category());
	  }
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
      }

      void
      assign(const protocol_type& __protocol,
	     const native_handle_type& __native_socket,
	     error_code& __ec)
      {
	if (is_open())
	  __ec = socket_errc::already_open;
	else
	  {
	    _M_protocol = __protocol;
	    _M_bits.native_non_blocking = -1;
	    _M_sockfd = __native_socket;
	    if (is_open())
	      {
		get_executor().context()._M_add_fd(_M_sockfd);
		__ec.clear();
	      }
	    else
	      __ec.assign(errno, std::generic_category());
	  }
      }

      native_handle_type release(error_code& __ec)
      {
	__glibcxx_assert(is_open());
	cancel(__ec);
	return std::__exchange(_M_sockfd, -1);
      }

      template<typename _SettableSocketOption>
	void
	set_option(const _SettableSocketOption& __option, error_code& __ec)
	{
# ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  int __result = ::setsockopt(_M_sockfd, __option.level(_M_protocol),
				      __option.name(_M_protocol),
				      __option.data(_M_protocol),
				      __option.size(_M_protocol));
	  if (__result == -1)
	    __ec.assign(errno, generic_category());
	  else
	    __ec.clear();
#else
	  __ec = std::make_error_code(std::errc::not_supported);
#endif
	}

      template<typename _GettableSocketOption>
	void
	get_option(_GettableSocketOption& __option, error_code& __ec) const
	{
# ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  int __result = ::getsockopt(_M_sockfd, __option.level(_M_protocol),
				      __option.name(_M_protocol),
				      __option.data(_M_protocol),
				      __option.size(_M_protocol));
	  if (__result == -1)
	    __ec.assign(errno, generic_category());
	  else
	    __ec.clear();
#else
	  __ec = std::make_error_code(std::errc::not_supported);
#endif
	}

      template<typename _IoControlCommand>
	void
	io_control(_IoControlCommand& __command, error_code& __ec)
	{
#ifdef _GLIBCXX_HAVE_SYS_IOCTL_H
	  int __result = ::ioctl(_M_sockfd, __command.name(),
				 __command.data());
	  if (__result == -1)
	    __ec.assign(errno, generic_category());
	  else
	    __ec.clear();
#else
	  __ec = std::make_error_code(std::errc::not_supported);
#endif
	}

      endpoint_type
      local_endpoint(error_code& __ec) const
      {
	endpoint_type __endpoint;
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	socklen_t __endpoint_len = __endpoint.capacity();
	if (::getsockname(_M_sockfd, (sockaddr*)__endpoint.data(),
			  &__endpoint_len) == -1)
	  {
	    __ec.assign(errno, generic_category());
	    return endpoint_type{};
	  }
	__ec.clear();
	__endpoint.resize(__endpoint_len);
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
	return __endpoint;
      }

      void
      bind(const endpoint_type& __endpoint, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	if (::bind(_M_sockfd, (sockaddr*)__endpoint.data(), __endpoint.size())
	    == -1)
	  __ec.assign(errno, generic_category());
	else
	  __ec.clear();
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
      }

      _Protocol	_M_protocol{ endpoint_type{}.protocol() };

    private:
      void
      _M_close()
      {
	if (is_open())
	  {
	    error_code __ec;
	    cancel(__ec);
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	    set_option(socket_base::linger{false, chrono::seconds{}}, __ec);
#endif
	    ::close(_M_sockfd);
	  }
      }
    };

  template<typename _Protocol>
    class basic_socket
    : public socket_base, private __basic_socket_impl<_Protocol>
    {
      using __base = __basic_socket_impl<_Protocol>;

    public:
      // types:

      using executor_type = io_context::executor_type;
      using native_handle_type = int;
      using protocol_type = _Protocol;
      using endpoint_type = typename protocol_type::endpoint;

      static_assert(__detail::__protocol<protocol_type>,
		    "protocol_type meets the Protocol requirements");

      // basic_socket operations:

      executor_type get_executor() noexcept { return __base::get_executor(); }

      native_handle_type
      native_handle() noexcept { return __base::native_handle(); }

      void
      open(const protocol_type& __protocol = protocol_type())
      { open(__protocol, __throw_on_error{"basic_socket::open"}); }

      void
      open(const protocol_type& __protocol, error_code& __ec)
      { __base::open(__protocol, __ec); }

      void
      assign(const protocol_type& __protocol,
	     const native_handle_type& __native_socket)
      {
	assign(__protocol, __native_socket,
	       __throw_on_error{"basic_socket::assign"});
      }

      void
      assign(const protocol_type& __protocol,
	     const native_handle_type& __native_socket,
	     error_code& __ec)
      { __base::assign(__protocol, __native_socket, __ec); }

      native_handle_type release()
      { return release(__throw_on_error{"basic_socket::release"}); }

      native_handle_type release(error_code& __ec)
      { return __base::release(__ec); }

      _GLIBCXX_NODISCARD bool
      is_open() const noexcept { return __base::is_open(); }

      void close() { close(__throw_on_error{"basic_socket::close"}); }

      void close(error_code& __ec) { __base::close(__ec); }

      void cancel() { cancel(__throw_on_error{"basic_socket::cancel"}); }

      void cancel(error_code& __ec) { __base::cancel(__ec); }

      template<typename _SettableSocketOption>
	void
	set_option(const _SettableSocketOption& __option)
	{ set_option(__option, __throw_on_error{"basic_socket::set_option"}); }

      template<typename _SettableSocketOption>
	void
	set_option(const _SettableSocketOption& __option, error_code& __ec)
	{ __base::set_option(__option, __ec); }

      template<typename _GettableSocketOption>
	void
	get_option(_GettableSocketOption& __option) const
	{ get_option(__option, __throw_on_error{"basic_socket::get_option"}); }

      template<typename _GettableSocketOption>
	void
	get_option(_GettableSocketOption& __option, error_code& __ec) const
	{ __base::get_option(__option, __ec); }

      template<typename _IoControlCommand>
	void
	io_control(_IoControlCommand& __command)
	{
	  io_control(__command, __throw_on_error{"basic_socket::io_control"});
	}

      template<typename _IoControlCommand>
	void
	io_control(_IoControlCommand& __command, error_code& __ec)
	{ __base::io_control(__command, __ec); }

      void
      non_blocking(bool __mode)
      { non_blocking(__mode, __throw_on_error{"basic_socket::non_blocking"}); }

      void
      non_blocking(bool __mode, error_code& __ec)
      { __base::non_blocking(__mode, __ec); }

      bool non_blocking() const { return __base::non_blocking(); }

      void
      native_non_blocking(bool __mode)
      {
	native_non_blocking(__mode, __throw_on_error{
	    "basic_socket::native_non_blocking"});
      }

      void
      native_non_blocking(bool __mode, error_code& __ec)
      { __base::native_non_blocking(__mode, __ec); }

      bool
      native_non_blocking() const
      { return __base::native_non_blocking(); }

      bool at_mark() const
      { return at_mark(__throw_on_error{"basic_socket::at_mark"}); }

      bool
      at_mark(error_code& __ec) const
      {
#ifdef _GLIBCXX_HAVE_SOCKATMARK
	const int __result = ::sockatmark(native_handle());
	if (__result == -1)
	  {
	    __ec.assign(errno, generic_category());
	    return false;
	  }
	__ec.clear();
	return (bool)__result;
#else
	__ec = std::make_error_code(errc::operation_not_supported);
	return false;
#endif
      }

      size_t
      available() const
      { return available(__throw_on_error{"basic_socket::available"}); }

      size_t
      available(error_code& __ec) const
      {
	if (!is_open())
	  {
	    __ec = std::make_error_code(errc::bad_file_descriptor);
	    return 0;
	  }
#if defined _GLIBCXX_HAVE_SYS_IOCTL_H && defined FIONREAD
	int __avail = 0;
	if (::ioctl(this->_M_sockfd, FIONREAD, &__avail) == -1)
	  {
	    __ec.assign(errno, generic_category());
	    return 0;
	  }
	__ec.clear();
	return __avail;
#else
	return 0;
#endif
      }

      void
      bind(const endpoint_type& __endpoint)
      { return bind(__endpoint, __throw_on_error{"basic_socket::bind"}); }

      void
      bind(const endpoint_type& __endpoint, error_code& __ec)
      { __base::bind(__endpoint, __ec); }

      void shutdown(shutdown_type __what)
      { return shutdown(__what, __throw_on_error{"basic_socket::shutdown"}); }

      void
      shutdown(shutdown_type __what, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	if (::shutdown(native_handle(), static_cast<int>(__what)) == -1)
	  __ec.assign(errno, generic_category());
	else
	  __ec.clear();
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
      }

      endpoint_type
      local_endpoint() const
      {
	return local_endpoint(
	    __throw_on_error{"basic_socket::local_endpoint"});
      }

      endpoint_type
      local_endpoint(error_code& __ec) const
      { return __base::local_endpoint(__ec); }

      endpoint_type
      remote_endpoint() const
      {
	return remote_endpoint(
	    __throw_on_error{"basic_socket::remote_endpoint"});
      }

      endpoint_type
      remote_endpoint(error_code& __ec) const
      {
	endpoint_type __endpoint;
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	socklen_t __endpoint_len = __endpoint.capacity();
	if (::getpeername(this->_M_sockfd, (sockaddr*)__endpoint.data(),
			  &__endpoint_len)
	    == -1)
	  {
	    __ec.assign(errno, generic_category());
	    return endpoint_type{};
	  }
	__ec.clear();
	__endpoint.resize(__endpoint_len);
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
	return __endpoint;
      }

      void
      connect(const endpoint_type& __endpoint)
      {
	return connect(__endpoint, __throw_on_error{"basic_socket::connect"});
      }

      void
      connect(const endpoint_type& __endpoint, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	if (!is_open())
	  {
	    open(__endpoint.protocol(), __ec);
	    if (__ec)
	      return;
	  }
	if (::connect(native_handle(), (const sockaddr*)__endpoint.data(),
		      __endpoint.size()) == -1)
	  __ec.assign(errno, generic_category());
	else
	  __ec.clear();
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
      }

      template<typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code)>
	async_connect(const endpoint_type& __endpoint,
		      _CompletionToken&& __token)
	{
	  async_completion<_CompletionToken, void(error_code)> __init{__token};

	  if (!is_open())
	    {
	      error_code __ec;
	      open(__endpoint.protocol(), __ec);
	      if (__ec)
		{
                  auto __ex = net::get_associated_executor(
                      __init.completion_handler, get_executor());
                  auto __a = get_associated_allocator(
                      __init.completion_handler, std::allocator<void>());
                  __ex.post(
                      [__h = std::move(__init.completion_handler), __ec]
                      () mutable
                      { __h(__ec); }, __a);
		  return __init.result.get();
		}
	    }

	  get_executor().context().async_wait( native_handle(),
	      (int) socket_base::wait_read,
	      [__h = std::move(__init.completion_handler),
               __ep = std::move(__endpoint),
               __fd = native_handle()]
               (error_code __ec) mutable {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  if (!__ec && ::connect(__fd, (const sockaddr*)__ep.data(),
					 __ep.size()) == -1)
                    __ec.assign(errno, generic_category());
#else
		  __ec = std::make_error_code(errc::operation_not_supported);
#endif
		  __h(__ec);
	      });
	  return __init.result.get();
	}

      void
      wait(wait_type __w)
      { return wait(__w, __throw_on_error{"basic_socket::wait"}); }

      void
      wait(wait_type __w, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_POLL_H
	::pollfd __fd;
	__fd.fd = native_handle();
	__fd.events = static_cast<int>(__w);
	int __res = ::poll(&__fd, 1, -1);
	if (__res == -1)
	  __ec.assign(errno, generic_category());
	else
	  __ec.clear();
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
      }

      template<typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code)>
	async_wait(wait_type __w, _CompletionToken&& __token)
	{
	  async_completion<_CompletionToken, void(error_code)> __init{__token};
	  get_executor().context().async_wait( native_handle(),
	      static_cast<int>(__w),
	      [__h = std::move(__init.completion_handler)]
              (error_code __ec) mutable {
		  __h(__ec);
	      });
	  return __init.result.get();
	}

    protected:
      // construct / copy / destroy:

      using __base::__base;

      explicit
      basic_socket(io_context& __ctx) : __base(__ctx) { }

      basic_socket(io_context& __ctx, const protocol_type& __protocol)
      : __base(__ctx)
      { open(__protocol); }

      basic_socket(io_context& __ctx, const endpoint_type& __endpoint)
      : basic_socket(__ctx, __endpoint.protocol())
      { bind(__endpoint); }

      basic_socket(io_context& __ctx, const protocol_type& __protocol,
		   const native_handle_type& __native_socket)
      : __base(__ctx)
      { assign(__protocol, __native_socket); }

      basic_socket(const basic_socket&) = delete;

      basic_socket(basic_socket&& __rhs) = default;

      template<typename _OtherProtocol, typename _Requires
	       = _Require<is_convertible<_OtherProtocol, _Protocol>>>
	basic_socket(basic_socket<_OtherProtocol>&& __rhs)
	: __base(std::move(__rhs)) { }

      ~basic_socket() = default;

      basic_socket& operator=(const basic_socket&) = delete;

      basic_socket& operator=(basic_socket&& __rhs) = default;

      template<typename _OtherProtocol>
	enable_if_t<is_convertible<_OtherProtocol, _Protocol>::value,
		    basic_socket&>
	operator=(basic_socket<_OtherProtocol>&& __rhs)
        { return *this = basic_socket{std::move(__rhs)}; }
    };

  template<typename _Protocol>
    class basic_datagram_socket : public basic_socket<_Protocol>
    {
      using __base = basic_socket<_Protocol>;

    public:
      // types:

      using native_handle_type = int;
      using protocol_type = _Protocol;
      using endpoint_type = typename protocol_type::endpoint;

      // construct / copy / destroy:

      explicit
      basic_datagram_socket(io_context& __ctx) : __base(__ctx) { }

      basic_datagram_socket(io_context& __ctx, const protocol_type& __protocol)
      : __base(__ctx, __protocol) { }

      basic_datagram_socket(io_context& __ctx, const endpoint_type& __endpoint)
      : __base(__ctx, __endpoint) { }

      basic_datagram_socket(io_context& __ctx, const protocol_type& __protocol,
			    const native_handle_type& __native_socket)
      : __base(__ctx, __protocol, __native_socket) { }

      basic_datagram_socket(const basic_datagram_socket&) = delete;

      basic_datagram_socket(basic_datagram_socket&& __rhs) = default;

      template<typename _OtherProtocol, typename _Requires
	       = _Require<is_convertible<_OtherProtocol, _Protocol>>>
	basic_datagram_socket(basic_datagram_socket<_OtherProtocol>&& __rhs)
	: __base(std::move(__rhs)) { }

      ~basic_datagram_socket() = default;

      basic_datagram_socket& operator=(const basic_datagram_socket&) = delete;

      basic_datagram_socket& operator=(basic_datagram_socket&& __rhs) = default;

      template<typename _OtherProtocol>
	enable_if_t<is_convertible<_OtherProtocol, _Protocol>::value,
		    basic_datagram_socket&>
	operator=(basic_datagram_socket<_OtherProtocol>&& __rhs)
	{
	  __base::operator=(std::move(__rhs));
	  return *this;
	}

      // basic_datagram_socket operations:

      template<typename _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers)
	{
	  return receive(__buffers, socket_base::message_flags(),
			 __throw_on_error{"basic_datagram_socket::receive"});
	}

      template<typename _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers, error_code& __ec)
        { return receive(__buffers, socket_base::message_flags(), __ec); }

      template<typename _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers,
		       socket_base::message_flags __flags)
	{
	  return receive(__buffers, __flags,
			 __throw_on_error{"basic_datagram_socket::receive"});
	}

      template<typename _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers,
		socket_base::message_flags __flags, error_code& __ec)
	{
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  socket_base::__msg_hdr __msg(__buffers);
	  ssize_t __result = ::recvmsg(this->native_handle(), &__msg,
				       static_cast<int>(__flags));
	  if (__result == -1)
            {
              __ec.assign(errno, generic_category());
              return 0;
            }
          __ec.clear();
          return __result;
#else
	  __ec = std::make_error_code(errc::operation_not_supported);
	  return 0;
#endif
	}

      template<typename _MutableBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_receive(const _MutableBufferSequence& __buffers,
		      _CompletionToken&& __token)
	{
	  return async_receive(__buffers, socket_base::message_flags(),
			       std::forward<_CompletionToken>(__token));
	}

      template<typename _MutableBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_receive(const _MutableBufferSequence& __buffers,
		      socket_base::message_flags __flags,
		      _CompletionToken&& __token)
	{
          async_completion<_CompletionToken, void(error_code, size_t)>
            __init{__token};

	  this->get_executor().context().async_wait(this->native_handle(),
	      (int) socket_base::wait_read,
	      [__h = std::move(__init.completion_handler),
               &__buffers, __flags = static_cast<int>(__flags),
               __fd = this->native_handle()]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec);
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  socket_base::__msg_hdr __msg(__buffers);
                  ssize_t __result = ::recvmsg(__fd, &__msg, __flags);
                  if (__result == -1)
                    {
                      __ec.assign(errno, generic_category());
                      __result = 0;
                    }
                  else
                    __ec.clear();
		  __h(__ec, __result);
#else
		  __h(std::make_error_code(errc::operation_not_supported), 0);
#endif
	      });
	  return __init.result.get();
	}

      template<typename _MutableBufferSequence>
	size_t
	receive_from(const _MutableBufferSequence& __buffers,
		     endpoint_type& __sender)
	{
	  return receive_from(__buffers, __sender,
			      socket_base::message_flags(),
			      __throw_on_error{
				  "basic_datagram_socket::receive_from"});
	}

      template<typename _MutableBufferSequence>
	size_t
	receive_from(const _MutableBufferSequence& __buffers,
		     endpoint_type& __sender, error_code& __ec)
	{
	  return receive_from(__buffers, __sender,
			      socket_base::message_flags(), __ec);
	}

      template<typename _MutableBufferSequence>
	size_t
	receive_from(const _MutableBufferSequence& __buffers,
		     endpoint_type& __sender,
		     socket_base::message_flags __flags)
	{
	  return receive_from(__buffers, __sender, __flags,
			      __throw_on_error{
				  "basic_datagram_socket::receive_from"});
	}

      template<typename _MutableBufferSequence>
	size_t
	receive_from(const _MutableBufferSequence& __buffers,
		     endpoint_type& __sender,
		     socket_base::message_flags __flags,
		     error_code& __ec)
	{
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  socket_base::__msg_hdr __msg(__buffers, __sender);
	  ssize_t __result = ::recvmsg(this->native_handle(), &__msg,
				       static_cast<int>(__flags));
	  if (__result == -1)
            {
              __ec.assign(errno, generic_category());
              return 0;
            }
          __ec.clear();
          __sender.resize(__msg.msg_namelen);
          return __result;
#else
	  __ec = std::make_error_code(errc::operation_not_supported);
	  return 0;
#endif
	}

      template<typename _MutableBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_receive_from(const _MutableBufferSequence& __buffers,
			   endpoint_type& __sender,
			   _CompletionToken&& __token)
	{
	  return async_receive_from(__buffers, __sender,
				    socket_base::message_flags(),
				    std::forward<_CompletionToken>(__token));
	}

      template<typename _MutableBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_receive_from(const _MutableBufferSequence& __buffers,
			   endpoint_type& __sender,
			   socket_base::message_flags __flags,
			   _CompletionToken&& __token)
	{
	  async_completion<_CompletionToken, void(error_code, size_t)>
            __init{__token};

	  this->get_executor().context().async_wait( this->native_handle(),
	      (int) socket_base::wait_read,
	      [__h = std::move(__init.completion_handler),
               &__buffers, __flags = static_cast<int>(__flags),
               __sender = std::move(__sender),
               __fd = this->native_handle()]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec);
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  socket_base::__msg_hdr __msg(__buffers, __sender);
                  ssize_t __result = ::recvmsg(__fd, &__msg, __flags);
                  if (__result == -1)
                    {
                      __ec.assign(errno, generic_category());
                      __result = 0;
                    }
                  else
                    {
                      __ec.clear();
                      __sender.resize(__msg.msg_namelen);
                    }
		  __h(__ec, __result);
#else
		  __h(std::make_error_code(errc::operation_not_supported), 0);
#endif
	      });
	  return __init.result.get();
	}

      template<typename _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers)
	{
	  return send(__buffers, socket_base::message_flags(),
		      __throw_on_error{"basic_datagram_socket::send"});
	}

      template<typename _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers, error_code& __ec)
	{ return send(__buffers, socket_base::message_flags(), __ec); }

      template<typename _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers,
	     socket_base::message_flags __flags)
	{
	  return send(__buffers, __flags,
		      __throw_on_error{"basic_datagram_socket::send"});
	}

      template<typename _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers,
	     socket_base::message_flags __flags, error_code& __ec)
	{
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  socket_base::__msg_hdr __msg(__buffers);
	  ssize_t __result = ::sendmsg(this->native_handle(), &__msg,
				       static_cast<int>(__flags));
	  if (__result == -1)
            {
              __ec.assign(errno, generic_category());
              return 0;
            }
          __ec.clear();
          return __result;
#else
	  __ec = std::make_error_code(errc::operation_not_supported);
	  return 0;
#endif
	}

      template<typename _ConstBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_send(const _ConstBufferSequence& __buffers,
			_CompletionToken&& __token)
	{
	  return async_send(__buffers, socket_base::message_flags(),
			    std::forward<_CompletionToken>(__token));
	}

      template<typename _ConstBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_send(const _ConstBufferSequence& __buffers,
		   socket_base::message_flags __flags,
		   _CompletionToken&& __token)
	{
	  async_completion<_CompletionToken, void(error_code, size_t)>
            __init{__token};

	  this->get_executor().context().async_wait( this->native_handle(),
	      (int) socket_base::wait_write,
	      [__h = std::move(__init.completion_handler),
               &__buffers, __flags = static_cast<int>(__flags),
               __fd = this->native_handle()]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec);
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  socket_base::__msg_hdr __msg(__buffers);
                  ssize_t __result = ::sendmsg(__fd, &__msg, __flags);
                  if (__result == -1)
                    {
                      __ec.assign(errno, generic_category());
                      __result = 0;
                    }
                  else
                    __ec.clear();
		  __h(__ec, __result);
#else
		  __h(std::make_error_code(errc::operation_not_supported), 0);
#endif
	      });
	  return __init.result.get();
	}

      template<typename _ConstBufferSequence>
	size_t
	send_to(const _ConstBufferSequence& __buffers,
	        const endpoint_type& __recipient)
	{
	  return send_to(__buffers, __recipient,
			 socket_base::message_flags(),
			 __throw_on_error{"basic_datagram_socket::send_to"});
	}

      template<typename _ConstBufferSequence>
	size_t
	send_to(const _ConstBufferSequence& __buffers,
		const endpoint_type& __recipient, error_code& __ec)
	{
	  return send_to(__buffers, __recipient,
			 socket_base::message_flags(), __ec);
	}

      template<typename _ConstBufferSequence>
	size_t
	send_to(const _ConstBufferSequence& __buffers,
		const endpoint_type& __recipient,
		socket_base::message_flags __flags)
	{
	  return send_to(__buffers, __recipient, __flags,
			 __throw_on_error{"basic_datagram_socket::send_to"});
	}

      template<typename _ConstBufferSequence>
	size_t
	send_to(const _ConstBufferSequence& __buffers,
	        const endpoint_type& __recipient,
		socket_base::message_flags __flags, error_code& __ec)
	{
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  socket_base::__msg_hdr __msg(__buffers, __recipient);
	  ssize_t __result = ::sendmsg(this->native_handle(), &__msg,
				       static_cast<int>(__flags));
	  if (__result == -1)
            {
              __ec.assign(errno, generic_category());
              return 0;
            }
          __ec.clear();
          __recipient.resize(__msg.msg_namelen);
          return __result;
#else
	  __ec = std::make_error_code(errc::operation_not_supported);
	  return 0;
#endif
	}

      template<typename _ConstBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_send_to(const _ConstBufferSequence& __buffers,
		      const endpoint_type& __recipient,
		      _CompletionToken&& __token)
	{
	  return async_send_to(__buffers, __recipient,
			       socket_base::message_flags(),
			       std::forward<_CompletionToken>(__token));
	}

      template<typename _ConstBufferSequence, typename _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_send_to(const _ConstBufferSequence& __buffers,
		      const endpoint_type& __recipient,
		      socket_base::message_flags __flags,
		      _CompletionToken&& __token)
	{
	  async_completion<_CompletionToken, void(error_code, size_t)>
            __init{__token};

	  this->get_executor().context().async_wait( this->native_handle(),
	      (int) socket_base::wait_write,
	      [__h = std::move(__init.completion_handler),
               &__buffers, __flags = static_cast<int>(__flags),
               __recipient = std::move(__recipient),
               __fd = this->native_handle()]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec);
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  socket_base::__msg_hdr __msg(__buffers, __recipient);
                  ssize_t __result = ::sendmsg(__fd, &__msg, __flags);
                  if (__result == -1)
                    {
                      __ec.assign(errno, generic_category());
                      __result = 0;
                    }
                  else
                    {
                      __ec.clear();
                      __recipient.resize(__msg.msg_namelen);
                    }
		  __h(__ec, __result);
#else
		  __h(std::make_error_code(errc::operation_not_supported), 0);
#endif
	      });
	  return __init.result.get();
	}
    };

  template<typename _Protocol>
    class basic_stream_socket : public basic_socket<_Protocol>
    {
      using __base = basic_socket<_Protocol>;

    public:
      // types:

      using native_handle_type = int;
      using protocol_type = _Protocol;
      using endpoint_type = typename protocol_type::endpoint;

      // construct / copy / destroy:

      explicit
      basic_stream_socket(io_context& __ctx) : __base(__ctx) { }

      basic_stream_socket(io_context& __ctx, const protocol_type& __protocol)
      : __base(__ctx, __protocol) { }

      basic_stream_socket(io_context& __ctx, const endpoint_type& __endpoint)
      : __base(__ctx, __endpoint) { }

      basic_stream_socket(io_context& __ctx, const protocol_type& __protocol,
			  const native_handle_type& __native_socket)
      : __base(__ctx, __protocol, __native_socket) { }

      basic_stream_socket(const basic_stream_socket&) = delete;

      basic_stream_socket(basic_stream_socket&& __rhs) = default;

      template<typename _OtherProtocol, typename _Requires
	       = _Require<is_convertible<_OtherProtocol, _Protocol>>>
	basic_stream_socket(basic_stream_socket<_OtherProtocol>&& __rhs)
	: __base(std::move(__rhs)) { }

      ~basic_stream_socket() = default;

      basic_stream_socket& operator=(const basic_stream_socket&) = delete;

      basic_stream_socket& operator=(basic_stream_socket&& __rhs) = default;

      template<class _OtherProtocol>
	enable_if_t<is_convertible<_OtherProtocol, _Protocol>::value,
		    basic_stream_socket&>
	operator=(basic_stream_socket<_OtherProtocol>&& __rhs)
	{
	  __base::operator=(std::move(__rhs));
	  return *this;
	}

      // basic_stream_socket operations:

      template<class _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers)
	{
	  return receive(__buffers, socket_base::message_flags(),
			 __throw_on_error{"basic_stream_socket::receive"});
	}

      template<class _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers, error_code& __ec)
        { return receive(__buffers, socket_base::message_flags(), __ec); }

      template<class _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers,
		socket_base::message_flags __flags)
	{
	  return receive(__buffers, __flags,
			 __throw_on_error{"basic_stream_socket::receive"});
	}

      template<class _MutableBufferSequence>
	size_t
	receive(const _MutableBufferSequence& __buffers,
		socket_base::message_flags __flags, error_code& __ec)
	{
	  if (__buffer_empty(__buffers))
	    {
	      __ec.clear();
	      return 0;
	    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  socket_base::__msg_hdr __msg(__buffers);
	  ssize_t __result = ::recvmsg(this->native_handle(), &__msg,
				       static_cast<int>(__flags));
	  if (__result >= 0)
	    {
	      __ec.clear();
	      return __result;
	    }
	  __ec.assign(errno, generic_category());
#else
	  __ec = std::make_error_code(errc::operation_not_supported);
#endif
	  return 0;
	}

      template<class _MutableBufferSequence, class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_receive(const _MutableBufferSequence& __buffers,
		      _CompletionToken&& __token)
	{
	  return async_receive(__buffers, socket_base::message_flags(),
			       std::forward<_CompletionToken>(__token));
	}

      template<class _MutableBufferSequence, class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_receive(const _MutableBufferSequence& __buffers,
		      socket_base::message_flags __flags,
		      _CompletionToken&& __token)
	{
	  async_completion<_CompletionToken, void(error_code, size_t)>
            __init{__token};

          if (__buffer_empty(__buffers))
	    {
              auto __ex = net::get_associated_executor(
                  __init.completion_handler, this->get_executor());
              auto __a = get_associated_allocator(
                  __init.completion_handler, std::allocator<void>());
              __ex.post(
                  [__h=std::move(__init.completion_handler)] () mutable
                  { __h(error_code{}, 0); }, __a);
              return __init.result.get();
	    }

          this->get_executor().context().async_wait(this->native_handle(),
	      (int) socket_base::wait_read,
	      [__h = std::move(__init.completion_handler),
               &__buffers, __flags = static_cast<int>(__flags),
               __fd = this->native_handle()]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec);
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  socket_base::__msg_hdr __msg(__buffers);
                  ssize_t __result = ::recvmsg(__fd, &__msg, __flags);
                  if (__result == -1)
                    {
                      __ec.assign(errno, generic_category());
                      __result = 0;
                    }
                  else
                    __ec.clear();
		  __h(__ec, __result);
#else
		  __h(std::make_error_code(errc::operation_not_supported), 0);
#endif
	      });
	  return __init.result.get();
	}

      template<class _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers)
	{
	  return send(__buffers, socket_base::message_flags(),
		      __throw_on_error{"basic_stream_socket::send"});
	}

      template<class _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers, error_code& __ec)
	{ return send(__buffers, socket_base::message_flags(), __ec); }

      template<class _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers,
	     socket_base::message_flags __flags)
	{
	  return send(__buffers, socket_base::message_flags(),
		      __throw_on_error{"basic_stream_socket::send"});
	}

      template<class _ConstBufferSequence>
	size_t
	send(const _ConstBufferSequence& __buffers,
	     socket_base::message_flags __flags, error_code& __ec)
	{
	  if (__buffer_empty(__buffers))
	    {
	      __ec.clear();
	      return 0;
	    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	  socket_base::__msg_hdr __msg(__buffers);
	  ssize_t __result = ::sendmsg(this->native_handle(), &__msg,
				       static_cast<int>(__flags));
	  if (__result >= 0)
	    {
	      __ec.clear();
	      return __result;
	    }
	  __ec.assign(errno, generic_category());
#else
	  __ec = std::make_error_code(errc::operation_not_supported);
#endif
	  return 0;
	}

      template<class _ConstBufferSequence, class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_send(const _ConstBufferSequence& __buffers,
		   _CompletionToken&& __token)
	{
	  return async_send(__buffers, socket_base::message_flags(),
			    std::forward<_CompletionToken>(__token));
	}

      template<class _ConstBufferSequence, class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_send(const _ConstBufferSequence& __buffers,
		   socket_base::message_flags __flags,
		   _CompletionToken&& __token)
	{
	  async_completion<_CompletionToken, void(error_code, size_t)>
            __init{__token};

          if (__buffer_empty(__buffers))
	    {
              auto __ex = net::get_associated_executor(
                  __init.completion_handler, this->get_executor());
              auto __a = get_associated_allocator(
                  __init.completion_handler, std::allocator<void>());
              __ex.post(
                  [__h=std::move(__init.completion_handler)] () mutable
                  { __h(error_code{}, 0); }, __a);
              return __init.result.get();
	    }

          this->get_executor().context().async_wait(this->native_handle(),
	      (int) socket_base::wait_write,
	      [__h = std::move(__init.completion_handler),
               &__buffers, __flags = static_cast<int>(__flags),
               __fd = this->native_handle()]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec);
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  socket_base::__msg_hdr __msg(__buffers);
                  ssize_t __result = ::sendmsg(__fd, &__msg, __flags);
                  if (__result == -1)
                    {
                      __ec.assign(errno, generic_category());
                      __result = 0;
                    }
                  else
                    __ec.clear();
		  __h(__ec, __result);
#else
		  __h(std::make_error_code(errc::operation_not_supported), 0);
#endif
	      });
	  return __init.result.get();
	}

      template<class _MutableBufferSequence>
	size_t
	read_some(const _MutableBufferSequence& __buffers)
	{
	  return receive(__buffers,
			 __throw_on_error{"basic_stream_socket::read_some"});
	}

      template<class _MutableBufferSequence>
	size_t
	read_some(const _MutableBufferSequence& __buffers, error_code& __ec)
	{ return receive(__buffers, __ec); }

      template<class _MutableBufferSequence, class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_read_some(const _MutableBufferSequence& __buffers,
			_CompletionToken&& __token)
	{
	  return async_receive(__buffers,
			       std::forward<_CompletionToken>(__token));
	}

      template<class _ConstBufferSequence>
	size_t
	write_some(const _ConstBufferSequence& __buffers)
	{
	  return send(__buffers,
		      __throw_on_error{"basic_stream_socket:write_some"});
	}

      template<class _ConstBufferSequence>
	size_t
	write_some(const _ConstBufferSequence& __buffers, error_code& __ec)
	{  return send(__buffers, __ec); }

      template<class _ConstBufferSequence, class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, size_t)>
	async_write_some(const _ConstBufferSequence& __buffers,
			      _CompletionToken&& __token)
	{
	  return async_send(__buffers,
			    std::forward<_CompletionToken>(__token));
	}
    };

  template<typename _AcceptableProtocol>
    class basic_socket_acceptor
    : public socket_base, private __basic_socket_impl<_AcceptableProtocol>
    {
      using __base = __basic_socket_impl<_AcceptableProtocol>;

    public:
      // types:

      using executor_type = io_context::executor_type;
      using native_handle_type = int;
      using protocol_type = _AcceptableProtocol;
      using endpoint_type = typename protocol_type::endpoint;
      using socket_type = typename protocol_type::socket;

      static_assert(__detail::__acceptable_protocol<protocol_type>,
		    "protocol_type meets the AcceptableProtocol requirements");

      // construct / copy / destroy:

      explicit
      basic_socket_acceptor(io_context& __ctx)
      : __base(__ctx), _M_protocol(endpoint_type{}.protocol()) { }

      basic_socket_acceptor(io_context& __ctx,
			    const protocol_type& __protocol)
      : __base(__ctx), _M_protocol(__protocol)
      { open(__protocol); }

      basic_socket_acceptor(io_context& __ctx, const endpoint_type& __endpoint,
			    [[__maybe_unused__]] bool __reuse_addr = true)
      : basic_socket_acceptor(__ctx, __endpoint.protocol())
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	if (__reuse_addr)
	  set_option(reuse_address(true));
#endif
	bind(__endpoint);
	listen();
      }

      basic_socket_acceptor(io_context& __ctx, const protocol_type& __protocol,
			    const native_handle_type& __native_acceptor)
      : basic_socket_acceptor(__ctx, __protocol)
      { assign(__protocol, __native_acceptor); }

      basic_socket_acceptor(const basic_socket_acceptor&) = delete;

      basic_socket_acceptor(basic_socket_acceptor&&) = default;

      template<typename _OtherProtocol, typename _Requires
	       = _Require<is_convertible<_OtherProtocol, protocol_type>>>
	basic_socket_acceptor(basic_socket_acceptor<_OtherProtocol>&& __rhs)
	: __base(std::move(__rhs)) { }

      ~basic_socket_acceptor() = default;

      basic_socket_acceptor& operator=(const basic_socket_acceptor&) = delete;

      basic_socket_acceptor& operator=(basic_socket_acceptor&&) = default;

      template<class _OtherProtocol>
	enable_if_t<is_convertible<_OtherProtocol, protocol_type>::value,
		    basic_socket_acceptor&>
	operator=(basic_socket_acceptor<_OtherProtocol>&& __rhs)
	{
	  __base::operator=(std::move(__rhs));
	  return *this;
	}

      // basic_socket_acceptor operations:

      executor_type get_executor() noexcept { return __base::get_executor(); }

      native_handle_type
      native_handle() noexcept { return __base::native_handle(); }

      void
      open(const protocol_type& __protocol = protocol_type())
      { open(__protocol, __throw_on_error{"basic_socket_acceptor::open"}); }

      void
      open(const protocol_type& __protocol, error_code& __ec)
      { __base::open(__protocol, __ec); }

      void
      assign(const protocol_type& __protocol,
	     const native_handle_type& __native_acceptor)
      {
	assign(__protocol, __native_acceptor,
	       __throw_on_error{"basic_socket_acceptor::assign"});
      }

      void
      assign(const protocol_type& __protocol,
	     const native_handle_type& __native_acceptor,
	     error_code& __ec)
      { __base::assign(__protocol, __native_acceptor, __ec); }

      native_handle_type release()
      { return release(__throw_on_error{"basic_socket_acceptor::release"}); }

      native_handle_type release(error_code& __ec)
      { return __base::release(__ec); }

      _GLIBCXX_NODISCARD bool
      is_open() const noexcept { return __base::is_open(); }

      void
      close() { close(__throw_on_error{"basic_socket_acceptor::close"}); }

      void
      close(error_code& __ec) { __base::_close(__ec); }

      void
      cancel() { cancel(__throw_on_error{"basic_socket_acceptor::cancel"}); }

      void
      cancel(error_code& __ec) { __base::cancel(__ec); }

      template<typename _SettableSocketOption>
	void
	set_option(const _SettableSocketOption& __option)
	{
	  set_option(__option,
		     __throw_on_error{"basic_socket_acceptor::set_option"});
	}

      template<typename _SettableSocketOption>
	void
	set_option(const _SettableSocketOption& __option, error_code& __ec)
	{ __base::set_option(__option, __ec); }

      template<typename _GettableSocketOption>
	void
	get_option(_GettableSocketOption& __option) const
	{
	  get_option(__option,
		     __throw_on_error{"basic_socket_acceptor::get_option"});
	}

      template<typename _GettableSocketOption>
	void
	get_option(_GettableSocketOption& __option, error_code& __ec) const
	{ __base::get_option(__option, __ec); }

      template<typename _IoControlCommand>
	void
	io_control(_IoControlCommand& __command)
	{
	  io_control(__command,
		     __throw_on_error{"basic_socket_acceptor::io_control"});
	}

      template<typename _IoControlCommand>
	void
	io_control(_IoControlCommand& __command, error_code& __ec)
	{ __base::io_control(__command, __ec); }

      void
      non_blocking(bool __mode)
      {
	non_blocking(__mode,
		     __throw_on_error{"basic_socket_acceptor::non_blocking"});
      }

      void
      non_blocking(bool __mode, error_code& __ec)
      { __base::non_blocking(__mode, __ec); }

      bool non_blocking() const { return __base::non_blocking(); }

      void
      native_non_blocking(bool __mode)
      {
	native_non_blocking(__mode, __throw_on_error{
	    "basic_socket_acceptor::native_non_blocking"});
      }

      void
      native_non_blocking(bool __mode, error_code& __ec)
      { __base::native_non_blocking(__mode, __ec); }

      bool
      native_non_blocking() const
      { return __base::native_non_blocking(); }

      void
      bind(const endpoint_type& __endpoint)
      {
	return bind(__endpoint,
		    __throw_on_error{"basic_socket_acceptor::bind"});
      }

      void
      bind(const endpoint_type& __endpoint, error_code& __ec)
      { __base::bind(__endpoint, __ec); }

      void
      listen(int __backlog = max_listen_connections)
      {
	return listen(__backlog,
		      __throw_on_error{"basic_socket_acceptor::listen"});
      }

      void
      listen(int __backlog, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	if (::listen(native_handle(), __backlog) == -1)
	  __ec.assign(errno, generic_category());
	else
	  __ec.clear();
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
      }

      endpoint_type
      local_endpoint() const
      {
	return local_endpoint(
	    __throw_on_error{"basic_socket_acceptor::local_endpoint"});
      }

      endpoint_type
      local_endpoint(error_code& __ec) const
      { return __base::local_endpoint(__ec); }

      void
      enable_connection_aborted(bool __mode)
      { __base::_M_bits.enable_connection_aborted = __mode; }

      bool
      enable_connection_aborted() const
      { return __base::_M_bits.enable_connection_aborted; }

      socket_type
      accept()
      { return accept(__throw_on_error{"basic_socket_acceptor::accept"}); }

      socket_type
      accept(error_code& __ec)
      { return accept(get_executor().context(), __ec); }

      socket_type accept(io_context& __ctx)
      {
	return accept(__ctx,
		      __throw_on_error{"basic_socket_acceptor::accept"});
      }

      socket_type
      accept(io_context& __ctx, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	do
	  {
	    int __h = ::accept(native_handle(), nullptr, 0);
	    if (__h != -1)
	      {
		__ec.clear();
		return socket_type{__ctx, _M_protocol, __h};
	      }
	  } while (errno == ECONNABORTED && enable_connection_aborted());
	__ec.assign(errno, generic_category());
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
	return socket_type{__ctx};
      }

      template<class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, socket_type)>
	async_accept(_CompletionToken&& __token)
	{
	  return async_accept(get_executor().context(),
			      std::forward<_CompletionToken>(__token));
	}

      template<class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, socket_type)>
	async_accept(io_context& __ctx, _CompletionToken&& __token)
	{
          async_completion<_CompletionToken, void(error_code, socket_type)>
            __init{__token};

	  __ctx.async_wait(native_handle(),
	      (int) socket_base::wait_read,
	      [__h = std::move(__init.completion_handler),
               __connabort = enable_connection_aborted(),
               __fd = native_handle(),
               __protocol = _M_protocol,
               &__ctx
              ]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec, socket_type(__ctx));
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  do
                    {
                      int __newfd = ::accept(__fd, nullptr, 0);
                      if (__newfd != -1)
                        {
                          __ec.clear();
                          __h(__ec, socket_type{__ctx, __protocol, __newfd});
                          return;
                        }
                    } while (errno == ECONNABORTED && __connabort);
                  __ec.assign(errno, generic_category());
                  __h(__ec, socket_type(__ctx));
#else
		  __h(std::make_error_code(errc::operation_not_supported), 0);
#endif
	      });
	  return __init.result.get();
	}

      socket_type
      accept(endpoint_type& __endpoint)
      {
	return accept(get_executor().context(), __endpoint,
		      __throw_on_error{"basic_socket_acceptor::accept"});
      }

      socket_type
      accept(endpoint_type& __endpoint, error_code& __ec)
      { return accept(get_executor().context(), __endpoint, __ec); }

      socket_type
      accept(io_context& __ctx, endpoint_type& __endpoint)
      {
	return accept(__ctx, __endpoint,
		      __throw_on_error{"basic_socket_acceptor::accept"});
      }

      socket_type
      accept(io_context& __ctx, endpoint_type& __endpoint, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
	do
	  {
	    socklen_t __len = __endpoint.capacity();
	    int __h = ::accept(native_handle(), (sockaddr*)__endpoint.data(),
			       &__len);
	    if (__h != -1)
	      {
		__endpoint.resize(__len);
		return socket_type{__ctx, _M_protocol, __h};
	      }
	  } while (errno == ECONNABORTED && enable_connection_aborted());
	__ec.assign(errno, generic_category());
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
	return socket_type{__ctx};
      }

      template<class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, socket_type)>
	async_accept(endpoint_type& __endpoint,
			     _CompletionToken&& __token)
	{
	  return async_accept(get_executor().context(), __endpoint,
			      std::forward<_CompletionToken>(__token));
	}

      template<class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code, socket_type)>
	async_accept(io_context& __ctx, endpoint_type& __endpoint,
			     _CompletionToken&& __token)
        {
          async_completion<_CompletionToken, void(error_code, socket_type)>
            __init{__token};

	  __ctx.async_wait(native_handle(),
	      (int) socket_base::wait_read,
	      [__h = std::move(__init.completion_handler),
              __ep = std::move(__endpoint),
               __connabort = enable_connection_aborted(),
               __fd = native_handle(),
               &__ctx
              ]
              (error_code __ec) mutable {
                  if (__ec)
                    {
                      __h(__ec, socket_type(__ctx));
                      return;
                    }
#ifdef _GLIBCXX_HAVE_SYS_SOCKET_H
                  do
                    {
                      socklen_t __len = __ep.capacity();
                      int __newfd = ::accept(__fd, __ep.data, &__len);
                      if (__newfd != -1)
                        {
                          __ep.resize(__len);
                          auto __protocol = __ep.protocol();
                          __ec.clear();
                          __h(__ec, socket_type{__ctx, __protocol, __newfd});
                          return;
                        }
                    } while (errno == ECONNABORTED && __connabort);
                  __ec.assign(errno, generic_category());
#else
		  __ec = std::make_error_code(errc::operation_not_supported);
#endif
                  __h(__ec, socket_type(__ctx));
	      });
	  return __init.result.get();
        }

      void
      wait(wait_type __w)
      { wait(__w, __throw_on_error{"basic_socket_acceptor::wait"}); }

      void
      wait(wait_type __w, error_code& __ec)
      {
#ifdef _GLIBCXX_HAVE_POLL_H
	::pollfd __fds;
	__fds.fd = native_handle();
	__fds.events = __w; // __w | POLLIN;
	if (::poll(&__fds, 1, -1) == -1)
	  __ec.assign(errno, generic_category());
	else
	  __ec.clear();
#else
	__ec = std::make_error_code(errc::operation_not_supported);
#endif
      }

      template<class _CompletionToken>
	__deduced_t<_CompletionToken, void(error_code)>
	async_wait(wait_type __w, _CompletionToken&& __token)
        {
	  async_completion<_CompletionToken, void(error_code)> __init{__token};
	  get_executor().context().async_wait( native_handle(),
	      static_cast<int>(__w),
	      [__h = std::move(__init.completion_handler)]
              (error_code __ec) mutable {
		  __h(__ec);
	      });
	  return __init.result.get();
	}

    private:
      protocol_type _M_protocol;
    };

  /// @}

  /** @brief Socket streams
   * @{
   */

  template<typename _Protocol, typename _Clock, typename _WaitTraits>
    class basic_socket_streambuf : public basic_streambuf<char>
    {
    public:
      // types:

      using protocol_type = _Protocol;
      using endpoint_type = typename protocol_type::endpoint;
      using clock_type = _Clock;
      using time_point = typename clock_type::time_point;
      using duration = typename clock_type::duration;
      using wait_traits_type = _WaitTraits;

      // construct / copy / destroy:

      basic_socket_streambuf() : _M_socket(_S_ctx()) { }

      explicit
      basic_socket_streambuf(basic_stream_socket<protocol_type> __s)
      : _M_socket(std::move(__s)) { }

      basic_socket_streambuf(const basic_socket_streambuf&) = delete;

      basic_socket_streambuf(basic_socket_streambuf&& __rhs); // TODO


      virtual ~basic_socket_streambuf(); // TODO

      basic_socket_streambuf& operator=(const basic_socket_streambuf&) = delete;

      basic_socket_streambuf& operator=(basic_socket_streambuf&& __rhs); // TODO

      // members:

      basic_socket_streambuf* connect(const endpoint_type& __e); // TODO

      template<typename... _Args>
	basic_socket_streambuf* connect(_Args&&... ); // TODO

      basic_socket_streambuf* close(); // TODO

      basic_socket<protocol_type>& socket() { return _M_socket; }

      error_code error() const noexcept { return _M_ec; }

      time_point expiry() const { return _M_expiry; }

      void
      expires_at(const time_point& __t)
      { _M_expiry = __t; }

      void
      expires_after(const duration& __d)
      { expires_at(clock_type::now() + __d); }

    protected:
      // overridden virtual functions: // TODO
      virtual int_type underflow() override;
      virtual int_type pbackfail(int_type __c = traits_type::eof()) override;
      virtual int_type overflow(int_type __c = traits_type::eof()) override;
      virtual int sync() override;
      virtual streambuf* setbuf(char_type* __s, streamsize __n) override;

    private:
      static io_context&
      _S_ctx()
      {
	static io_context __ctx;
	return __ctx;
      }

      basic_stream_socket<protocol_type> _M_socket;
      error_code _M_ec;
      time_point _M_expiry{ time_point::max() };
    };

  template<typename _Protocol, class _Clock, typename _WaitTraits>
    class basic_socket_iostream : public basic_iostream<char>
    {
      using __streambuf_type
	= basic_socket_streambuf<_Protocol, _Clock, _WaitTraits>;

    public:
      // types:

      using protocol_type = _Protocol;
      using endpoint_type = typename protocol_type::endpoint;
      using clock_type = _Clock;
      using time_point = typename clock_type::time_point;
      using duration = typename clock_type::duration;
      using wait_traits_type = _WaitTraits;

      // construct / copy / destroy:

      // TODO base-from-member ?
      basic_socket_iostream() : basic_iostream(nullptr), _M_sb()
      {
	this->init(std::addressof(_M_sb));
	this->setf(std::ios::unitbuf);
      }

      explicit
      basic_socket_iostream(basic_stream_socket<protocol_type> __s)
      : basic_iostream(nullptr), _M_sb(std::move(__s))
      {
	this->init(std::addressof(_M_sb));
	this->setf(std::ios::unitbuf);
      }

      basic_socket_iostream(const basic_socket_iostream&) = delete;

      basic_socket_iostream(basic_socket_iostream&& __rhs)
      : basic_iostream(nullptr), _M_sb(std::move(__rhs._M_sb))
	// XXX ???     ^^^^^^^
      {
	// XXX ??? this->init(std::addressof(_M_sb));
	this->set_rbduf(std::addressof(_M_sb));
      }

      template<typename... _Args>
	explicit
	basic_socket_iostream(_Args&&... __args)
	: basic_iostream(nullptr), _M_sb()
	{
	  this->init(std::addressof(_M_sb));
	  this->setf(std::ios::unitbuf);
	  connect(forward<_Args>(__args)...);
	}

      basic_socket_iostream& operator=(const basic_socket_iostream&) = delete;

      basic_socket_iostream& operator=(basic_socket_iostream&& __rhs); // TODO

      // members:

      template<typename... _Args>
	void
	connect(_Args&&... __args)
	{
	  if (rdbuf()->connect(forward<_Args>(__args)...) == nullptr)
	    this->setstate(failbit);
	}

      void
      close()
      {
	if (rdbuf()->close() == nullptr)
	  this->setstate(failbit);
      }

      basic_socket_streambuf<protocol_type, clock_type, wait_traits_type>*
      rdbuf() const
      { return const_cast<__streambuf_type*>(std::addressof(_M_sb)); }

      basic_socket<protocol_type>& socket() { return rdbuf()->socket(); }
      error_code error() const noexcept { return rdbuf()->error(); }

      time_point expiry() const { return rdbuf()->expiry(); }
      void expires_at(const time_point& __t) { rdbuf()->expires_at(__t); }
      void expires_after(const duration& __d) { rdbuf()->expires_after(__d); }

    private:
      __streambuf_type _M_sb;
    };

  /// @}

  /** @brief synchronous connect operations
   * @{
   */

  template<typename _Protocol, typename _EndpointSequence,
	   typename _ConnectCondition>
    inline typename _Protocol::endpoint
    connect(basic_socket<_Protocol>& __s,
	    const _EndpointSequence& __endpoints,
	    _ConnectCondition __c, error_code& __ec)
    {
      __ec.clear();
      bool __found = false;
      for (auto& __ep : __endpoints)
	{
	  if (__c(__ec, __ep))
	    {
	      __found = true;
	      __s.close(__ec);
	      if (!__ec)
		__s.open(__ep.protocol(), __ec);
	      if (!__ec)
		__s.connect(__ep, __ec);
	      if (!__ec)
		return __ep;
	    }
	}
      if (!__found)
	__ec = socket_errc::not_found;
      return typename _Protocol::endpoint{};
    }

  template<typename _Protocol, typename _InputIterator,
	   typename _ConnectCondition>
    inline _InputIterator
    connect(basic_socket<_Protocol>& __s,
	    _InputIterator __first, _InputIterator __last,
	    _ConnectCondition __c, error_code& __ec)
    {
      __ec.clear();
      bool __found = false;
      for (auto __i = __first; __i != __last; ++__i)
	{
	  if (__c(__ec, *__i))
	    {
	      __found = true;
	      __s.close(__ec);
	      if (!__ec)
		__s.open(typename _Protocol::endpoint(*__i).protocol(), __ec);
	      if (!__ec)
		__s.connect(*__i, __ec);
	      if (!__ec)
		return __i;
	    }
	}
      if (!__found)
	__ec = socket_errc::not_found;
      return __last;
    }

  template<typename _Protocol, typename _EndpointSequence,
	   typename _ConnectCondition>
    inline typename _Protocol::endpoint
    connect(basic_socket<_Protocol>& __s,
	    const _EndpointSequence& __endpoints,
	    _ConnectCondition __c)
    {
      return net::connect(__s, __endpoints, __c, __throw_on_error{"connect"});
    }

  template<typename _Protocol, typename _InputIterator,
	   typename _ConnectCondition>
    inline _InputIterator
    connect(basic_socket<_Protocol>& __s,
	    _InputIterator __first, _InputIterator __last,
	    _ConnectCondition __c)
    {
      return net::connect(__s, __first, __last, __c,
			  __throw_on_error{"connect"});
    }

  template<typename _Protocol, typename _EndpointSequence>
    inline typename _Protocol::endpoint
    connect(basic_socket<_Protocol>& __s,
	    const _EndpointSequence& __endpoints)
    {
      return net::connect(__s, __endpoints, [](auto, auto){ return true; },
			  __throw_on_error{"connect"});
    }

  template<typename _Protocol, typename _EndpointSequence>
    inline typename _Protocol::endpoint
    connect(basic_socket<_Protocol>& __s,
	    const _EndpointSequence& __endpoints,
	    error_code& __ec)
    {
      return net::connect(__s, __endpoints, [](auto, auto){ return true; },
			  __ec);
    }

  template<typename _Protocol, typename _InputIterator>
    inline _InputIterator
    connect(basic_socket<_Protocol>& __s,
	    _InputIterator __first, _InputIterator __last)
    {
      return net::connect(__s, __first, __last, [](auto, auto){ return true; },
			  __throw_on_error{"connect"});
    }

  template<typename _Protocol, typename _InputIterator>
    inline _InputIterator
    connect(basic_socket<_Protocol>& __s,
	    _InputIterator __first, _InputIterator __last,
	    error_code& __ec)
    {
      return net::connect(__s, __first, __last, [](auto, auto){ return true; },
			  __ec);
    }

  /// @}

  /** @brief asynchronous connect operations
   * @{
   */

  template<typename _Protocol, typename _EndpointSequence,
	   typename _ConnectCondition, typename _CompletionToken>
    inline
    __deduced_t<_CompletionToken,
		void(error_code, typename _Protocol::endpoint)>
    async_connect(basic_socket<_Protocol>& __s,
		  const _EndpointSequence& __endpoints,
		  _ConnectCondition __c, _CompletionToken&& __token); // TODO

  template<typename _Protocol, typename _EndpointSequence,
	   typename _CompletionToken>
    inline
    __deduced_t<_CompletionToken,
		void(error_code, typename _Protocol::endpoint)>
    async_connect(basic_socket<_Protocol>& __s,
		  const _EndpointSequence& __endpoints,
		  _CompletionToken&& __token)
    {
      return net::async_connect(__s, __endpoints,
				[](auto, auto){ return true; },
				forward<_CompletionToken>(__token));
    }

  template<typename _Protocol, typename _InputIterator,
	   typename _ConnectCondition, typename _CompletionToken>
    inline
    __deduced_t<_CompletionToken, void(error_code, _InputIterator)>
    async_connect(basic_socket<_Protocol>& __s,
		  _InputIterator __first, _InputIterator __last,
		  _ConnectCondition __c, _CompletionToken&& __token); // TODO

  template<typename _Protocol, typename _InputIterator,
	   typename _CompletionToken>
    inline
    __deduced_t<_CompletionToken, void(error_code, _InputIterator)>
    async_connect(basic_socket<_Protocol>& __s,
		  _InputIterator __first, _InputIterator __last,
		  _CompletionToken&& __token)
    {
      return net::async_connect(__s, __first, __last,
				[](auto, auto){ return true; },
				forward<_CompletionToken>(__token));
    }

  /// @}

#endif  // _GLIBCXX_HAVE_UNISTD_H

  /// @}

} // namespace v1
} // namespace net
} // namespace experimental

_GLIBCXX_END_NAMESPACE_VERSION
} // namespace std

#endif // C++14

#endif // _GLIBCXX_EXPERIMENTAL_SOCKET
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         HL         HHPxHD   H    E$@   t      CPt-SlHf1ҁ  W   u  ƃC   H    HT$eH+%(     sHH[]HA\A]A^    HD$eH+%(     H[]A\A]A^    HH h  (  
   RLcHL    C     4   |  A  t
CP  C   Ll$ h     HD$    蠈L   H者L   Hp4   G  D$uHԾ	C  L    HH f f9u#H f f9tpHPX   ;      LcHL    C  O4   u
A  =  C   6tB69v3B  fA  C   sTHH$    $D4$Ll$A   dD$    D$    D$    CQ B |     H   HHT$ȃD$ h     H軆L   H蛃L   H苃D$       H{   HHT$Y   HHT$GD$ DsTH@  5KPrD$CQ     HD$       HHT$ۂT$   H  ԅKP   oHHXHPXHH   H   BHH    H    sTHHT$D$    6HD$
t}7   C     HD$       HLl$LD$ 6sT1HT$HD$ʁT$sTHȄcPƃC     w    @     AUATL+  USWH+  Љр@DG,  L獨h      HHhxH   L    f1=  Uu;+  t1   E1   HCEH@8u3[]A\A]    K   A E5uɺ   A E5HCH@8ẗ́tɋ,  L獨d      HHhxHHD([L]A\A]    HC[]A\H@8A]    L牃+  ,  h    HHhxH   L       ݺ   A E5k,  Pff.     @     AWAVIAUATUS1HL  L  A$AL$1ȩ  3  LLCA  Åt4    I +  HtI  I(+  H@    A      AD$6A  9   I  I  Nf= fA9  XI  f @@fA9  .A  tV6A9  1ʁ  L    AG     A  I
      H[]A\A]A^A_    M+  L    I  Rփ9t*I        LȷI  R%      9   I         A,  T  I+  HǉT$H$    IT$HƉQxI   H<$L$    |$   H<$    A+  A,  QIfA+  %  QxI   H<$    L    IH h  A`,  
   IH h      0   IH   A  
   bLS듐    AW   AVAUI 0  ATUSHXHt$   eH%(   HD$P1HD$(    HD$0    HD$8    HD$@    HD$H        H
  IƺD  
  H=    I   Aǆ	      Ip  Aǆ5      HD$I	  H$    I5  H  L    Å    A      H    L    Aą    L    A   A(6      M	  Ix7  H    H    M	  Iǆ 5           ( 1LH   Iǆ4      I4  I4  I4  I4      I	  H       PhI	    A$5  f1=  	WN  A}l     A	  L
EA	        A$5  f1ҁ  	Wt"=  W9
    Wu'H   "  A}m tA	     HL|$HL      H|$    Å    A	    A$5  =  W	  = W
  H<$I5  hH       HI	  PxI	     H    H      HI	  PxI	  D   H    EyAƆd7  f1A |  |   SS
  E6  H߉T$    T$HI	  PxI	     H@0     IVD    H=         D$$    I6      IHtrH<$   H¾   wI   LuDHʉPH9uHL$$H       L    x|$$uHA<1028L
  L        H    H         HD$AA6  HߍhL    HI	  hxI	     H    HT$   HcE1AAƄ6  bBHAƄ6  cBHcAƄ6   DʉAǿ   ڍ   d      8AD  @     ɉ̉AfuAtAGMcACƄ>6  .~AA6  Hߍ       HI	  hxI	     H       tAƆ25     t}LA	  @M   Dd$I
       tCA6  HD      HI	  D`xI	  D   H    A   tLDd$MIA6  Hߍ      HI	  hxI	     H                A6  HߍL      HI	  hxI	     H    A   AGMcCƄ>6   s)ALl$MDAHG4<    HI	  DpxI	  D   H߃A    A,  H4$AHD4A~MLl$A6  HߍhP    HI	  hxI	     Hf    fA5  A6  HߍhT    HI	  hxI	     H    A5  H<$   Aǆ4     2A	    H    IL5  H   B Id5  AǆX5  P  Aǆ4     HHAǆ(5     H  %  Ip5  A<6  I	  E1H   ŀ      H  HcA$5  Ix5  HIp5  I5  f1I5  =  	W  A5    A5  AƆ5  A6  Hߍh<    HI	  hxI	     H       uA	  AƆ25   A$5  f1=  W  Aǆ(5      D  A5  7  I    HL    Å    A	  I	  Hl   A	  I	  Kpʃ  s      !   fA05  |  A	   @  	IcA0A>6  1   L    A,6      A	        rAƆ5   gA$5  f1ҍ      A5  H<$A$5    Wt$=WtI	  H      @  A	  I	  AƆ25   H   1    A$5    W  f1=  W  H<$3I4  11AƆ5  E1H        H    HH   I4  A6        HI	  hxI	     H         Iǆ`9      L    H=    Iǆ       Aǆ    Iǆ      MH  t	H<$LI5  1   L    A$5  H    H.   (#  I   f1=  	WH    HDI      HI   Hʀ  I	   I   A	  u	I   L    Å        I	  H I	  %CAL5  fAN5  AX5  fAZ5  AP5  fAR5  Ad5  fAf5  AT5  fAV5  A\5  fA^5  A`5  fAb5      1ҾXt  "      HHBH<w       H    A	  6      	WA5  A6  Hߍ      HI	  hxI	     H    A7  Ad7  Hߍ,      HI	  hxI	  D   H    AW A}n A	  ҃߃BfA05  ,A	  HL$$H       L    T$$HL7    хt0A6    AƄ6   HI	  H   A	  1A25   @    A}@<1nA5      ^u   fA05  A}m A	     I	    A5  р*ʁ   u&HAd7   HAB   fE05  xHHI6  HtHt
HuHH)H)1AN N9rd   fA05  2   fA05  HD$PeH+%(      HX[]A\A]A^A_    A	  @   I	  L    LIǆ	          L    I5      Ix5  Ht:I	  Ip5  E1Ic<6  H       Iǆx5      Iǆ5      L    3L    A	  WA6  ыDD
f1    l    DfD
Af.         AWIAVAAUAATUH+  S+  D`,  fD+  EA	XD+  H    HIXxID   Hd       Eubtn
       A,  HDp    HIDpxID   H    DD1fuA   A   t	A   um1[]A\A]A^A_    D%   =   tA,  DH D`       HID`xI   H    E    땸f    AUATAUHSx,  +  蝌y,     t,  Ӄ   كEىكEىكEىك  EىـɀEىL+    LEډڀ 	Eډ؀HED؋,  D`    HHE D`xHE    H+  L    H    H1ɺ      H    1[]A\A]    h,     f	X    fHfdt 1f
61ۀj,  Ã"1ۀj,  Í   
f    ATUSHeH%(   HD$1D$    HG8  H5     H 5  L	  H    5     uV5  HT$L#hT$5  L激kH    1HT$eH+%(      H[]A\    5   @  LjH           H    ƃ5  H5    H4  ǃ4  S   HS    P5  LgHD$    D$aWM    f         AWAVAUATUSHHeH%(   HD$1+  D$    D$      y,     h,  f	  f  E1+  HT$Hf+  Hߋl$HT$f+  Dd$Ef1A=  	W  =  W  D@AAA@Aj,  u
AA AD|$E9u	E  l,     +  DH%i+  DHi   zlx,  +  蘈t,  +  HHT$ 	e+  HHT$eƃ{,   Dd$9l$   l,   i  +  Hh+  DH߀hH5    ǃ0+  S   H8+  ƃ{,  HS    HD$eH+%(     H1[]A\A]A^A_    HD$eH+%(     H@[]A\A]A^A_Dfh,  	rD`gh,  f=	O  f=FHjDσ@6A   HeH9+  DH߃}g+  DH߀igƃl,   H{    +  HDHgH0+   @  HL+  !gL           L    ^mjAD$    +  f1=  	W   +  HT$HcT$u;+  f1=  	WE1     HfH|g+  HA   `f+  f1=  	Wn믺      ;fh        AWAVAUATL 5  UHSHHD5  D5  eH%(   HD$15  D5  LD$    E	t<a  :5     	5  H  HS8;  }   U<       	    D}AA1D$5  HC8D5  E1fD5  D5  uwL    HD$eH+%(     HD[]A\A]A^A_    }HS8dHu@H|$A    }	    T$/Ѓ@Һo   D^5  tu	H	  5  tLQA_  t	  u$D}Au	  5  A UA5  T$%    !@у@E             AU   ATUSHHeH%(   HD$1+  $    Ǉ+      +  H      H+  H      H+  H  +  tHz,  k,  t+     HmHD$eH+%(     H[]A\A]    H   +`$   HH+  `+  $	+  tK+  f1=  W  =  W"  =  	W  z,  k,  _HtRD$      /  Hl$  HC      H߁        rbH   HR_T$   H߀@NbH   H._T$   H߃*bp     HbH   H^L$+  f1  T$=  	Wu  L$ʾ   Ha8aHH߾   ^T$H߾   aHH߾   ~^T$   H߃zaK  +  H     HC      H߁     D$      (aH߾   HT$^T$H߾     @`Hߺ l     `H߾   HT$]T$   H߁    `$Hߺ     `HH߾   ]T$   H߀T$|`      He`
      HS`1     HA`      H/`     H`@     H`     H_      H_+  6H+  D$    =        Hǃ+     _      H_1Ҿ   H|_   HHl$HW\T$   H߃T$O_H   H/\T$   H߃T$'_+  q  +    W  =W  ,  L+  LhX    HHhxH   L    l$,  LDh<    HHDhxHD   L    A@Hߺ      g^H߾   X^1Ҿ   HI^     HH      H+  H      H+  H      H+  D$    ] 8     H]     H]        H]   HHl$HZT$   H߃T$]      Hw]H   HWZT$+    H߾   T$B]HߺP     0]HH߾   ZT$H߾   ]Hߺp     \Hߺ      \     H\?H|+  +  f1=  WXEHH߾   YT$   H߃T$\bHt|Hߺ      ]\HH߾   =YT$H߾   9\1Ҿ   H*\3 \H߾   HT$XT$H߾   [Hߺ l     [H߾   HT$XT$   H߁  ["YH{    f.         USHƇ5   DFE5  DNEt	5  5  NЃtЃ5  HC8u	1[]    H 5  H    5  t5  H	  5  tH    1[]    H             AW   	AVAUATUS+   HG  k,   Dy,  Dt,  uƇy,  Ǉt,  O   H+  H    z,  tk,  H+  c  H    HCDy,  Dt,  H3H0  f  AH3ȉ  HH    k,      
  H     HH
HH=  uH     1H     H    H     H  HH   H3     Cu<
   ŋ+  u+  f1=  	WtWH1[   ]A\A]A^A_[]A\A]A^A_    (  fn,  	
  ~,  L+  LD       HHDhxHD   LE    ,  LA   D       HHDxxHD   L    1ɺ   H,  L獨       HHhxHHD([L]A\A]A^A_    f.         AWAVIAUATAUSI+  /pD   L      tHA,      HIhxKVAHI   H    IH   A+  f1=  	WItR  PhA+  =  W@  
   IHh tMƧ      uH        _      IH    S      I   PhIH   =    D1ɺ   L   |Ņt[]A\A]A^A_    M+  L    LEk,  A+  t	E8k,  uxL    A+  =  WtlAFtI       1 I   1 I     I  []A\A]A^A_           L#{I     H=             IHK  I     H       HIPxID   H    A     E1H        HIHx    I   H    H       HIPxID   H    A   uDA  AfE,DH       HIPxID   H    A  [EtNAGM<DAHAA	A	    A   HIHxID   H    LII9uL    f         AWAVAUATL	  UHLS1H	`7  ~$HcÃHHPHI|@    ;`7  |e       H  E1HH$eD-          t]DHHH  L   L    D   H    H      ǃ   LA    D;  rH<$       H=        H4         	  u25        L18L0L\LqLXe`7  ~)HcÃHHPHI|@        ;`7  |ƅ5   H	      1H[]A\A]A^A_    ff.     f    AVAUATUSLgxID$8M$	  uL1[]A\A]A^    I$4         L轄L    I$4         A$	  tjLM$	  I$L6  1A$`7  - I  H A;$`7  }; t{L    LZLcp@A$25        zf    AV   AUATUSHHeH%(   HD$H   G  f+   #X# #X DH;  +  =  WuHH\   H3\  C  H (   +  A`   f1ҁ  	W  AEAE=  WtqHD  HD  11DHV@uA@AF@HD  H3  11DHjV@uEtF+  wEAEA{D%   `AeID9,  M  H/  1HL`H/  HhHlHU,   HlIT$PH    H?ŅtYHD$eH+%(     H[]A\A]A^    +  #X# f1Ɂ  WF  W#\# E0I$   H    HŅuIT$xH    HŅlLH    HŅPIT$(H    HuŅ2HSHCH;H0  f  AH3ȉ  HH<   +  f1Ɂ  	WuEH;<    A H<  H <     H(  HHP   %H3
@  P  ,  |,  H},  ~,  ,  ,  Љ  HCH;D   AF   =  G    L+  A9LDBAk'|DHiMbD)-  HiMbH&BH&6	       HHPxH   L    A$LD, ,        MiQ   HiMbHHI%AH&AD	PxH   L    CvL,    HiMb    HHH&D	PxH   L    Hc,  H  1    H  H,       H  H9uǃ-    H     H+  h  H+  h  H,  h  H,  h  H+   h  H+  (h  H+  $h  H+  0h  H+  ,h  H+  4h  H+  8h  C H#  1҉Dh  H  @h  +     = W   DStH3  i  S
      EHh  f+  L      X  HHPx   H   L     j       -  ~[H;Hǉ7H;LD+  A8H;LD+  A8H;LD+  A8H;H|+  $9-  H     h    Hh  H{W+  f1=  	WuHH   
   H;  1ɺ     H%H  HH   O     HH h   `,  QH0    
   HH     t!      u,  E1E1J ,  H   HxH     Hǀ      HH)HL   HHH  B@,  HBD,    DH   @  
   HH      @
S      u۽S,  H{1fT$HT$p    ,  T$H{  p    	    X  HHPx1H   L    }+  Dh          AUATA   USHH߉{QHgt
[]A\A]    HŅuHL+  ūL    HDRH~+  uL    []A\A]    Hdf    USH_xHC8   H	  11H	      H	  1    H	  H    %H	  
      H	  H   H	    H    HN   H   H{1[]    ff.         NH  Hf9Gξ   fd5  Jf9Gξ  fP5  Jf9Gξ   ff5  Jf9Gξ  fR5  Jf9Gξ   fX5  Jf9Gξ  fL5  Jf9Gξ   fZ5  J f9GfN5  R$	   t.t7 B @B t*H	  h5  HR8u1        wց  SH	     HxH1}H߾   Pz1[             ATUSHHH  L	      H        H    H    H    HE8u       []A\       LtHE8u       []A\    `7  1ۅ~$HcÃHHPHI|@    ;`7  |H    @     ATUSHHeH%(   HD$11fD$    HH@8   H   HMwH   HT$    D$tN   H    ǃ      Hx    HD$eH+%(   u.H[]A\    H    H            ff.         AWAVAUIATAUSHHGH@8   L)mIEA+  H@8u1H[]A\A]A^A_    Etg5    L\A-  %  I]@E1H    I}H޹@   AH      LA-  H    A9|ʅu'LKdŅuL}JŅu  A-  ~'1HcÃHHPHI|@    A;-  |I}    H,  H ,  AHBH~LZTDD8D@LMM9DGDHD@HHI9uҸP   HH=D  u   Lu   LEt^LsA-  ~.E1IcAHHPHI|@        E;-  |LFKL`LTA-  IU@M,  E1A AH  I E;-  }A> tA~HH$    H$1L4ŅwI-  H    A .  u'H       LuhAUA.     tjA-  Aǅ      A .  HA.  HcI  HB),  I.  A.  A.  Aǅ.     \Aǅ     A .  A  A  1f       	  t=    V9rB;Fr=l7  F1҉h7  HG8u    4  @5     Hǀ	  5ff.          FHǀ	  =  wV J   w	1ɉ    f            4  1Hǀ	  +  ff.         AWH	  IHzAVHAUATUSH0HD$HH)Ht$(H1HT$0H    HB(    HFuH|$tHD$H@   HD$(HH|$UUtHD$H@    HD$(HH|$jtHD$H@(   HD$(HH0[]A\A]A^A_    H\$   Hmq   H HGH1~]E$5  l   H    fE1
A  	Wuu}I	  DJDRH9I	  HI	  H	1D!D!	  I	  H)I	  H	D!A9I	    1LD{  A9B
Hf`A$5  H    D$    I5  f1=  	WH    HEHD$ T$   @I    D$  E1D$AA9  D$HAm E$    HI	  D`xI	     H    H    HI	  D`xI	  D   H    D9tHD$H@   HD$(HI	  H@8<  Lt$   L$L
FL[  I 5  H    Lt$   LH    1Li   L@ri  HD$@Hh  I	  H@8     A	  uA25        H|$c   A5         uII    uHT$ D$HcD$HT$I	  H@8HD$H@   III    I	  HƉ>HD$H    HD$(H{HD$HHhHD$(HI	  H@8H\$   H   HoLHL躞;D      AWAVIAUATUSH	  HHp9     L    A`7  It  1Ʌ~*I	  H   
   H  A;`7  |I	  H    5    HSAą   A`7  1~%HcŃHHPHH|@    A;`7  |H1[CHXH[kHLA`7  ~*HcŃHHPHH|@        A;`7  |Ip9   t#Ih9      Ip9      Iǆp9      HD[]A\A]A^A_    A`7  I	  E1H    ~cI	  H@   AH      HA`7  H    A9|ǅ~&IcAHHPHH|@    E;`7  |H$ZAąHQ@Aą   HyAąwH    A4  I4  HH    I5  Aǆ	      HxH     Hǀ<      HH)D  1HH/A	         A  1҅HHI     A;  ru$5  H$    H    f1ҁ  	Wu%H    =  	WH    HEH    H$I	  Ih9  HH       Aą   I	  H4$Ip9  H       Aą   Mh9  Ip9  Ml$LxHD$I<$       LLU    Iu(LU    IuPLU    IuxL}U    I   LfU    Hl$H} W    AWAw   HA   T    AW4Aw0   HA   T    Ih9      EHL    A	        D      ATAUSHH  H    H    At|HE8uH           []A\    L	     LhH4         L'L
?LTuLYuLm       []A\             AUATUSHH  Ht-HHŀ	  t!    HC8u=    t+[]A\A]    []A\A]    H	      =    uH1H	  25   @    $5    Wu25   tH	         녋85  uf1=  	Wuً6  L5  L獨      HH	  hxH	     L    6  L   D      HH	  DhxH	     L    S    H    H    H            AVH    AUAȝ ATUSHoHH    +  ș Hf1=  	W D DD    HDA    DHH        H߾P     H߾P A    H߾ P A    EDHH        H߾ P     H߾P A    H߾P A    EDHH        HH        ,  Hߍp    HAŋ,  p    HAċ,  p    EDHH        ,  Hߍp    H        ,  Hߍ      HAċ,  p     DHH        ,  Hߍ      H        ,  Hߍ      HAƋ,        HAŋ,        HAċ,        H{EEAV  H        ,  Hߍ      HAƋ,        HAŋ,        HAċ,        H{EEAV  H        ,  Hߍ      HAƋ,        HAŋ,        HAċ,        H{EEAV  H        ,  Hߍ      HAƋ,        HAŋ,        HAċ,        H{EEAV  H        ,  HH       HH        H[H    ]A\A]A^    H        H        H        H            AWH    AVAUL	  ATE1USHH H	  eH%(   HD$1H    A$    L    I$    HIH        I   uH    HA P     Et$At$ L    DLD$    DLA    At$LA    At$LD$    DLD$    DL$DHH    A   AQAWAVDL$DD$     HA P `HH        H    H    H	  T  H    ?HE1    H    HA       H	  D$T  H	  D4T     H	   T  1H	   T  t	dtH	  0T  H	  D,T  H	  H4T   EDEPAH    HA    ZA \HH        H	  1H	  HT$   D$D$    	  L$HH        (6  H	  HT$p    H	  h   HT$    L$T$HH        H	    H	    H    H    H	    H    H    H	  Hh  H    H    	  tH	     H    H    LHD$eH+%(   t    H5    H     H4  []A\A]A^A_    H    H{H        +        HD$eH+%(     H[]A\A]    +  H    H    HEH{H        q,  t r  H        H        H        fl,      n,  E1DL$f=	   w f=   wf
5  fd?  y,   t5+  HT$H    +  HHT$    D$    ŋ,  L+  LD`    HHD`xH   HD$eH+%(   uPHL[]A\A]    p,  YH    +  H    H    HE=    p,  +  JaDH        q,  H        pp,  p,  
IE+  DH    T$    Dl    H|$H        L    I5          A	  A  M0  M  tgHPCI ExprD$0ess HD$(A$5  H    LAHD$AEH@ATAUAUH    LL$@    H    D$(PCI H|$+udtsH 32-bit HHGA05  H        pH|$H        I	  L    L    L        fD$+-XH|$-D$- H 64-bit HHGH|$H            H|$H        L    L    I5          H        H|$H            H|$H    D        H|$H        L    L    I5          H|$H        H|$H            H|$H            H|$H            H|$H            H        H        L.    H            H   H            HH    E1    D;  }IcAHi  H|@    H            H    L        H4$H        Ip9  A    E1Ih9  Mp9          HH        H    L        I	  H        1I	  H   A9`7  ~Hi  At     H͋ H       H    1H    AŅuH    AŅtHE        H        H                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        bnx2 DEBUG: shmem states:
 c drv_pulse_mb[%08x]
 c condition[%08x]
 DEBUG: 0x3fc[%08x]
 %s-%d <--- start FTQ dump --->
 %s %08x
 CPU states:
 <--- end FTQ dump --->
 <--- start TBDC dump --->
 TBDC free cnt: %ld
 <--- end TBDC dump --->
 DEBUG: PBA[%08x]
 include/linux/dma-mapping.h full half SerDes Copper Remote Copper c, receive  c& transmit  c, transmit  cflow control ON c
 NIC %s Link is Down
 &bp->cnic_lock Not PCIE, aborting
 MN V0 Cannot register net device
  %dMHz failed to reset NIC, closing
 bnx2/bnx2-rv2p-06-6.0.15.fw bnx2/bnx2-mips-06-6.2.3.fw bnx2/bnx2-rv2p-09ax-6.0.17.fw bnx2/bnx2-mips-09-6.2.1b.fw bnx2/bnx2-rv2p-09-6.0.17.fw using MSI
 using MSIX
 RV2P_PFTQ_CTL RV2P_TFTQ_CTL RV2P_MFTQ_CTL TBDR_FTQ_CTL TDMA_FTQ_CTL TXP_FTQ_CTL TPAT_FTQ_CTL RXP_CFTQ_CTL RXP_FTQ_CTL COM_COMXQ_FTQ_CTL COM_COMTQ_FTQ_CTL COM_COMQ_FTQ_CTL CP_CPQ_FTQ_CTL 5709 Buffered flash (256kB) EEPROM - slow Entry 0001 Non-buffered flash (128kB) Non-buffered flash (256kB) Entry 0100 Non-buffered flash (64kB) EEPROM - fast Entry 1001 Entry 1010 Buffered flash (128kB) Entry 1100 Entry 1101 Entry 1110 (Atmel) Buffered flash (256kB)       <--- start MCP states dump --->
        DEBUG: MCP_STATE_P0[%08x] MCP_STATE_P1[%08x]
   DEBUG: MCP mode[%08x] state[%08x] evt_mask[%08x]
       DEBUG: pc[%08x] pc[%08x] instr[%08x]
   DEBUG: drv_mb[%08x] fw_mb[%08x] link_status[%08x]       DEBUG: dev_info_signature[%08x] reset_type[%08x]        DEBUG: %08x: %08x %08x %08x %08x
       <--- end MCP states dump --->
  %06x mode %x state %x evt_mask %x pc %x pc %x instr %x
 LINE     CID  BIDX   CMD  VALIDS
       %02x    %06x  %04lx   %02x    [%x]
     DEBUG: intr_sem[%x] PCI_CMD[%08x]
      DEBUG: PCI_PM[%08x] PCI_MISC_CFG[%08x]
 DEBUG: EMAC_TX_STATUS[%08x] EMAC_RX_STATUS[%08x]
       DEBUG: RPM_MGMT_PKT_CTRL[%08x]
 DEBUG: HC_STATS_INTERRUPT_STATUS[%08x]
 %s %s: rejecting DMA map of vmalloc memory
     NIC %s Link is Up, %d Mbps %s duplex    BUG! Tx ring full when queue awake!
    include/linux/dynamic_queue_limits.h    init'ed rx page ring %d with %d/%d pages only
  init'ed rx ring %d with %d/%d skbs only
        1bnx2: Unknown flash/EEPROM type
      Cannot enable PCI device, aborting
     Cannot find PCI device base address, aborting
  Cannot obtain PCI resources, aborting
  Cannot find power management capability, aborting
      Cannot map register space, aborting
    Cannot find PCIX capability, aborting
  dma_set_coherent_mask failed, aborting
 System does not support DMA, aborting
  5706 A1 can only be used in a PCIX bus, aborting
       Firmware not running, aborting
 %s (%c%d) %s found at mem %lx, IRQ %d, node addr %pM
   3bnx2: fw sync timeout, reset code = %x
       3bnx2: Chip reset did not complete
    3bnx2: Chip not in correct endian mode
        Cannot re-enable PCI device after reset
        3bnx2: Firmware file "%s" is invalid
  No interrupt was generated using MSI, switching to INTx mode. Please report this failure to the PCI maintainer and include system chipset information.
 Entry 0101: ST M45PE10 (128kB non-buffered)     Entry 0110: ST M45PE20 (256kB non-buffered)     Broadcom NetXtreme II BCM5706 1000Base-T        HP NC370T Multifunction Gigabit Server Adapter  HP NC370i Multifunction Gigabit Server Adapter  Broadcom NetXtreme II BCM5706 1000Base-SX       HP NC370F Multifunction Gigabit Server Adapter  Broadcom NetXtreme II BCM5708 1000Base-T        Broadcom NetXtreme II BCM5708 1000Base-SX       Broadcom NetXtreme II BCM5709 1000Base-T        Broadcom NetXtreme II BCM5709 1000Base-SX       Broadcom NetXtreme II BCM5716 1000Base-T        Broadcom NetXtreme II BCM5716 1000Base-SX                                                       |+              +              +              S              _              S             S             S             S             S             |S             S             S             S     strscpy                                   \            0  
       @  H                  \         X            @  T                @  T               0   #   $   (   (  0(  P(  @+  ,  /  X0   <  <   @  @  @  @  C  XD   L  L  @L  TL  O  P  S  DT   \  \  \  \  _   `   d  (d   h  Hh  Lh  `h  h  i             UUUUUUUU        l       ?                       ?                      $         (        P       T     X          T     h     wwwwl     wwwwp     wwwwt     wwww                s                                                                        	     ?                                         (         (      ?  (  ??    (        (        (        (        4(      @(      D(      H(      L(     ,         ,         <         <         <  q    <  ?    <      <      <      <        <       P         P        \         \        \         \        \      \      q\      3  \      ss \      7 h        h      h      h      h      h       h        $h        (h        ,h      0h      4h      8h      <h        @h        Dh        Lh      Ph      Th      Xh      \h      i       i                     strnlen         __fortify_strlen                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              register_test (offline)         memory_test (offline)           loopback_test (offline)         nvram_test (online)             interrupt_test (online)         link_test (online)                                                  
                                                                                                                               !       "       #       $       %       &       '       (       )       *       +       ,       -       .       /       0       1       2       3       5       7       9       :       P               rx_bytes                        rx_error_bytes                  tx_bytes                        tx_error_bytes                  rx_ucast_packets                rx_mcast_packets                rx_bcast_packets                tx_ucast_packets                tx_mcast_packets                tx_bcast_packets                tx_mac_errors                   tx_carrier_errors               rx_crc_errors                   rx_align_errors                 tx_single_collisions            tx_multi_collisions             tx_deferred                     tx_excess_collisions            tx_late_collisions              tx_total_collisions             rx_fragments                    rx_jabbers                      rx_undersize_packets            rx_oversize_packets             rx_64_byte_packets              rx_65_to_127_byte_packets       rx_128_to_255_byte_packets      rx_256_to_511_byte_packets      rx_512_to_1023_byte_packets     rx_1024_to_1522_byte_packets    rx_1523_to_9022_byte_packets    tx_64_byte_packets              tx_65_to_127_byte_packets       tx_128_to_255_byte_packets      tx_256_to_511_byte_packets      tx_512_to_1023_byte_packets     tx_1024_to_1522_byte_packets    tx_1523_to_9022_byte_packets    rx_xon_frames                   rx_xoff_frames                  tx_xon_frames                   tx_xoff_frames                  rx_mac_ctrl_frames              rx_filtered_packets             rx_ftq_discards                 rx_discards                     rx_fw_discards                                                                                 @  S                           K  S                          G  S                         O  S                          S  S                          [  S                         W  S                         _  S                          "b  S                           *k  S                          &g  S                          .sn W Sh     	                    3s  S                          ;{  S                          7sv W Sh     	                     ?s~ W Sh     	                    J  <  1                           J  <  1                           J                            L                             <  1                                                                                 9                           :                           ;          	                 <          
                                                                                                                                               disable_msi                              P       P   R P P  P 4P                        P       P   R P P  P 4P   
                     P       P   R P P  P 4P                        P       P   R P P  P 4P                        P       P   R P P  P 4P       parm=disable_msi:Disable Message Signaled Interrupt (MSI) parmtype=disable_msi:int firmware=bnx2/bnx2-rv2p-09ax-6.0.17.fw firmware=bnx2/bnx2-rv2p-09-6.0.17.fw firmware=bnx2/bnx2-mips-09-6.2.1b.fw firmware=bnx2/bnx2-rv2p-06-6.0.15.fw firmware=bnx2/bnx2-mips-06-6.2.3.fw license=GPL description=QLogic BCM5706/5708/5709/5716 Driver author=Michael Chan <mchan@broadcom.com> alias=pci:v000014E4d0000163Csv*sd*bc*sc*i* alias=pci:v000014E4d0000163Bsv*sd*bc*sc*i* alias=pci:v000014E4d0000163Asv*sd*bc*sc*i* alias=pci:v000014E4d00001639sv*sd*bc*sc*i* alias=pci:v000014E4d000016ACsv*sd*bc*sc*i* alias=pci:v000014E4d000016AAsv*sd*bc*sc*i* alias=pci:v000014E4d000016AAsv0000103Csd00003102bc*sc*i* alias=pci:v000014E4d0000164Csv*sd*bc*sc*i* alias=pci:v000014E4d0000164Asv*sd*bc*sc*i* alias=pci:v000014E4d0000164Asv0000103Csd00003106bc*sc*i* alias=pci:v000014E4d0000164Asv0000103Csd00003101bc*sc*i* depends= retpoline=Y intree=Y name=bnx2 vermagic=6.1.0-37-amd64 SMP preempt mod_unload modversions                                $                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 (                 (                 (                (            (                                                                                                                                                         (          (                       (  8  (                 8                         (  0  (                   0                                               (  0  (                   0  (                              (                                        (    0  8  @  8  0  (                                                                                                    (    0  8  `  8  0  (                     `  8  0  (                     `                                                                             (  0  (                   0                         (  0  (                                                                                                                                                           (                                      (                                                                                        (    0  8    8  0  (                                                                                                          (    0  8  0  (                                             (          (          (                         (    0  8  @  8  0  (                     @  8  0  (                     @                                                                                 (  H  (                 H                                                                      (          (                         (    0  8  @  8  0  (                     @                       (                 (                 (                                                            (    0  8    8  0  (                                      (  x  (                 x                                                 (  8                     8               8                                                   (    0  8  X  8  0  (                     X                         (    0  8  P  8  0  (                     P                     (              (                                  (    0  8    8  0  (                                              (    0  8    8  0  (                       8  0  (                                              (    0  8    8  0  (                                              (  0  (                                          (  0  (                   0                             (    0  8  x  8  0  (                     x                                                                                 (                 (                 (                                                                                                                        (                 (                   (    0  8  @  8  0  (                                            (  0  (                   0  (                                          (  0  H  0  (                   H  0  (                   H                       (                 (                 (                 (                         (    0  8  H  8  0  (                     H                         (    0  8    8  0  (                                              (    0  8  0  (                     8                       (                 (                     8               8                         (    0  8  P  8  0  (                     P  8  0  (                     P                         (    0  8  H  8  0  (                     H                       (  8  (                 8                                                       (    0  8  0  (                     8  0  (                     8  0  (                                            (    0  8  0  (                     8  0  (                     8                         (    0  8  @  8  0  (                                            (  0  (                   0                         (  0  @  0  (                   @                       (                 (                 (                                                                                                        0               0                         (    0  8  @  8  0  (                     @                                                       (    0  8  h  8  0  (                     h                         (    0  8  H  8  0  (                     H                                                                 (                 (                 (                                   (  0  8  @  H  P  0  (                   (                     (    0  8  X  `  h  p  X  `  X  8  0  (                     8  (                 8  (                 8  x            8     0  H                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          // Random number extensions -*- C++ -*-

// Copyright (C) 2012-2022 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/** @file ext/random
 *  This file is a GNU extension to the Standard C++ Library.
 */

#ifndef _EXT_RANDOM
#define _EXT_RANDOM 1

#pragma GCC system_header

#if __cplusplus < 201103L
# include <bits/c++0x_warning.h>
#else

#include <random>
#include <algorithm>
#include <array>
#include <ext/cmath>
#ifdef __SSE2__
# include <emmintrin.h>
#endif

#if defined(_GLIBCXX_USE_C99_STDINT_TR1) && defined(UINT32_C)

namespace __gnu_cxx _GLIBCXX_VISIBILITY(default)
{
_GLIBCXX_BEGIN_NAMESPACE_VERSION

#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__

  /* Mersenne twister implementation optimized for vector operations.
   *
   * Reference: http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/
   */
  template<typename _UIntType, size_t __m,
	   size_t __pos1, size_t __sl1, size_t __sl2,
	   size_t __sr1, size_t __sr2,
	   uint32_t __msk1, uint32_t __msk2,
	   uint32_t __msk3, uint32_t __msk4,
	   uint32_t __parity1, uint32_t __parity2,
	   uint32_t __parity3, uint32_t __parity4>
    class simd_fast_mersenne_twister_engine
    {
      static_assert(std::is_unsigned<_UIntType>::value, "template argument "
		    "substituting _UIntType not an unsigned integral type");
      static_assert(__sr1 < 32, "first right shift too large");
      static_assert(__sr2 < 16, "second right shift too large");
      static_assert(__sl1 < 32, "first left shift too large");
      static_assert(__sl2 < 16, "second left shift too large");

    public:
      typedef _UIntType result_type;

    private:
      static constexpr size_t m_w = sizeof(result_type) * 8;
      static constexpr size_t _M_nstate = __m / 128 + 1;
      static constexpr size_t _M_nstate32 = _M_nstate * 4;

      static_assert(std::is_unsigned<_UIntType>::value, "template argument "
		    "substituting _UIntType not an unsigned integral type");
      static_assert(__pos1 < _M_nstate, "POS1 not smaller than state size");
      static_assert(16 % sizeof(_UIntType) == 0,
		    "UIntType size must divide 16");

      template<typename _Sseq>
	using _If_seed_seq
	  = typename std::enable_if<std::__detail::__is_seed_seq<
	    _Sseq, simd_fast_mersenne_twister_engine, result_type>::value
	    >::type;

    public:
      static constexpr size_t state_size = _M_nstate * (16
							/ sizeof(result_type));
      static constexpr result_type default_seed = 5489u;

      // constructors and member functions

      simd_fast_mersenne_twister_engine()
      : simd_fast_mersenne_twister_engine(default_seed)
      { }

      explicit
      simd_fast_mersenne_twister_engine(result_type __sd)
      { seed(__sd); }

      template<typename _Sseq, typename = _If_seed_seq<_Sseq>>
	explicit
	simd_fast_mersenne_twister_engine(_Sseq& __q)
	{ seed(__q); }

      void
      seed(result_type __sd = default_seed);

      template<typename _Sseq>
	_If_seed_seq<_Sseq>
	seed(_Sseq& __q);

      static constexpr result_type
      min()
      { return 0; }

      static constexpr result_type
      max()
      { return std::numeric_limits<result_type>::max(); }

      void
      discard(unsigned long long __z);

      result_type
      operator()()
      {
	if (__builtin_expect(_M_pos >= state_size, 0))
	  _M_gen_rand();

	return _M_stateT[_M_pos++];
      }

      template<typename _UIntType_2, size_t __m_2,
	       size_t __pos1_2, size_t __sl1_2, size_t __sl2_2,
	       size_t __sr1_2, size_t __sr2_2,
	       uint32_t __msk1_2, uint32_t __msk2_2,
	       uint32_t __msk3_2, uint32_t __msk4_2,
	       uint32_t __parity1_2, uint32_t __parity2_2,
	       uint32_t __parity3_2, uint32_t __parity4_2>
	friend bool
	operator==(const simd_fast_mersenne_twister_engine<_UIntType_2,
		   __m_2, __pos1_2, __sl1_2, __sl2_2, __sr1_2, __sr2_2,
		   __msk1_2, __msk2_2, __msk3_2, __msk4_2,
		   __parity1_2, __parity2_2, __parity3_2, __parity4_2>& __lhs,
		   const simd_fast_mersenne_twister_engine<_UIntType_2,
		   __m_2, __pos1_2, __sl1_2, __sl2_2, __sr1_2, __sr2_2,
		   __msk1_2, __msk2_2, __msk3_2, __msk4_2,
		   __parity1_2, __parity2_2, __parity3_2, __parity4_2>& __rhs);

      template<typename _UIntType_2, size_t __m_2,
	       size_t __pos1_2, size_t __sl1_2, size_t __sl2_2,
	       size_t __sr1_2, size_t __sr2_2,
	       uint32_t __msk1_2, uint32_t __msk2_2,
	       uint32_t __msk3_2, uint32_t __msk4_2,
	       uint32_t __parity1_2, uint32_t __parity2_2,
	       uint32_t __parity3_2, uint32_t __parity4_2,
	       typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const __gnu_cxx::simd_fast_mersenne_twister_engine
		   <_UIntType_2,
		   __m_2, __pos1_2, __sl1_2, __sl2_2, __sr1_2, __sr2_2,
		   __msk1_2, __msk2_2, __msk3_2, __msk4_2,
		   __parity1_2, __parity2_2, __parity3_2, __parity4_2>& __x);

      template<typename _UIntType_2, size_t __m_2,
	       size_t __pos1_2, size_t __sl1_2, size_t __sl2_2,
	       size_t __sr1_2, size_t __sr2_2,
	       uint32_t __msk1_2, uint32_t __msk2_2,
	       uint32_t __msk3_2, uint32_t __msk4_2,
	       uint32_t __parity1_2, uint32_t __parity2_2,
	       uint32_t __parity3_2, uint32_t __parity4_2,
	       typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::simd_fast_mersenne_twister_engine<_UIntType_2,
		   __m_2, __pos1_2, __sl1_2, __sl2_2, __sr1_2, __sr2_2,
		   __msk1_2, __msk2_2, __msk3_2, __msk4_2,
		   __parity1_2, __parity2_2, __parity3_2, __parity4_2>& __x);

    private:
      union
      {
#ifdef __SSE2__
	__m128i _M_state[_M_nstate];
#endif
#ifdef __ARM_NEON
#ifdef __aarch64__
	__Uint32x4_t _M_state[_M_nstate];
#endif
#endif
	uint32_t _M_state32[_M_nstate32];
	result_type _M_stateT[state_size];
      } __attribute__ ((__aligned__ (16)));
      size_t _M_pos;

      void _M_gen_rand(void);
      void _M_period_certification();
  };


  template<typename _UIntType, size_t __m,
	   size_t __pos1, size_t __sl1, size_t __sl2,
	   size_t __sr1, size_t __sr2,
	   uint32_t __msk1, uint32_t __msk2,
	   uint32_t __msk3, uint32_t __msk4,
	   uint32_t __parity1, uint32_t __parity2,
	   uint32_t __parity3, uint32_t __parity4>
    inline bool
    operator!=(const __gnu_cxx::simd_fast_mersenne_twister_engine<_UIntType,
	       __m, __pos1, __sl1, __sl2, __sr1, __sr2, __msk1, __msk2, __msk3,
	       __msk4, __parity1, __parity2, __parity3, __parity4>& __lhs,
	       const __gnu_cxx::simd_fast_mersenne_twister_engine<_UIntType,
	       __m, __pos1, __sl1, __sl2, __sr1, __sr2, __msk1, __msk2, __msk3,
	       __msk4, __parity1, __parity2, __parity3, __parity4>& __rhs)
    { return !(__lhs == __rhs); }


  /* Definitions for the SIMD-oriented Fast Mersenne Twister as defined
   * in the C implementation by Daito and Matsumoto, as both a 32-bit
   * and 64-bit version.
   */
  typedef simd_fast_mersenne_twister_engine<uint32_t, 607, 2,
					    15, 3, 13, 3,
					    0xfdff37ffU, 0xef7f3f7dU,
					    0xff777b7dU, 0x7ff7fb2fU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x5986f054U>
    sfmt607;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 607, 2,
					    15, 3, 13, 3,
					    0xfdff37ffU, 0xef7f3f7dU,
					    0xff777b7dU, 0x7ff7fb2fU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x5986f054U>
    sfmt607_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 1279, 7,
					    14, 3, 5, 1,
					    0xf7fefffdU, 0x7fefcfffU,
					    0xaff3ef3fU, 0xb5ffff7fU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x20000000U>
    sfmt1279;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 1279, 7,
					    14, 3, 5, 1,
					    0xf7fefffdU, 0x7fefcfffU,
					    0xaff3ef3fU, 0xb5ffff7fU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x20000000U>
    sfmt1279_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 2281, 12,
					    19, 1, 5, 1,
					    0xbff7ffbfU, 0xfdfffffeU,
					    0xf7ffef7fU, 0xf2f7cbbfU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x41dfa600U>
    sfmt2281;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 2281, 12,
					    19, 1, 5, 1,
					    0xbff7ffbfU, 0xfdfffffeU,
					    0xf7ffef7fU, 0xf2f7cbbfU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x41dfa600U>
    sfmt2281_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 4253, 17,
					    20, 1, 7, 1,
					    0x9f7bffffU, 0x9fffff5fU,
					    0x3efffffbU, 0xfffff7bbU,
					    0xa8000001U, 0xaf5390a3U,
					    0xb740b3f8U, 0x6c11486dU>
    sfmt4253;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 4253, 17,
					    20, 1, 7, 1,
					    0x9f7bffffU, 0x9fffff5fU,
					    0x3efffffbU, 0xfffff7bbU,
					    0xa8000001U, 0xaf5390a3U,
					    0xb740b3f8U, 0x6c11486dU>
    sfmt4253_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 11213, 68,
					    14, 3, 7, 3,
					    0xeffff7fbU, 0xffffffefU,
					    0xdfdfbfffU, 0x7fffdbfdU,
					    0x00000001U, 0x00000000U,
					    0xe8148000U, 0xd0c7afa3U>
    sfmt11213;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 11213, 68,
					    14, 3, 7, 3,
					    0xeffff7fbU, 0xffffffefU,
					    0xdfdfbfffU, 0x7fffdbfdU,
					    0x00000001U, 0x00000000U,
					    0xe8148000U, 0xd0c7afa3U>
    sfmt11213_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 19937, 122,
					    18, 1, 11, 1,
					    0xdfffffefU, 0xddfecb7fU,
					    0xbffaffffU, 0xbffffff6U,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x13c9e684U>
    sfmt19937;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 19937, 122,
					    18, 1, 11, 1,
					    0xdfffffefU, 0xddfecb7fU,
					    0xbffaffffU, 0xbffffff6U,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0x13c9e684U>
    sfmt19937_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 44497, 330,
					    5, 3, 9, 3,
					    0xeffffffbU, 0xdfbebfffU,
					    0xbfbf7befU, 0x9ffd7bffU,
					    0x00000001U, 0x00000000U,
					    0xa3ac4000U, 0xecc1327aU>
    sfmt44497;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 44497, 330,
					    5, 3, 9, 3,
					    0xeffffffbU, 0xdfbebfffU,
					    0xbfbf7befU, 0x9ffd7bffU,
					    0x00000001U, 0x00000000U,
					    0xa3ac4000U, 0xecc1327aU>
    sfmt44497_64;

#if __SIZE_WIDTH__ >= 32

  typedef simd_fast_mersenne_twister_engine<uint32_t, 86243, 366,
					    6, 7, 19, 1,
					    0xfdbffbffU, 0xbff7ff3fU,
					    0xfd77efffU, 0xbf9ff3ffU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0xe9528d85U>
    sfmt86243;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 86243, 366,
					    6, 7, 19, 1,
					    0xfdbffbffU, 0xbff7ff3fU,
					    0xfd77efffU, 0xbf9ff3ffU,
					    0x00000001U, 0x00000000U,
					    0x00000000U, 0xe9528d85U>
    sfmt86243_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 132049, 110,
					    19, 1, 21, 1,
					    0xffffbb5fU, 0xfb6ebf95U,
					    0xfffefffaU, 0xcff77fffU,
					    0x00000001U, 0x00000000U,
					    0xcb520000U, 0xc7e91c7dU>
    sfmt132049;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 132049, 110,
					    19, 1, 21, 1,
					    0xffffbb5fU, 0xfb6ebf95U,
					    0xfffefffaU, 0xcff77fffU,
					    0x00000001U, 0x00000000U,
					    0xcb520000U, 0xc7e91c7dU>
    sfmt132049_64;


  typedef simd_fast_mersenne_twister_engine<uint32_t, 216091, 627,
					    11, 3, 10, 1,
					    0xbff7bff7U, 0xbfffffffU,
					    0xbffffa7fU, 0xffddfbfbU,
					    0xf8000001U, 0x89e80709U,
					    0x3bd2b64bU, 0x0c64b1e4U>
    sfmt216091;

  typedef simd_fast_mersenne_twister_engine<uint64_t, 216091, 627,
					    11, 3, 10, 1,
					    0xbff7bff7U, 0xbfffffffU,
					    0xbffffa7fU, 0xffddfbfbU,
					    0xf8000001U, 0x89e80709U,
					    0x3bd2b64bU, 0x0c64b1e4U>
    sfmt216091_64;
#endif // __SIZE_WIDTH__ >= 32

#endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__

  /**
   * @brief A beta continuous distribution for random numbers.
   *
   * The formula for the beta probability density function is:
   * @f[
   *     p(x|\alpha,\beta) = \frac{1}{B(\alpha,\beta)}
   *                         x^{\alpha - 1} (1 - x)^{\beta - 1}
   * @f]
   */
  template<typename _RealType = double>
    class beta_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef beta_distribution<_RealType> distribution_type;
	friend class beta_distribution<_RealType>;

	param_type() : param_type(1) { }

	explicit
	param_type(_RealType __alpha_val, _RealType __beta_val = _RealType(1))
	: _M_alpha(__alpha_val), _M_beta(__beta_val)
	{
	  __glibcxx_assert(_M_alpha > _RealType(0));
	  __glibcxx_assert(_M_beta > _RealType(0));
	}

	_RealType
	alpha() const
	{ return _M_alpha; }

	_RealType
	beta() const
	{ return _M_beta; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return (__p1._M_alpha == __p2._M_alpha
		  && __p1._M_beta == __p2._M_beta); }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void
	_M_initialize();

	_RealType _M_alpha;
	_RealType _M_beta;
      };

    public:
      beta_distribution() : beta_distribution(1.0) { }

      /**
       * @brief Constructs a beta distribution with parameters
       * @f$\alpha@f$ and @f$\beta@f$.
       */
      explicit
      beta_distribution(_RealType __alpha_val,
			_RealType __beta_val = _RealType(1))
      : _M_param(__alpha_val, __beta_val)
      { }

      explicit
      beta_distribution(const param_type& __p)
      : _M_param(__p)
      { }

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { }

      /**
       * @brief Returns the @f$\alpha@f$ of the distribution.
       */
      _RealType
      alpha() const
      { return _M_param.alpha(); }

      /**
       * @brief Returns the @f$\beta@f$ of the distribution.
       */
      _RealType
      beta() const
      { return _M_param.beta(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return result_type(0); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return result_type(1); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, _M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two beta distributions have the same
       *        parameters and the sequences that would be generated
       *        are equal.
       */
      friend bool
      operator==(const beta_distribution& __d1,
		 const beta_distribution& __d2)
      { return __d1._M_param == __d2._M_param; }

      /**
       * @brief Inserts a %beta_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %beta_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const __gnu_cxx::beta_distribution<_RealType1>& __x);

      /**
       * @brief Extracts a %beta_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x  A %beta_distribution random number generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::beta_distribution<_RealType1>& __x);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
    };

  /**
   * @brief Return true if two beta distributions are different.
   */
  template<typename _RealType>
    inline bool
    operator!=(const __gnu_cxx::beta_distribution<_RealType>& __d1,
	       const __gnu_cxx::beta_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A multi-variate normal continuous distribution for random numbers.
   *
   * The formula for the normal probability density function is
   * @f[
   *     p(\overrightarrow{x}|\overrightarrow{\mu },\Sigma) =
   *       \frac{1}{\sqrt{(2\pi )^k\det(\Sigma))}}
   *       e^{-\frac{1}{2}(\overrightarrow{x}-\overrightarrow{\mu})^\text{T}
   *          \Sigma ^{-1}(\overrightarrow{x}-\overrightarrow{\mu})}
   * @f]
   *
   * where @f$\overrightarrow{x}@f$ and @f$\overrightarrow{\mu}@f$ are
   * vectors of dimension @f$k@f$ and @f$\Sigma@f$ is the covariance
   * matrix (which must be positive-definite).
   */
  template<std::size_t _Dimen, typename _RealType = double>
    class normal_mv_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");
      static_assert(_Dimen != 0, "dimension is zero");

    public:
      /** The type of the range of the distribution. */
      typedef std::array<_RealType, _Dimen> result_type;
      /** Parameter type. */
      class param_type
      {
	static constexpr size_t _M_t_size = _Dimen * (_Dimen + 1) / 2;

      public:
	typedef normal_mv_distribution<_Dimen, _RealType> distribution_type;
	friend class normal_mv_distribution<_Dimen, _RealType>;

	param_type()
	{
	  std::fill(_M_mean.begin(), _M_mean.end(), _RealType(0));
	  auto __it = _M_t.begin();
	  for (size_t __i = 0; __i < _Dimen; ++__i)
	    {
	      std::fill_n(__it, __i, _RealType(0));
	      __it += __i;
	      *__it++ = _RealType(1);
	    }
	}

	template<typename _ForwardIterator1, typename _ForwardIterator2>
	  param_type(_ForwardIterator1 __meanbegin,
		     _ForwardIterator1 __meanend,
		     _ForwardIterator2 __varcovbegin,
		     _ForwardIterator2 __varcovend)
	{
	  __glibcxx_function_requires(_ForwardIteratorConcept<
				      _ForwardIterator1>)
	  __glibcxx_function_requires(_ForwardIteratorConcept<
				      _ForwardIterator2>)
	  _GLIBCXX_DEBUG_ASSERT(std::distance(__meanbegin, __meanend)
				<= _Dimen);
	  const auto __dist = std::distance(__varcovbegin, __varcovend);
	  _GLIBCXX_DEBUG_ASSERT(__dist == _Dimen * _Dimen
				|| __dist == _Dimen * (_Dimen + 1) / 2
				|| __dist == _Dimen);

	  if (__dist == _Dimen * _Dimen)
	    _M_init_full(__meanbegin, __meanend, __varcovbegin, __varcovend);
	  else if (__dist == _Dimen * (_Dimen + 1) / 2)
	    _M_init_lower(__meanbegin, __meanend, __varcovbegin, __varcovend);
	  else
	    {
	      __glibcxx_assert(__dist == _Dimen);
	      _M_init_diagonal(__meanbegin, __meanend,
			       __varcovbegin, __varcovend);
	    }
	}

	param_type(std::initializer_list<_RealType> __mean,
		   std::initializer_list<_RealType> __varcov)
	{
	  _GLIBCXX_DEBUG_ASSERT(__mean.size() <= _Dimen);
	  _GLIBCXX_DEBUG_ASSERT(__varcov.size() == _Dimen * _Dimen
				|| __varcov.size() == _Dimen * (_Dimen + 1) / 2
				|| __varcov.size() == _Dimen);

	  if (__varcov.size() == _Dimen * _Dimen)
	    _M_init_full(__mean.begin(), __mean.end(),
			 __varcov.begin(), __varcov.end());
	  else if (__varcov.size() == _Dimen * (_Dimen + 1) / 2)
	    _M_init_lower(__mean.begin(), __mean.end(),
			  __varcov.begin(), __varcov.end());
	  else
	    {
	      __glibcxx_assert(__varcov.size() == _Dimen);
	      _M_init_diagonal(__mean.begin(), __mean.end(),
			       __varcov.begin(), __varcov.end());
	    }
	}

	std::array<_RealType, _Dimen>
	mean() const
	{ return _M_mean; }

	std::array<_RealType, _M_t_size>
	varcov() const
	{ return _M_t; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_mean == __p2._M_mean && __p1._M_t == __p2._M_t; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	template <typename _InputIterator1, typename _InputIterator2>
	  void _M_init_full(_InputIterator1 __meanbegin,
			    _InputIterator1 __meanend,
			    _InputIterator2 __varcovbegin,
			    _InputIterator2 __varcovend);
	template <typename _InputIterator1, typename _InputIterator2>
	  void _M_init_lower(_InputIterator1 __meanbegin,
			     _InputIterator1 __meanend,
			     _InputIterator2 __varcovbegin,
			     _InputIterator2 __varcovend);
	template <typename _InputIterator1, typename _InputIterator2>
	  void _M_init_diagonal(_InputIterator1 __meanbegin,
				_InputIterator1 __meanend,
				_InputIterator2 __varbegin,
				_InputIterator2 __varend);

	// param_type constructors apply Cholesky decomposition to the
	// varcov matrix in _M_init_full and _M_init_lower, but the
	// varcov matrix output ot a stream is already decomposed, so
	// we need means to restore it as-is when reading it back in.
	template<size_t _Dimen1, typename _RealType1,
		 typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::normal_mv_distribution<_Dimen1, _RealType1>&
		   __x);
	param_type(std::array<_RealType, _Dimen> const &__mean,
		   std::array<_RealType, _M_t_size> const &__varcov)
	  : _M_mean (__mean), _M_t (__varcov)
	{}

	std::array<_RealType, _Dimen> _M_mean;
	std::array<_RealType, _M_t_size> _M_t;
      };

    public:
      normal_mv_distribution()
      : _M_param(), _M_nd()
      { }

      template<typename _ForwardIterator1, typename _ForwardIterator2>
	normal_mv_distribution(_ForwardIterator1 __meanbegin,
			       _ForwardIterator1 __meanend,
			       _ForwardIterator2 __varcovbegin,
			       _ForwardIterator2 __varcovend)
	: _M_param(__meanbegin, __meanend, __varcovbegin, __varcovend),
	  _M_nd()
	{ }

      normal_mv_distribution(std::initializer_list<_RealType> __mean,
			     std::initializer_list<_RealType> __varcov)
      : _M_param(__mean, __varcov), _M_nd()
      { }

      explicit
      normal_mv_distribution(const param_type& __p)
      : _M_param(__p), _M_nd()
      { }

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { _M_nd.reset(); }

      /**
       * @brief Returns the mean of the distribution.
       */
      result_type
      mean() const
      { return _M_param.mean(); }

      /**
       * @brief Returns the compact form of the variance/covariance
       * matrix of the distribution.
       */
      std::array<_RealType, _Dimen * (_Dimen + 1) / 2>
      varcov() const
      { return _M_param.varcov(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { result_type __res;
	__res.fill(std::numeric_limits<_RealType>::lowest());
	return __res; }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { result_type __res;
	__res.fill(std::numeric_limits<_RealType>::max());
	return __res; }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, _M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ return this->__generate_impl(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ return this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two multi-variant normal distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      template<size_t _Dimen1, typename _RealType1>
	friend bool
	operator==(const
		   __gnu_cxx::normal_mv_distribution<_Dimen1, _RealType1>&
		   __d1,
		   const
		   __gnu_cxx::normal_mv_distribution<_Dimen1, _RealType1>&
		   __d2);

      /**
       * @brief Inserts a %normal_mv_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %normal_mv_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<size_t _Dimen1, typename _RealType1,
	       typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const
		   __gnu_cxx::normal_mv_distribution<_Dimen1, _RealType1>&
		   __x);

      /**
       * @brief Extracts a %normal_mv_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x  A %normal_mv_distribution random number generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error
       *          state.
       */
      template<size_t _Dimen1, typename _RealType1,
	       typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::normal_mv_distribution<_Dimen1, _RealType1>&
		   __x);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
      std::normal_distribution<_RealType> _M_nd;
  };

  /**
   * @brief Return true if two multi-variate normal distributions are
   * different.
   */
  template<size_t _Dimen, typename _RealType>
    inline bool
    operator!=(const __gnu_cxx::normal_mv_distribution<_Dimen, _RealType>&
	       __d1,
	       const __gnu_cxx::normal_mv_distribution<_Dimen, _RealType>&
	       __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A Rice continuous distribution for random numbers.
   *
   * The formula for the Rice probability density function is
   * @f[
   *     p(x|\nu,\sigma) = \frac{x}{\sigma^2}
   *                       \exp\left(-\frac{x^2+\nu^2}{2\sigma^2}\right)
   *                       I_0\left(\frac{x \nu}{\sigma^2}\right)
   * @f]
   * where @f$I_0(z)@f$ is the modified Bessel function of the first kind
   * of order 0 and @f$\nu >= 0@f$ and @f$\sigma > 0@f$.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$\sqrt{\pi/2}L_{1/2}(-\nu^2/2\sigma^2)@f$</td></tr>
   * <tr><td>Variance</td><td>@f$2\sigma^2 + \nu^2
   *                   + (\pi\sigma^2/2)L^2_{1/2}(-\nu^2/2\sigma^2)@f$</td></tr>
   * <tr><td>Range</td><td>@f$[0, \infty)@f$</td></tr>
   * </table>
   * where @f$L_{1/2}(x)@f$ is the Laguerre polynomial of order 1/2.
   */
  template<typename _RealType = double>
    class
    rice_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");
    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef rice_distribution<result_type> distribution_type;

	param_type() : param_type(0) { }

	param_type(result_type __nu_val,
		   result_type __sigma_val = result_type(1))
	: _M_nu(__nu_val), _M_sigma(__sigma_val)
	{
	  __glibcxx_assert(_M_nu >= result_type(0));
	  __glibcxx_assert(_M_sigma > result_type(0));
	}

	result_type
	nu() const
	{ return _M_nu; }

	result_type
	sigma() const
	{ return _M_sigma; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_nu == __p2._M_nu && __p1._M_sigma == __p2._M_sigma; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void _M_initialize();

	result_type _M_nu;
	result_type _M_sigma;
      };

      /**
       * @brief Constructors.
       * @{
       */

      rice_distribution() : rice_distribution(0) { }

      explicit
      rice_distribution(result_type __nu_val,
			result_type __sigma_val = result_type(1))
      : _M_param(__nu_val, __sigma_val),
	_M_ndx(__nu_val, __sigma_val),
	_M_ndy(result_type(0), __sigma_val)
      { }

      explicit
      rice_distribution(const param_type& __p)
      : _M_param(__p),
	_M_ndx(__p.nu(), __p.sigma()),
	_M_ndy(result_type(0), __p.sigma())
      { }

      /// @}

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      {
	_M_ndx.reset();
	_M_ndy.reset();
      }

      /**
       * @brief Return the parameters of the distribution.
       */
      result_type
      nu() const
      { return _M_param.nu(); }

      result_type
      sigma() const
      { return _M_param.sigma(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return result_type(0); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return std::numeric_limits<result_type>::max(); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{
	  result_type __x = this->_M_ndx(__urng);
	  result_type __y = this->_M_ndy(__urng);
#if _GLIBCXX_USE_C99_MATH_TR1
	  return std::hypot(__x, __y);
#else
	  return std::sqrt(__x * __x + __y * __y);
#endif
	}

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{
	  typename std::normal_distribution<result_type>::param_type
	    __px(__p.nu(), __p.sigma()), __py(result_type(0), __p.sigma());
	  result_type __x = this->_M_ndx(__px, __urng);
	  result_type __y = this->_M_ndy(__py, __urng);
#if _GLIBCXX_USE_C99_MATH_TR1
	  return std::hypot(__x, __y);
#else
	  return std::sqrt(__x * __x + __y * __y);
#endif
	}

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two Rice distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      friend bool
      operator==(const rice_distribution& __d1,
		 const rice_distribution& __d2)
      { return (__d1._M_param == __d2._M_param
		&& __d1._M_ndx == __d2._M_ndx
		&& __d1._M_ndy == __d2._M_ndy); }

      /**
       * @brief Inserts a %rice_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %rice_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>&,
		   const rice_distribution<_RealType1>&);

      /**
       * @brief Extracts a %rice_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x A %rice_distribution random number
       *            generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>&,
		   rice_distribution<_RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;

      std::normal_distribution<result_type> _M_ndx;
      std::normal_distribution<result_type> _M_ndy;
    };

  /**
   * @brief Return true if two Rice distributions are not equal.
   */
  template<typename _RealType1>
    inline bool
    operator!=(const rice_distribution<_RealType1>& __d1,
	       const rice_distribution<_RealType1>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A Nakagami continuous distribution for random numbers.
   *
   * The formula for the Nakagami probability density function is
   * @f[
   *     p(x|\mu,\omega) = \frac{2\mu^\mu}{\Gamma(\mu)\omega^\mu}
   *                       x^{2\mu-1}e^{-\mu x / \omega}
   * @f]
   * where @f$\Gamma(z)@f$ is the gamma function and @f$\mu >= 0.5@f$
   * and @f$\omega > 0@f$.
   */
  template<typename _RealType = double>
    class
    nakagami_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef nakagami_distribution<result_type> distribution_type;

	param_type() : param_type(1) { }

	param_type(result_type __mu_val,
		   result_type __omega_val = result_type(1))
	: _M_mu(__mu_val), _M_omega(__omega_val)
	{
	  __glibcxx_assert(_M_mu >= result_type(0.5L));
	  __glibcxx_assert(_M_omega > result_type(0));
	}

	result_type
	mu() const
	{ return _M_mu; }

	result_type
	omega() const
	{ return _M_omega; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_mu == __p2._M_mu && __p1._M_omega == __p2._M_omega; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void _M_initialize();

	result_type _M_mu;
	result_type _M_omega;
      };

      /**
       * @brief Constructors.
       * @{
       */

      nakagami_distribution() : nakagami_distribution(1) { }

      explicit
      nakagami_distribution(result_type __mu_val,
			    result_type __omega_val = result_type(1))
      : _M_param(__mu_val, __omega_val),
	_M_gd(__mu_val, __omega_val / __mu_val)
      { }

      explicit
      nakagami_distribution(const param_type& __p)
      : _M_param(__p),
	_M_gd(__p.mu(), __p.omega() / __p.mu())
      { }

      /// @}

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { _M_gd.reset(); }

      /**
       * @brief Return the parameters of the distribution.
       */
      result_type
      mu() const
      { return _M_param.mu(); }

      result_type
      omega() const
      { return _M_param.omega(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return result_type(0); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return std::numeric_limits<result_type>::max(); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return std::sqrt(this->_M_gd(__urng)); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{
	  typename std::gamma_distribution<result_type>::param_type
	    __pg(__p.mu(), __p.omega() / __p.mu());
	  return std::sqrt(this->_M_gd(__pg, __urng));
	}

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two Nakagami distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      friend bool
      operator==(const nakagami_distribution& __d1,
		 const nakagami_distribution& __d2)
      { return (__d1._M_param == __d2._M_param
		&& __d1._M_gd == __d2._M_gd); }

      /**
       * @brief Inserts a %nakagami_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %nakagami_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>&,
		   const nakagami_distribution<_RealType1>&);

      /**
       * @brief Extracts a %nakagami_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x A %nakagami_distribution random number
       *            generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>&,
		   nakagami_distribution<_RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;

      std::gamma_distribution<result_type> _M_gd;
    };

  /**
   * @brief Return true if two Nakagami distributions are not equal.
   */
  template<typename _RealType>
    inline bool
    operator!=(const nakagami_distribution<_RealType>& __d1,
	       const nakagami_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A Pareto continuous distribution for random numbers.
   *
   * The formula for the Pareto cumulative probability function is
   * @f[
   *     P(x|\alpha,\mu) = 1 - \left(\frac{\mu}{x}\right)^\alpha
   * @f]
   * The formula for the Pareto probability density function is
   * @f[
   *     p(x|\alpha,\mu) = \frac{\alpha + 1}{\mu}
   *                       \left(\frac{\mu}{x}\right)^{\alpha + 1}
   * @f]
   * where @f$x >= \mu@f$ and @f$\mu > 0@f$, @f$\alpha > 0@f$.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$\alpha \mu / (\alpha - 1)@f$
   *              for @f$\alpha > 1@f$</td></tr>
   * <tr><td>Variance</td><td>@f$\alpha \mu^2 / [(\alpha - 1)^2(\alpha - 2)]@f$
   *              for @f$\alpha > 2@f$</td></tr>
   * <tr><td>Range</td><td>@f$[\mu, \infty)@f$</td></tr>
   * </table>
   */
  template<typename _RealType = double>
    class
    pareto_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef pareto_distribution<result_type> distribution_type;

	param_type() : param_type(1) { }

	param_type(result_type __alpha_val,
		   result_type __mu_val = result_type(1))
	: _M_alpha(__alpha_val), _M_mu(__mu_val)
	{
	  __glibcxx_assert(_M_alpha > result_type(0));
	  __glibcxx_assert(_M_mu > result_type(0));
	}

	result_type
	alpha() const
	{ return _M_alpha; }

	result_type
	mu() const
	{ return _M_mu; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_alpha == __p2._M_alpha && __p1._M_mu == __p2._M_mu; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void _M_initialize();

	result_type _M_alpha;
	result_type _M_mu;
      };

      /**
       * @brief Constructors.
       * @{
       */

      pareto_distribution() : pareto_distribution(1) { }

      explicit
      pareto_distribution(result_type __alpha_val,
			  result_type __mu_val = result_type(1))
      : _M_param(__alpha_val, __mu_val),
	_M_ud()
      { }

      explicit
      pareto_distribution(const param_type& __p)
      : _M_param(__p),
	_M_ud()
      { }

      /// @}

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      {
	_M_ud.reset();
      }

      /**
       * @brief Return the parameters of the distribution.
       */
      result_type
      alpha() const
      { return _M_param.alpha(); }

      result_type
      mu() const
      { return _M_param.mu(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return this->mu(); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return std::numeric_limits<result_type>::max(); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{
	  return this->mu() * std::pow(this->_M_ud(__urng),
				       -result_type(1) / this->alpha());
	}

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{
	  return __p.mu() * std::pow(this->_M_ud(__urng),
					   -result_type(1) / __p.alpha());
	}

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two Pareto distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      friend bool
      operator==(const pareto_distribution& __d1,
		 const pareto_distribution& __d2)
      { return (__d1._M_param == __d2._M_param
		&& __d1._M_ud == __d2._M_ud); }

      /**
       * @brief Inserts a %pareto_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %pareto_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>&,
		   const pareto_distribution<_RealType1>&);

      /**
       * @brief Extracts a %pareto_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x A %pareto_distribution random number
       *            generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>&,
		   pareto_distribution<_RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;

      std::uniform_real_distribution<result_type> _M_ud;
    };

  /**
   * @brief Return true if two Pareto distributions are not equal.
   */
  template<typename _RealType>
    inline bool
    operator!=(const pareto_distribution<_RealType>& __d1,
	       const pareto_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A K continuous distribution for random numbers.
   *
   * The formula for the K probability density function is
   * @f[
   *     p(x|\lambda, \mu, \nu) = \frac{2}{x}
   *             \left(\frac{\lambda\nu x}{\mu}\right)^{\frac{\lambda + \nu}{2}}
   *             \frac{1}{\Gamma(\lambda)\Gamma(\nu)}
   *             K_{\nu - \lambda}\left(2\sqrt{\frac{\lambda\nu x}{\mu}}\right)
   * @f]
   * where @f$I_0(z)@f$ is the modified Bessel function of the second kind
   * of order @f$\nu - \lambda@f$ and @f$\lambda > 0@f$, @f$\mu > 0@f$
   * and @f$\nu > 0@f$.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$\mu@f$</td></tr>
   * <tr><td>Variance</td><td>@f$\mu^2\frac{\lambda + \nu + 1}{\lambda\nu}@f$</td></tr>
   * <tr><td>Range</td><td>@f$[0, \infty)@f$</td></tr>
   * </table>
   */
  template<typename _RealType = double>
    class
    k_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef k_distribution<result_type> distribution_type;

	param_type() : param_type(1) { }

	param_type(result_type __lambda_val,
		   result_type __mu_val = result_type(1),
		   result_type __nu_val = result_type(1))
	: _M_lambda(__lambda_val), _M_mu(__mu_val), _M_nu(__nu_val)
	{
	  __glibcxx_assert(_M_lambda > result_type(0));
	  __glibcxx_assert(_M_mu > result_type(0));
	  __glibcxx_assert(_M_nu > result_type(0));
	}

	result_type
	lambda() const
	{ return _M_lambda; }

	result_type
	mu() const
	{ return _M_mu; }

	result_type
	nu() const
	{ return _M_nu; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{
	  return __p1._M_lambda == __p2._M_lambda
	      && __p1._M_mu == __p2._M_mu
	      && __p1._M_nu == __p2._M_nu;
	}

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void _M_initialize();

	result_type _M_lambda;
	result_type _M_mu;
	result_type _M_nu;
      };

      /**
       * @brief Constructors.
       * @{
       */

      k_distribution() : k_distribution(1) { }

      explicit
      k_distribution(result_type __lambda_val,
		     result_type __mu_val = result_type(1),
		     result_type __nu_val = result_type(1))
      : _M_param(__lambda_val, __mu_val, __nu_val),
	_M_gd1(__lambda_val, result_type(1) / __lambda_val),
	_M_gd2(__nu_val, __mu_val / __nu_val)
      { }

      explicit
      k_distribution(const param_type& __p)
      : _M_param(__p),
	_M_gd1(__p.lambda(), result_type(1) / __p.lambda()),
	_M_gd2(__p.nu(), __p.mu() / __p.nu())
      { }

      /// @}

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      {
	_M_gd1.reset();
	_M_gd2.reset();
      }

      /**
       * @brief Return the parameters of the distribution.
       */
      result_type
      lambda() const
      { return _M_param.lambda(); }

      result_type
      mu() const
      { return _M_param.mu(); }

      result_type
      nu() const
      { return _M_param.nu(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return result_type(0); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return std::numeric_limits<result_type>::max(); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator&);

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator&, const param_type&);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two K distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      friend bool
      operator==(const k_distribution& __d1,
		 const k_distribution& __d2)
      { return (__d1._M_param == __d2._M_param
		&& __d1._M_gd1 == __d2._M_gd1
		&& __d1._M_gd2 == __d2._M_gd2); }

      /**
       * @brief Inserts a %k_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %k_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>&,
		   const k_distribution<_RealType1>&);

      /**
       * @brief Extracts a %k_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x A %k_distribution random number
       *            generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>&,
		   k_distribution<_RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;

      std::gamma_distribution<result_type> _M_gd1;
      std::gamma_distribution<result_type> _M_gd2;
    };

  /**
   * @brief Return true if two K distributions are not equal.
   */
  template<typename _RealType>
    inline bool
    operator!=(const k_distribution<_RealType>& __d1,
	       const k_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief An arcsine continuous distribution for random numbers.
   *
   * The formula for the arcsine probability density function is
   * @f[
   *     p(x|a,b) = \frac{1}{\pi \sqrt{(x - a)(b - x)}}
   * @f]
   * where @f$x >= a@f$ and @f$x <= b@f$.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$ (a + b) / 2 @f$</td></tr>
   * <tr><td>Variance</td><td>@f$ (b - a)^2 / 8 @f$</td></tr>
   * <tr><td>Range</td><td>@f$[a, b]@f$</td></tr>
   * </table>
   */
  template<typename _RealType = double>
    class
    arcsine_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef arcsine_distribution<result_type> distribution_type;

	param_type() : param_type(0) { }

	param_type(result_type __a, result_type __b = result_type(1))
	: _M_a(__a), _M_b(__b)
	{
	  __glibcxx_assert(_M_a <= _M_b);
	}

	result_type
	a() const
	{ return _M_a; }

	result_type
	b() const
	{ return _M_b; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_a == __p2._M_a && __p1._M_b == __p2._M_b; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void _M_initialize();

	result_type _M_a;
	result_type _M_b;
      };

      /**
       * @brief Constructors.
       * :{
       */

      arcsine_distribution() : arcsine_distribution(0) { }

      explicit
      arcsine_distribution(result_type __a, result_type __b = result_type(1))
      : _M_param(__a, __b),
	_M_ud(-1.5707963267948966192313216916397514L,
	      +1.5707963267948966192313216916397514L)
      { }

      explicit
      arcsine_distribution(const param_type& __p)
      : _M_param(__p),
	_M_ud(-1.5707963267948966192313216916397514L,
	      +1.5707963267948966192313216916397514L)
      { }

      /// @}

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { _M_ud.reset(); }

      /**
       * @brief Return the parameters of the distribution.
       */
      result_type
      a() const
      { return _M_param.a(); }

      result_type
      b() const
      { return _M_param.b(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return this->a(); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return this->b(); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{
	  result_type __x = std::sin(this->_M_ud(__urng));
	  return (__x * (this->b() - this->a())
		  + this->a() + this->b()) / result_type(2);
	}

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{
	  result_type __x = std::sin(this->_M_ud(__urng));
	  return (__x * (__p.b() - __p.a())
		  + __p.a() + __p.b()) / result_type(2);
	}

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two arcsine distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      friend bool
      operator==(const arcsine_distribution& __d1,
		 const arcsine_distribution& __d2)
      { return (__d1._M_param == __d2._M_param
		&& __d1._M_ud == __d2._M_ud); }

      /**
       * @brief Inserts a %arcsine_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %arcsine_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>&,
		   const arcsine_distribution<_RealType1>&);

      /**
       * @brief Extracts a %arcsine_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x A %arcsine_distribution random number
       *            generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>&,
		   arcsine_distribution<_RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;

      std::uniform_real_distribution<result_type> _M_ud;
    };

  /**
   * @brief Return true if two arcsine distributions are not equal.
   */
  template<typename _RealType>
    inline bool
    operator!=(const arcsine_distribution<_RealType>& __d1,
	       const arcsine_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A Hoyt continuous distribution for random numbers.
   *
   * The formula for the Hoyt probability density function is
   * @f[
   *     p(x|q,\omega) = \frac{(1 + q^2)x}{q\omega}
   *                     \exp\left(-\frac{(1 + q^2)^2 x^2}{4 q^2 \omega}\right)
   *                       I_0\left(\frac{(1 - q^4) x^2}{4 q^2 \omega}\right)
   * @f]
   * where @f$I_0(z)@f$ is the modified Bessel function of the first kind
   * of order 0 and @f$0 < q < 1@f$.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$ \sqrt{\frac{2}{\pi}} \sqrt{\frac{\omega}{1 + q^2}}
   *                       E(1 - q^2) @f$</td></tr>
   * <tr><td>Variance</td><td>@f$ \omega \left(1 - \frac{2E^2(1 - q^2)}
   *                                      {\pi (1 + q^2)}\right) @f$</td></tr>
   * <tr><td>Range</td><td>@f$[0, \infty)@f$</td></tr>
   * </table>
   * where @f$E(x)@f$ is the elliptic function of the second kind.
   */
  template<typename _RealType = double>
    class
    hoyt_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef hoyt_distribution<result_type> distribution_type;

	param_type() : param_type(0.5) { }

	param_type(result_type __q, result_type __omega = result_type(1))
	: _M_q(__q), _M_omega(__omega)
	{
	  __glibcxx_assert(_M_q > result_type(0));
	  __glibcxx_assert(_M_q < result_type(1));
	}

	result_type
	q() const
	{ return _M_q; }

	result_type
	omega() const
	{ return _M_omega; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_q == __p2._M_q && __p1._M_omega == __p2._M_omega; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void _M_initialize();

	result_type _M_q;
	result_type _M_omega;
      };

      /**
       * @brief Constructors.
       * @{
       */

      hoyt_distribution() : hoyt_distribution(0.5) { }

      explicit
      hoyt_distribution(result_type __q, result_type __omega = result_type(1))
      : _M_param(__q, __omega),
	_M_ad(result_type(0.5L) * (result_type(1) + __q * __q),
	      result_type(0.5L) * (result_type(1) + __q * __q)
				/ (__q * __q)),
	_M_ed(result_type(1))
      { }

      explicit
      hoyt_distribution(const param_type& __p)
      : _M_param(__p),
	_M_ad(result_type(0.5L) * (result_type(1) + __p.q() * __p.q()),
	      result_type(0.5L) * (result_type(1) + __p.q() * __p.q())
				/ (__p.q() * __p.q())),
	_M_ed(result_type(1))
      { }

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      {
	_M_ad.reset();
	_M_ed.reset();
      }

      /**
       * @brief Return the parameters of the distribution.
       */
      result_type
      q() const
      { return _M_param.q(); }

      result_type
      omega() const
      { return _M_param.omega(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return result_type(0); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return std::numeric_limits<result_type>::max(); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng);

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two Hoyt distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      friend bool
      operator==(const hoyt_distribution& __d1,
		 const hoyt_distribution& __d2)
      { return (__d1._M_param == __d2._M_param
		&& __d1._M_ad == __d2._M_ad
		&& __d1._M_ed == __d2._M_ed); }

      /**
       * @brief Inserts a %hoyt_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %hoyt_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>&,
		   const hoyt_distribution<_RealType1>&);

      /**
       * @brief Extracts a %hoyt_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x A %hoyt_distribution random number
       *            generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>&,
		   hoyt_distribution<_RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;

      __gnu_cxx::arcsine_distribution<result_type> _M_ad;
      std::exponential_distribution<result_type> _M_ed;
    };

  /**
   * @brief Return true if two Hoyt distributions are not equal.
   */
  template<typename _RealType>
    inline bool
    operator!=(const hoyt_distribution<_RealType>& __d1,
	       const hoyt_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A triangular distribution for random numbers.
   *
   * The formula for the triangular probability density function is
   * @f[
   *                  / 0                          for x < a
   *     p(x|a,b,c) = | \frac{2(x-a)}{(c-a)(b-a)}  for a <= x <= b
   *                  | \frac{2(c-x)}{(c-a)(c-b)}  for b < x <= c
   *                  \ 0                          for c < x
   * @f]
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$ \frac{a+b+c}{2} @f$</td></tr>
   * <tr><td>Variance</td><td>@f$ \frac{a^2+b^2+c^2-ab-ac-bc}
   *                                   {18}@f$</td></tr>
   * <tr><td>Range</td><td>@f$[a, c]@f$</td></tr>
   * </table>
   */
  template<typename _RealType = double>
    class triangular_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	friend class triangular_distribution<_RealType>;

	param_type() : param_type(0) { }

	explicit
	param_type(_RealType __a,
		   _RealType __b = _RealType(0.5),
		   _RealType __c = _RealType(1))
	: _M_a(__a), _M_b(__b), _M_c(__c)
	{
	  __glibcxx_assert(_M_a <= _M_b);
	  __glibcxx_assert(_M_b <= _M_c);
	  __glibcxx_assert(_M_a < _M_c);

	  _M_r_ab = (_M_b - _M_a) / (_M_c - _M_a);
	  _M_f_ab_ac = (_M_b - _M_a) * (_M_c - _M_a);
	  _M_f_bc_ac = (_M_c - _M_b) * (_M_c - _M_a);
	}

	_RealType
	a() const
	{ return _M_a; }

	_RealType
	b() const
	{ return _M_b; }

	_RealType
	c() const
	{ return _M_c; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{
	  return (__p1._M_a == __p2._M_a && __p1._M_b == __p2._M_b
		  && __p1._M_c == __p2._M_c);
	}

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:

	_RealType _M_a;
	_RealType _M_b;
	_RealType _M_c;
	_RealType _M_r_ab;
	_RealType _M_f_ab_ac;
	_RealType _M_f_bc_ac;
      };

      triangular_distribution() : triangular_distribution(0.0) { }

      /**
       * @brief Constructs a triangle distribution with parameters
       * @f$ a @f$, @f$ b @f$ and @f$ c @f$.
       */
      explicit
      triangular_distribution(result_type __a,
			      result_type __b = result_type(0.5),
			      result_type __c = result_type(1))
      : _M_param(__a, __b, __c)
      { }

      explicit
      triangular_distribution(const param_type& __p)
      : _M_param(__p)
      { }

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { }

      /**
       * @brief Returns the @f$ a @f$ of the distribution.
       */
      result_type
      a() const
      { return _M_param.a(); }

      /**
       * @brief Returns the @f$ b @f$ of the distribution.
       */
      result_type
      b() const
      { return _M_param.b(); }

      /**
       * @brief Returns the @f$ c @f$ of the distribution.
       */
      result_type
      c() const
      { return _M_param.c(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return _M_param._M_a; }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return _M_param._M_c; }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, _M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{
	  std::__detail::_Adaptor<_UniformRandomNumberGenerator, result_type>
	    __aurng(__urng);
	  result_type __rnd = __aurng();
	  if (__rnd <= __p._M_r_ab)
	    return __p.a() + std::sqrt(__rnd * __p._M_f_ab_ac);
	  else
	    return __p.c() - std::sqrt((result_type(1) - __rnd)
				       * __p._M_f_bc_ac);
	}

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two triangle distributions have the same
       *        parameters and the sequences that would be generated
       *        are equal.
       */
      friend bool
      operator==(const triangular_distribution& __d1,
		 const triangular_distribution& __d2)
      { return __d1._M_param == __d2._M_param; }

      /**
       * @brief Inserts a %triangular_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %triangular_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const __gnu_cxx::triangular_distribution<_RealType1>& __x);

      /**
       * @brief Extracts a %triangular_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x  A %triangular_distribution random number generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::triangular_distribution<_RealType1>& __x);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
    };

  /**
   * @brief Return true if two triangle distributions are different.
   */
  template<typename _RealType>
    inline bool
    operator!=(const __gnu_cxx::triangular_distribution<_RealType>& __d1,
	       const __gnu_cxx::triangular_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A von Mises distribution for random numbers.
   *
   * The formula for the von Mises probability density function is
   * @f[
   *     p(x|\mu,\kappa) = \frac{e^{\kappa \cos(x-\mu)}}
   *                            {2\pi I_0(\kappa)}
   * @f]
   *
   * The generating functions use the method according to:
   *
   * D. J. Best and N. I. Fisher, 1979. "Efficient Simulation of the
   * von Mises Distribution", Journal of the Royal Statistical Society.
   * Series C (Applied Statistics), Vol. 28, No. 2, pp. 152-157.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$ \mu @f$</td></tr>
   * <tr><td>Variance</td><td>@f$ 1-I_1(\kappa)/I_0(\kappa) @f$</td></tr>
   * <tr><td>Range</td><td>@f$[-\pi, \pi]@f$</td></tr>
   * </table>
   */
  template<typename _RealType = double>
    class von_mises_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	friend class von_mises_distribution<_RealType>;

	param_type() : param_type(0) { }

	explicit
	param_type(_RealType __mu, _RealType __kappa = _RealType(1))
	: _M_mu(__mu), _M_kappa(__kappa)
	{
	  const _RealType __pi = __gnu_cxx::__math_constants<_RealType>::__pi;
	  __glibcxx_assert(_M_mu >= -__pi && _M_mu <= __pi);
	  __glibcxx_assert(_M_kappa >= _RealType(0));

	  auto __tau = std::sqrt(_RealType(4) * _M_kappa * _M_kappa
				 + _RealType(1)) + _RealType(1);
	  auto __rho = ((__tau - std::sqrt(_RealType(2) * __tau))
			/ (_RealType(2) * _M_kappa));
	  _M_r = (_RealType(1) + __rho * __rho) / (_RealType(2) * __rho);
	}

	_RealType
	mu() const
	{ return _M_mu; }

	_RealType
	kappa() const
	{ return _M_kappa; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_mu == __p2._M_mu && __p1._M_kappa == __p2._M_kappa; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	_RealType _M_mu;
	_RealType _M_kappa;
	_RealType _M_r;
      };

      von_mises_distribution() : von_mises_distribution(0.0) { }

      /**
       * @brief Constructs a von Mises distribution with parameters
       * @f$\mu@f$ and @f$\kappa@f$.
       */
      explicit
      von_mises_distribution(result_type __mu,
			     result_type __kappa = result_type(1))
      : _M_param(__mu, __kappa)
      { }

      explicit
      von_mises_distribution(const param_type& __p)
      : _M_param(__p)
      { }

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { }

      /**
       * @brief Returns the @f$ \mu @f$ of the distribution.
       */
      result_type
      mu() const
      { return _M_param.mu(); }

      /**
       * @brief Returns the @f$ \kappa @f$ of the distribution.
       */
      result_type
      kappa() const
      { return _M_param.kappa(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      {
	return -__gnu_cxx::__math_constants<result_type>::__pi;
      }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      {
	return __gnu_cxx::__math_constants<result_type>::__pi;
      }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, _M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, _M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two von Mises distributions have the same
       *        parameters and the sequences that would be generated
       *        are equal.
       */
      friend bool
      operator==(const von_mises_distribution& __d1,
		 const von_mises_distribution& __d2)
      { return __d1._M_param == __d2._M_param; }

      /**
       * @brief Inserts a %von_mises_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %von_mises_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const __gnu_cxx::von_mises_distribution<_RealType1>& __x);

      /**
       * @brief Extracts a %von_mises_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x  A %von_mises_distribution random number generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::von_mises_distribution<_RealType1>& __x);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
    };

  /**
   * @brief Return true if two von Mises distributions are different.
   */
  template<typename _RealType>
    inline bool
    operator!=(const __gnu_cxx::von_mises_distribution<_RealType>& __d1,
	       const __gnu_cxx::von_mises_distribution<_RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A discrete hypergeometric random number distribution.
   *
   * The hypergeometric distribution is a discrete probability distribution
   * that describes the probability of @p k successes in @p n draws @a without
   * replacement from a finite population of size @p N containing exactly @p K
   * successes.
   *
   * The formula for the hypergeometric probability density function is
   * @f[
   *   p(k|N,K,n) = \frac{\binom{K}{k} \binom{N-K}{n-k}}{\binom{N}{n}}
   * @f]
   * where @f$N@f$ is the total population of the distribution,
   * @f$K@f$ is the total population of the distribution.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$ n\frac{K}{N} @f$</td></tr>
   * <tr><td>Variance</td><td>@f$ n\frac{K}{N}\frac{N-K}{N}\frac{N-n}{N-1}
   *   @f$</td></tr>
   * <tr><td>Range</td><td>@f$[max(0, n+K-N), min(K, n)]@f$</td></tr>
   * </table>
   */
  template<typename _UIntType = unsigned int>
    class hypergeometric_distribution
    {
      static_assert(std::is_unsigned<_UIntType>::value, "template argument "
		    "substituting _UIntType not an unsigned integral type");

    public:
      /** The type of the range of the distribution. */
      typedef _UIntType  result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef hypergeometric_distribution<_UIntType> distribution_type;
	friend class hypergeometric_distribution<_UIntType>;

	param_type() : param_type(10) { }

	explicit
	param_type(result_type __N, result_type __K = 5,
		   result_type __n = 1)
	: _M_N{__N}, _M_K{__K}, _M_n{__n}
	{
	  __glibcxx_assert(_M_N >= _M_K);
	  __glibcxx_assert(_M_N >= _M_n);
	}

	result_type
	total_size() const
	{ return _M_N; }

	result_type
	successful_size() const
	{ return _M_K; }

	result_type
	unsuccessful_size() const
	{ return _M_N - _M_K; }

	result_type
	total_draws() const
	{ return _M_n; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return (__p1._M_N == __p2._M_N)
	      && (__p1._M_K == __p2._M_K)
	      && (__p1._M_n == __p2._M_n); }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:

	result_type _M_N;
	result_type _M_K;
	result_type _M_n;
      };

      // constructors and member functions

      hypergeometric_distribution() : hypergeometric_distribution(10) { }

      explicit
      hypergeometric_distribution(result_type __N, result_type __K = 5,
				  result_type __n = 1)
      : _M_param{__N, __K, __n}
      { }

      explicit
      hypergeometric_distribution(const param_type& __p)
      : _M_param{__p}
      { }

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { }

      /**
       * @brief Returns the distribution parameter @p N,
       *	the total number of items.
       */
      result_type
      total_size() const
      { return this->_M_param.total_size(); }

      /**
       * @brief Returns the distribution parameter @p K,
       *	the total number of successful items.
       */
      result_type
      successful_size() const
      { return this->_M_param.successful_size(); }

      /**
       * @brief Returns the total number of unsuccessful items @f$ N - K @f$.
       */
      result_type
      unsuccessful_size() const
      { return this->_M_param.unsuccessful_size(); }

      /**
       * @brief Returns the distribution parameter @p n,
       *	the total number of draws.
       */
      result_type
      total_draws() const
      { return this->_M_param.total_draws(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return this->_M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { this->_M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      {
	using _IntType = typename std::make_signed<result_type>::type;
	return static_cast<result_type>(std::max(static_cast<_IntType>(0),
				static_cast<_IntType>(this->total_draws()
						- this->unsuccessful_size())));
      }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return std::min(this->successful_size(), this->total_draws()); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, this->_M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, this->_M_param); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

       /**
	* @brief Return true if two hypergeometric distributions have the same
	*        parameters and the sequences that would be generated
	*        are equal.
	*/
      friend bool
      operator==(const hypergeometric_distribution& __d1,
		 const hypergeometric_distribution& __d2)
      { return __d1._M_param == __d2._M_param; }

      /**
       * @brief Inserts a %hypergeometric_distribution random number
       * distribution @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %hypergeometric_distribution random number
       *             distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _UIntType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const __gnu_cxx::hypergeometric_distribution<_UIntType1>&
		   __x);

      /**
       * @brief Extracts a %hypergeometric_distribution random number
       * distribution @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x  A %hypergeometric_distribution random number generator
       *             distribution.
       *
       * @returns The input stream with @p __x extracted or in an error
       *          state.
       */
      template<typename _UIntType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::hypergeometric_distribution<_UIntType1>& __x);

    private:

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
    };

  /**
   * @brief Return true if two hypergeometric distributions are different.
   */
  template<typename _UIntType>
    inline bool
    operator!=(const __gnu_cxx::hypergeometric_distribution<_UIntType>& __d1,
	       const __gnu_cxx::hypergeometric_distribution<_UIntType>& __d2)
    { return !(__d1 == __d2); }

  /**
   * @brief A logistic continuous distribution for random numbers.
   *
   * The formula for the logistic probability density function is
   * @f[
   *     p(x|\a,\b) = \frac{e^{(x - a)/b}}{b[1 + e^{(x - a)/b}]^2}
   * @f]
   * where @f$b > 0@f$.
   *
   * The formula for the logistic probability function is
   * @f[
   *     cdf(x|\a,\b) = \frac{e^{(x - a)/b}}{1 + e^{(x - a)/b}}
   * @f]
   * where @f$b > 0@f$.
   *
   * <table border=1 cellpadding=10 cellspacing=0>
   * <caption align=top>Distribution Statistics</caption>
   * <tr><td>Mean</td><td>@f$a@f$</td></tr>
   * <tr><td>Variance</td><td>@f$b^2\pi^2/3@f$</td></tr>
   * <tr><td>Range</td><td>@f$[0, \infty)@f$</td></tr>
   * </table>
   */
  template<typename _RealType = double>
    class
    logistic_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");

    public:
      /** The type of the range of the distribution. */
      typedef _RealType result_type;

      /** Parameter type. */
      struct param_type
      {
	typedef logistic_distribution<result_type> distribution_type;

	param_type() : param_type(0) { }

	explicit
	param_type(result_type __a, result_type __b = result_type(1))
	: _M_a(__a), _M_b(__b)
	{
	  __glibcxx_assert(_M_b > result_type(0));
	}

	result_type
	a() const
	{ return _M_a; }

	result_type
	b() const
	{ return _M_b; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_a == __p2._M_a && __p1._M_b == __p2._M_b; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	void _M_initialize();

	result_type _M_a;
	result_type _M_b;
      };

      /**
       * @brief Constructors.
       * @{
       */
      logistic_distribution() : logistic_distribution(0.0) { }

      explicit
      logistic_distribution(result_type __a, result_type __b = result_type(1))
      : _M_param(__a, __b)
      { }

      explicit
      logistic_distribution(const param_type& __p)
      : _M_param(__p)
      { }

      /// @}

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { }

      /**
       * @brief Return the parameters of the distribution.
       */
      result_type
      a() const
      { return _M_param.a(); }

      result_type
      b() const
      { return _M_param.b(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       */
      result_type
      min() const
      { return -std::numeric_limits<result_type>::max(); }

      /**
       * @brief Returns the least upper bound value of the distribution.
       */
      result_type
      max() const
      { return std::numeric_limits<result_type>::max(); }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, this->_M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator&,
		   const param_type&);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, this->param()); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two logistic distributions have
       *        the same parameters and the sequences that would
       *        be generated are equal.
       */
      template<typename _RealType1>
	friend bool
	operator==(const logistic_distribution<_RealType1>& __d1,
		   const logistic_distribution<_RealType1>& __d2)
	{ return __d1.param() == __d2.param(); }

      /**
       * @brief Inserts a %logistic_distribution random number distribution
       * @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %logistic_distribution random number distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>&,
		   const logistic_distribution<_RealType1>&);

      /**
       * @brief Extracts a %logistic_distribution random number distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x A %logistic_distribution random number
       *            generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<typename _RealType1, typename _CharT, typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>&,
		   logistic_distribution<_RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
    };

  /**
   * @brief Return true if two logistic distributions are not equal.
   */
  template<typename _RealType1>
    inline bool
    operator!=(const logistic_distribution<_RealType1>& __d1,
	       const logistic_distribution<_RealType1>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A distribution for random coordinates on a unit sphere.
   *
   * The method used in the generation function is attributed by Donald Knuth
   * to G. W. Brown, Modern Mathematics for the Engineer (1956).
   */
  template<std::size_t _Dimen, typename _RealType = double>
    class uniform_on_sphere_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");
      static_assert(_Dimen != 0, "dimension is zero");

    public:
      /** The type of the range of the distribution. */
      typedef std::array<_RealType, _Dimen> result_type;

      /** Parameter type. */
      struct param_type
      {
	param_type() { }

	friend bool
	operator==(const param_type&, const param_type&)
	{ return true; }

	friend bool
	operator!=(const param_type&, const param_type&)
	{ return false; }
      };

      /**
       * @brief Constructs a uniform on sphere distribution.
       */
      uniform_on_sphere_distribution()
      : _M_param(), _M_nd()
      { }

      explicit
      uniform_on_sphere_distribution(const param_type& __p)
      : _M_param(__p), _M_nd()
      { }

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { _M_nd.reset(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       * This function makes no sense for this distribution.
       */
      result_type
      min() const
      {
	result_type __res;
	__res.fill(0);
	return __res;
      }

      /**
       * @brief Returns the least upper bound value of the distribution.
       * This function makes no sense for this distribution.
       */
      result_type
      max() const
      {
	result_type __res;
	__res.fill(0);
	return __res;
      }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, _M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, this->param()); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two uniform on sphere distributions have
       *        the same parameters and the sequences that would be
       *        generated are equal.
       */
      friend bool
      operator==(const uniform_on_sphere_distribution& __d1,
		 const uniform_on_sphere_distribution& __d2)
      { return __d1._M_nd == __d2._M_nd; }

      /**
       * @brief Inserts a %uniform_on_sphere_distribution random number
       *        distribution @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %uniform_on_sphere_distribution random number
       *             distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<size_t _Dimen1, typename _RealType1, typename _CharT,
	       typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const __gnu_cxx::uniform_on_sphere_distribution<_Dimen1,
								   _RealType1>&
		   __x);

      /**
       * @brief Extracts a %uniform_on_sphere_distribution random number
       *        distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x  A %uniform_on_sphere_distribution random number
       *             generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<std::size_t _Dimen1, typename _RealType1, typename _CharT,
	       typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::uniform_on_sphere_distribution<_Dimen1,
							     _RealType1>& __x);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
      std::normal_distribution<_RealType> _M_nd;
    };

  /**
   * @brief Return true if two uniform on sphere distributions are different.
   */
  template<std::size_t _Dimen, typename _RealType>
    inline bool
    operator!=(const __gnu_cxx::uniform_on_sphere_distribution<_Dimen,
	       _RealType>& __d1,
	       const __gnu_cxx::uniform_on_sphere_distribution<_Dimen,
	       _RealType>& __d2)
    { return !(__d1 == __d2); }


  /**
   * @brief A distribution for random coordinates inside a unit sphere.
   */
  template<std::size_t _Dimen, typename _RealType = double>
    class uniform_inside_sphere_distribution
    {
      static_assert(std::is_floating_point<_RealType>::value,
		    "template argument not a floating point type");
      static_assert(_Dimen != 0, "dimension is zero");

    public:
      /** The type of the range of the distribution. */
      using result_type = std::array<_RealType, _Dimen>;

      /** Parameter type. */
      struct param_type
      {
	using distribution_type
	  = uniform_inside_sphere_distribution<_Dimen, _RealType>;
	friend class uniform_inside_sphere_distribution<_Dimen, _RealType>;

	param_type() : param_type(1.0) { }

	explicit
	param_type(_RealType __radius)
	: _M_radius(__radius)
	{
	  __glibcxx_assert(_M_radius > _RealType(0));
	}

	_RealType
	radius() const
	{ return _M_radius; }

	friend bool
	operator==(const param_type& __p1, const param_type& __p2)
	{ return __p1._M_radius == __p2._M_radius; }

	friend bool
	operator!=(const param_type& __p1, const param_type& __p2)
	{ return !(__p1 == __p2); }

      private:
	_RealType _M_radius;
      };

      /**
       * @brief Constructors.
       * @{
       */

      uniform_inside_sphere_distribution()
      : uniform_inside_sphere_distribution(1.0)
      { }

      explicit
      uniform_inside_sphere_distribution(_RealType __radius)
      : _M_param(__radius), _M_uosd()
      { }

      explicit
      uniform_inside_sphere_distribution(const param_type& __p)
      : _M_param(__p), _M_uosd()
      { }

      /// @}

      /**
       * @brief Resets the distribution state.
       */
      void
      reset()
      { _M_uosd.reset(); }

      /**
       * @brief Returns the @f$radius@f$ of the distribution.
       */
      _RealType
      radius() const
      { return _M_param.radius(); }

      /**
       * @brief Returns the parameter set of the distribution.
       */
      param_type
      param() const
      { return _M_param; }

      /**
       * @brief Sets the parameter set of the distribution.
       * @param __param The new parameter set of the distribution.
       */
      void
      param(const param_type& __param)
      { _M_param = __param; }

      /**
       * @brief Returns the greatest lower bound value of the distribution.
       * This function makes no sense for this distribution.
       */
      result_type
      min() const
      {
	result_type __res;
	__res.fill(0);
	return __res;
      }

      /**
       * @brief Returns the least upper bound value of the distribution.
       * This function makes no sense for this distribution.
       */
      result_type
      max() const
      {
	result_type __res;
	__res.fill(0);
	return __res;
      }

      /**
       * @brief Generating functions.
       */
      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng)
	{ return this->operator()(__urng, _M_param); }

      template<typename _UniformRandomNumberGenerator>
	result_type
	operator()(_UniformRandomNumberGenerator& __urng,
		   const param_type& __p);

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng)
	{ this->__generate(__f, __t, __urng, this->param()); }

      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate(_ForwardIterator __f, _ForwardIterator __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      template<typename _UniformRandomNumberGenerator>
	void
	__generate(result_type* __f, result_type* __t,
		   _UniformRandomNumberGenerator& __urng,
		   const param_type& __p)
	{ this->__generate_impl(__f, __t, __urng, __p); }

      /**
       * @brief Return true if two uniform on sphere distributions have
       *        the same parameters and the sequences that would be
       *        generated are equal.
       */
      friend bool
      operator==(const uniform_inside_sphere_distribution& __d1,
		 const uniform_inside_sphere_distribution& __d2)
      { return __d1._M_param == __d2._M_param && __d1._M_uosd == __d2._M_uosd; }

      /**
       * @brief Inserts a %uniform_inside_sphere_distribution random number
       *        distribution @p __x into the output stream @p __os.
       *
       * @param __os An output stream.
       * @param __x  A %uniform_inside_sphere_distribution random number
       *             distribution.
       *
       * @returns The output stream with the state of @p __x inserted or in
       * an error state.
       */
      template<size_t _Dimen1, typename _RealType1, typename _CharT,
	       typename _Traits>
	friend std::basic_ostream<_CharT, _Traits>&
	operator<<(std::basic_ostream<_CharT, _Traits>& __os,
		   const __gnu_cxx::uniform_inside_sphere_distribution<_Dimen1,
								   _RealType1>&
		   );

      /**
       * @brief Extracts a %uniform_inside_sphere_distribution random number
       *        distribution
       * @p __x from the input stream @p __is.
       *
       * @param __is An input stream.
       * @param __x  A %uniform_inside_sphere_distribution random number
       *             generator engine.
       *
       * @returns The input stream with @p __x extracted or in an error state.
       */
      template<std::size_t _Dimen1, typename _RealType1, typename _CharT,
	       typename _Traits>
	friend std::basic_istream<_CharT, _Traits>&
	operator>>(std::basic_istream<_CharT, _Traits>& __is,
		   __gnu_cxx::uniform_inside_sphere_distribution<_Dimen1,
								 _RealType1>&);

    private:
      template<typename _ForwardIterator,
	       typename _UniformRandomNumberGenerator>
	void
	__generate_impl(_ForwardIterator __f, _ForwardIterator __t,
			_UniformRandomNumberGenerator& __urng,
			const param_type& __p);

      param_type _M_param;
      uniform_on_sphere_distribution<_Dimen, _RealType> _M_uosd;
    };

  /**
   * @brief Return true if two uniform on sphere distributions are different.
   */
  template<std::size_t _Dimen, typename _RealType>
    inline bool
    operator!=(const __gnu_cxx::uniform_inside_sphere_distribution<_Dimen,
	       _RealType>& __d1,
	       const __gnu_cxx::uniform_inside_sphere_distribution<_Dimen,
	       _RealType>& __d2)
    { return !(__d1 == __d2); }

_GLIBCXX_END_NAMESPACE_VERSION
} // namespace __gnu_cxx

#include <ext/opt_random.h>
#include <ext/random.tcc>

#endif // _GLIBCXX_USE_C99_STDINT_TR1 && UINT32_C

#endif // C++11

#endif // _EXT_RANDOM
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  T                  X                  \                  `                  d                  h                  l            +      p            2      t            ;      x            =      |            F                  J                  K                  R                  8                  :                  <                  O                  4                  K                                                                                                                                                                                                                                                                               2	                  6	                  7	                  9	                  ;	                  @	                   
                  +
                  
                   
                  
                  
                                                      
                  T
                  f                           $                                 :                                        $I                                        Z                           $             Z      (                                   .f                   h                                                                              Љ                   `4                                             p            `                                                  8                    P                     .symtab .strtab .shstrtab .note.gnu.build-id .note.Linux .rela.text .rela.init.text .rela.text.unlikely .rela.exit.text .rela__mcount_loc .rodata.str1.1 .rodata.str1.8 .rela.smp_locks .rela.rodata .modinfo .rela__param .rela.retpoline_sites .rela.return_sites .orc_unwind .rela.orc_unwind_ip __versions .rela__bug_table .rela__jump_table .rela.data .rela.exit.data .rela.init.data .data.once .rela.gnu.linkonce.this_module .bss .comment .note.GNU-stack .BTF .gnu_debuglink                                                                                            @       $                              .                     d       <                              ?                            >                             :      @                    `W      0                    J                                                         E      @               xR     x       0                    Z                           f                             U      @               R           0                    n                     c                                    i      @               i     0       0   	                 ~                     o                                   y      @               i     
      0                          2                     k                                  2               p      	                                                 4      ,                                    @               t           0                                         `                                          @               u     @      0                                         p                                                       P     (                                    @               }     `       0                                         x                                         @               (~     `       0                                              X                                   @               ~           0                                             <                                                 7     (                                  @                    x      0                    %                    `K     #                              5                     o     0                              0     @                           0                    F                    Po                                   A     @               H     H       0                     X                    `o     /                              S     @                           0   "                 c                    q                                   ^     @               8            0   $                 s                    q                                   n     @               P            0   &                 ~                    q                                                       q                   @                    @               h     0       0   )                                     @u                                        0               @u     P                                                  u                                                         u     M                                                  L                                                         X            1                    	                                                                                                                0	*H
01
0	`He0	*H
1a0]080 10UDebian Secure Boot CA2(oe:B&C0	`He0
	*H
  \o!{qDawBBɥϺs"i(tmb)k3nƼt|w$mj-h
#PvώÞݲ&g?~sZy9AcڿB:#DnH5=n`Djl\Y2?Pgl.Q
W,4ЋW&]dámv%A2JE
0`^qSNȔx3߭h"÷6         ~Module signature appended~
 070701000A03FF000041ED000000000000000000000002688C084600000000000000CA0000000200000000000000000000004A00000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/broadcom/bnx2x 070701000A0400000081A4000000000000000000000001682F6DA7001A3883000000CA0000000200000000000000000000005300000000usr/lib/modules/6.1.0-37-amd64/kernel/drivers/net/ethernet/broadcom/bnx2x/bnx2x.ko    ELF          >                    (         @     @ 8 7          GNU ˼k2h"	3D2}        Linux                Linux   6.1.0-37-amd64      HL AZ  N  v$D  A
  A+
  A9tf  AH3  
  JHD  
  +
  	  A9tH(     fJ@rB    AH2AX  ff.     @          LGIH   HcHWH APGA uHG011fG OfOHG     HG(        f    HG Ӊ  Z  =  tF=  t?=?  v?-  "wHH P   H<%     HW         ==  %     ׸           H    H=    tcH0  N8H H=    tFP9uHVDHD8   uZ  E1N  v	DG2APD9u    1    f    Hr  @uqZ  H8  -N  v#0
    ,
    @     &
    R3Ƌ 
  $
  э0  @          H          ATUSHHx   HG uvO3ʃHHH H  AAfZ  Ntyl      HC HD Hk E A9    H[]A\    DT Al      HC DT HC HT  A9    H[]A\    4 HcHw 1>Ajff.         AWAVAUATUSHHo3<MII <HG EALA Dp  AAAAAAt<AEߩ     X  A=N  u:  HHC $X  EE   AA=N  tl       HC LD8fZ  NtU	     t&K3          D HHC @ HcHk U H[]A\A]A^A_    A=N  *l       HC LD8AX  AE-X  A
  rf    fZ  Nt	
   ul  $u_       tH8    1G3f D    @ƃA  @ HG 0l  $            f.         AVAULcATMIU1SB	  Hl  t     fCll[]A\A]A^    t@ŀ	  t	  fu  HiQH%묍l l        d   d   f.         USHHJ8%   tHH    H   1[]            AVAUATUSHEAA   HG EADt  l          E!DAEH[]A\A]A^    D      fZ  NHW Htb@Ή0 H 0 Z  N  v8HH    Hx H   @   l             HP H Hx H @u1Hp   HH   l              Hp   Hx   l              l      g        ff.     f    tn)AVAAUL,ATUHSHH0  D#Ax   H    H0  D⾀       H0  1Ҿx       L9u[]A\A]A^        @     
  tF  $  $  fF1        f.         HeH%(   HD$1LL$LD$D$    HL$HT$@tIH        ~
l$0HcЃD uD$HT$eH+%(   u,H    H        HcЃD u    D      AUATUSZ  H-N  vwO2   DDc3l          A=B  PHS *HC    ul          1[]A\A]    B嘣     A   ff.     f    D  fA 0t
           Hǀ	              HPH   eH%(   HD$H1H|$HHD$    HD$@    HH  HHD$   H$HHD$   HHD$    HT$HeH+%(   u	HP        f    S   HHPeH%(   HD$H1H|$HHD$    HD$@    HH  HHD$   H$D$           HT$HeH+%(   u
HP[        fD      HPH   eH%(   HD$H1H|$HD$    HD$@    HH  t$HHH$HD$   D$       HT$HeH+%(   u	HP        ff.         AWHAVAULpATUHSHPeH%(   HD$H1HHH)L$HI
D    H@8  E        A   A   AH|$1   HD$    HD$@    H@t$LHHH$HD$   D$	   D$DD$T$    Å        HD$HeH+%(      HP[]A\A]A^A_    DE1ہ   AE1E1A18DAExjD9DNANA9~	AAAÃA@B  tg@tD)щșDxtș)AADHʙAډ))HDп   뗺      A   A   DDA   ,        ff.          AWAVAUATUSHL
  T$M   I]H   HcHvM| AG   EgfE9g   K,dIHLI0  DE1Hڹ  D$H       HCHtwD$DKAGAHD9|HI]H|$EOuHSHtI0  HE1sH       HC    H    1H[]A\A]A^A_    ff.         Z  @-N  v	u   t
   H   T$H<$H<$T$u   HH    f.         U@1SHYt[]       H?u总t   tZ  N  v	u   tH߾   []fD      AU   ATLcUMi  SHH   L'eH%(   H$   1HHHl      A$  Hi  HCHH  H$Hl$A$     A   .HHDl$D$       u;A$  ID9~Nl      HHDl$D$	       tH$   eH+%(   u_HĐ   []A\A]    HHD$       uHHD$ D$	       uHHD$ D$           fD      AU   ATUHSH   eH%(   H$   1HHH
  HD$   D$   HD$   tPE1IcHHHi  HE   Hi  HEHH  H$    Aą    AD;
  rE1H$   eH+%(   uHĐ   D[]A\A]        fD      UH8  SHH0H(  eH%(   HD$(1H,$HD$    HD$    HD$    HD$     e       Hh      Hx  H    H  H    u.H(  Hh      HD$(eH+%(   u$H0[]       HH    y                ATUSHP  eH%(   HD$Hp     H    H0        H=    L  3       H=        t       LH    uLH    tPH	   1HHl      L$$Hl$HHD$       HHD$       1HT$HeH+%(   u HP[]A\    HH  8    )             SHH      H      H      H      Hǃ      [    fD      SHx  HHt Hx@    Hx      Hǃx      H  Ht-H0  H  E1   H       Hǃ      [    ff.         UH  E1  SH0  H   H       H  H    H=    `         Hx  Ht=    Hx  Hc    HE@Hx  H@ t	1[]        1Hx  H0  H  E1   H  H       1H      f         ATUSG3HAĉAq          H(  H      H    ,0 1HcHC HHC AHHH HC 0HHC d   d     HcHk DE Et
l      []A\    ff.         SHeH%(   HD$1d  H$    HD$    HG G3<@  9rM`  d  H$Z  T$-N  w@H    X  HHT$H    H    HD$eH+%(   u0H[    HG HD4 D$HW d  HD8 D$        USHHA8   tcHHHH}%   H9H>H9|DHi ʚ;HFHŅ    H   H   H    1[]        H           UHAUATSHHHP8t=H   I    I    L    IT$I$1[A\A]]        ff.          SHH            	    H	    N  vU   tLHHHPH  Hȋ f  HC8t1H	          H    [            UHp  SHH    Hǃp      HǃP              ƃ
   H@      1Hǃ@      []    ff.          AVAUATUS      IH	   =      HL	  
      H=            HH    A$	  A$  uHI$	  D0E   A$
      L  HD    AŅ    EAADE   A@    A$
      D   Et-1HcЃHHHt
H4HL
H   ;   rH    D[]A\A]A^    A$
      AA$
  t        AWAAVL	  AUEATDUSHH
      AE   LDD    @EDH  D    L    H[]A\A]A^A_             AWA1AVAUL	  ATDUSHHeH%(   HD$1fT$
      A   ELDD    @H  DLD$D    L    
      ul$HD$eH+%(   uH[]A\A]A^A_                 Hǀ	  1Ҿ       1        HHB8tzUS  H%   = P      HhH    D$   u
H[]    HX  H       H5    Hڿ           H[]        ff.     f    AWAVA   AUATUHSHDeH%(   HD$1H$    AAM1II HLD A   ueD    
u'   H}h       HEh    HD$eH+%(      H[]A\A]A^A_    HEuc܇ H H؇ ۺ H	HHк   H}8H    H}hHH$I    'Y     HH  ۺ H	H        ff.          SH
  @u'H0      J1	    [    H	      f         SHr  @u;    H  H(            H߾       H[            '  
      H  G3f  f4  (     f0  f8  ǇD     ǇL     Ƈ  Ǉ     Ƈ  Ǉ  '     H    H߾   [    H߾       H[        AUATUSH   tE1[]DA\A]    Z  H=N  9  -O  I    H    H    LFH    HFl      H0  L  HLH         l      H0  LLH          Aąl  H   
      H  H  ǅ      H0Hg    L`E1CTCЉH9    II
uA4$AT$AD$ȉt&E1I<BGf9    IL9uAD$dȉLD DHPpA    A
    9    @    A\$  ˉ    H  H[  H  HrAT$ʉt<    1H΋ɉHH9uA$  ˉ    H  H  H  HJAT$ʉt6<    HTHǋJHH	HJɉHH9uA\$  ˉ    H  Hy  H  AT$H~t"D1HffHI9uH~AD$ȉHH  AD$$ȉHFH  AD$,ȉHFH  AD$4ȉHFH  AD$LȉHFH  AD$TȉHFH  AD$<ȉHFH  AD$DȉHF  H  El$XADH    H8  HtpH  AT$\ʉHQHII#AC1
HHɉHJɉfHfxJɉfHfxD9rH      H      H      AH      DHǅ      []A\A]         ^I    H    f         HG H 1HO  Hw  HO  Hw  HO  Hw  HO  Hw   HW      ff.     f    AVAUL	  ATUSHH// SGI's rope class -*- C++ -*-

// Copyright (C) 2001-2022 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

/*
 * Copyright (c) 1997
 * Silicon Graphics Computer Systems, Inc.
 *
 * Permission to use, copy, modify, distribute and sell this software
 * and its documentation for any purpose is hereby granted without fee,
 * provided that the above copyright notice appear in all copies and
 * that both that copyright notice and this permission notice appear
 * in supporting documentation.  Silicon Graphics makes no
 * representations about the suitability of this software for any
 * purpose.  It is provided "as is" without express or implied warranty.
 */

/** @file ext/rope
 *  This file is a GNU extension to the Standard C++ Library (possibly
 *  containing extensions from the HP/SGI STL subset). 
 */

#ifndef _ROPE
#define _ROPE 1

#pragma GCC system_header

#include <algorithm>
#include <iosfwd>
#include <bits/stl_construct.h>
#include <bits/stl_uninitialized.h>
#include <bits/stl_function.h>
#include <bits/stl_numeric.h>
#include <bits/allocator.h>
#include <bits/gthr.h>
#include <ext/alloc_traits.h>
#include <tr1/functional>

# ifdef __GC
#   define __GC_CONST const
# else
#   define __GC_CONST   // constant except for deallocation
# endif

#include <ext/memory> // For uninitialized_copy_n

namespace __gnu_cxx _GLIBCXX_VISIBILITY(default)
{
_GLIBCXX_BEGIN_NAMESPACE_VERSION

  namespace __detail
  {
    enum { _S_max_rope_depth = 45 };
    enum _Tag {_S_leaf, _S_concat, _S_substringfn, _S_function};
  } // namespace __detail

  // See libstdc++/36832.
  template<typename _ForwardIterator, typename _Allocator>
    void
    _Destroy_const(_ForwardIterator __first,
		   _ForwardIterator __last, _Allocator __alloc)
    {
      for (; __first != __last; ++__first)
	__alloc.destroy(&*__first);
    }

  template<typename _ForwardIterator, typename _Tp>
    inline void
    _Destroy_const(_ForwardIterator __first,
		   _ForwardIterator __last, std::allocator<_Tp>)
    { std::_Destroy(__first, __last); }

  // The _S_eos function is used for those functions that
  // convert to/from C-like strings to detect the end of the string.
  
  // The end-of-C-string character.
  // This is what the draft standard says it should be.
  template <class _CharT>
    inline _CharT
    _S_eos(_CharT*)
    { return _CharT(); }

  // Test for basic character types.
  // For basic character types leaves having a trailing eos.
  template <class _CharT>
    inline bool
    _S_is_basic_char_type(_CharT*)
    { return false; }
  
  template <class _CharT>
    inline bool
    _S_is_one_byte_char_type(_CharT*)
    { return false; }

  inline bool
  _S_is_basic_char_type(char*)
  { return true; }
  
  inline bool
  _S_is_one_byte_char_type(char*)
  { return true; }
  
  inline bool
  _S_is_basic_char_type(wchar_t*)
  { return true; }

  // Store an eos iff _CharT is a basic character type.
  // Do not reference _S_eos if it isn't.
  template <class _CharT>
    inline void
    _S_cond_store_eos(_CharT&) { }

  inline void
  _S_cond_store_eos(char& __c)
  { __c = 0; }

  inline void
  _S_cond_store_eos(wchar_t& __c)
  { __c = 0; }

  // char_producers are logically functions that generate a section of
  // a string.  These can be converted to ropes.  The resulting rope
  // invokes the char_producer on demand.  This allows, for example,
  // files to be viewed as ropes without reading the entire file.
  template <class _CharT>
    class char_producer
    {
    public:
      virtual ~char_producer() { }

      virtual void
      operator()(std::size_t __start_pos, std::size_t __len,
		 _CharT* __buffer) = 0;
      // Buffer should really be an arbitrary output iterator.
      // That way we could flatten directly into an ostream, etc.
      // This is thoroughly impossible, since iterator types don't
      // have runtime descriptions.
    };

  // Sequence buffers:
  //
  // Sequence must provide an append operation that appends an
  // array to the sequence.  Sequence buffers are useful only if
  // appending an entire array is cheaper than appending element by element.
  // This is true for many string representations.
  // This should  perhaps inherit from ostream<sequence::value_type>
  // and be implemented correspondingly, so that they can be used
  // for formatted.  For the sake of portability, we don't do this yet.
  //
  // For now, sequence buffers behave as output iterators.  But they also
  // behave a little like basic_ostringstream<sequence::value_type> and a
  // little like containers.

// Ignore warnings about std::iterator.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"

  template<class _Sequence, std::size_t _Buf_sz = 100>
    class sequence_buffer
    : public std::iterator<std::output_iterator_tag, void, void, void, void>
    {
    public:
      typedef typename _Sequence::value_type value_type;
    protected:
      _Sequence* _M_prefix;
      value_type _M_buffer[_Buf_sz];
      std::size_t _M_buf_count;
    public:

      void
      flush()
      {
	_M_prefix->append(_M_buffer, _M_buffer + _M_buf_count);
	_M_buf_count = 0;
      }
      
      ~sequence_buffer()
      { flush(); }
      
      sequence_buffer()
      : _M_prefix(0), _M_buf_count(0) { }

      sequence_buffer(const sequence_buffer& __x)
      {
	_M_prefix = __x._M_prefix;
	_M_buf_count = __x._M_buf_count;
	std::copy(__x._M_buffer, __x._M_buffer + __x._M_buf_count, _M_buffer);
      }
      
      // Non-const "copy" modifies the parameter - yuck
      sequence_buffer(sequence_buffer& __x)
      {
	__x.flush();
	_M_prefix = __x._M_prefix;
	_M_buf_count = 0;
      }
      
      sequence_buffer(_Sequence& __s)
      : _M_prefix(&__s), _M_buf_count(0) { }
      
      // Non-const "copy" modifies the parameter - yuck
      sequence_buffer&
      operator=(sequence_buffer& __x)
      {
	__x.flush();
	_M_prefix = __x._M_prefix;
	_M_buf_count = 0;
	return *this;
      }

      sequence_buffer&
      operator=(const sequence_buffer& __x)
      {
	_M_prefix = __x._M_prefix;
	_M_buf_count = __x._M_buf_count;
	std::copy(__x._M_buffer, __x._M_buffer + __x._M_buf_count, _M_buffer);
	return *this;
      }

#if __cplusplus >= 201103L
      sequence_buffer(sequence_buffer&& __x) : sequence_buffer(__x) { }
      sequence_buffer& operator=(sequence_buffer&& __x) { return *this = __x; }
#endif

      void
      push_back(value_type __x)
      {
	if (_M_buf_count < _Buf_sz)
	  {
	    _M_buffer[_M_buf_count] = __x;
	    ++_M_buf_count;
	  }
	else
	  {
	    flush();
	    _M_buffer[0] = __x;
	    _M_buf_count = 1;
	  }
      }
      
      void
      append(value_type* __s, std::size_t __len)
      {
	if (__len + _M_buf_count <= _Buf_sz)
	  {
	    std::size_t __i = _M_buf_count;
	    for (std::size_t __j = 0; __j < __len; __i++, __j++)
	      _M_buffer[__i] = __s[__j];
	    _M_buf_count += __len;
	  }
	else if (0 == _M_buf_count)
	  _M_prefix->append(__s, __s + __len);
	else
	  {
	    flush();
	    append(__s, __len);
	  }
      }

      sequence_buffer&
      write(value_type* __s, std::size_t __len)
      {
	append(__s, __len);
	return *this;
      }
      
      sequence_buffer&
      put(value_type __x)
      {
	push_back(__x);
	return *this;
      }
      
      sequence_buffer&
      operator=(const value_type& __rhs)
      {
	push_back(__rhs);
	return *this;
      }
      
      sequence_buffer&
      operator*()
      { return *this; }
      
      sequence_buffer&
      operator++()
      { return *this; }
      
      sequence_buffer
      operator++(int)
      { return *this; }
    };
#pragma GCC diagnostic pop
  
  // The following should be treated as private, at least for now.
  template<class _CharT>
    class _Rope_char_consumer
    {
    public:
      // If we had member templates, these should not be virtual.
      // For now we need to use run-time parametrization where
      // compile-time would do.  Hence this should all be private
      // for now.
      // The symmetry with char_producer is accidental and temporary.
      virtual ~_Rope_char_consumer() { }
  
      virtual bool
      operator()(const _CharT* __buffer, std::size_t __len) = 0;
    };
  
  // First a lot of forward declarations.  The standard seems to require
  // much stricter "declaration before use" than many of the implementations
  // that preceded it.
  template<class _CharT, class _Alloc = std::allocator<_CharT> >
    class rope;
  
  template<class _CharT, class _Alloc>
    struct _Rope_RopeConcatenation;

  template<class _CharT, class _Alloc>
    struct _Rope_RopeLeaf;
  
  template<class _CharT, class _Alloc>
    struct _Rope_RopeFunction;
  
  template<class _CharT, class _Alloc>
    struct _Rope_RopeSubstring;
  
  template<class _CharT, class _Alloc>
    class _Rope_iterator;
  
  template<class _CharT, class _Alloc>
    class _Rope_const_iterator;
  
  template<class _CharT, class _Alloc>
    class _Rope_char_ref_proxy;
  
  template<class _CharT, class _Alloc>
    class _Rope_char_ptr_proxy;

  template<class _CharT, class _Alloc>
    bool
    operator==(const _Rope_char_ptr_proxy<_CharT, _Alloc>& __x,
	       const _Rope_char_ptr_proxy<_CharT, _Alloc>& __y);

  template<class _CharT, class _Alloc>
    _Rope_const_iterator<_CharT, _Alloc>
    operator-(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	      std::ptrdiff_t __n);

  template<class _CharT, class _Alloc>
    _Rope_const_iterator<_CharT, _Alloc>
    operator+(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	      std::ptrdiff_t __n);

  template<class _CharT, class _Alloc>
    _Rope_const_iterator<_CharT, _Alloc>
    operator+(std::ptrdiff_t __n,
	      const _Rope_const_iterator<_CharT, _Alloc>& __x);

  template<class _CharT, class _Alloc>
    bool
    operator==(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	       const _Rope_const_iterator<_CharT, _Alloc>& __y);

  template<class _CharT, class _Alloc>
    bool
    operator<(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	      const _Rope_const_iterator<_CharT, _Alloc>& __y);
  
  template<class _CharT, class _Alloc>
    std::ptrdiff_t
    operator-(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	      const _Rope_const_iterator<_CharT, _Alloc>& __y);

  template<class _CharT, class _Alloc>
    _Rope_iterator<_CharT, _Alloc>
    operator-(const _Rope_iterator<_CharT, _Alloc>& __x, std::ptrdiff_t __n);

  template<class _CharT, class _Alloc>
    _Rope_iterator<_CharT, _Alloc>
    operator+(const _Rope_iterator<_CharT, _Alloc>& __x, std::ptrdiff_t __n);

  template<class _CharT, class _Alloc>
    _Rope_iterator<_CharT, _Alloc>
    operator+(std::ptrdiff_t __n, const _Rope_iterator<_CharT, _Alloc>& __x);

  template<class _CharT, class _Alloc>
    bool
    operator==(const _Rope_iterator<_CharT, _Alloc>& __x,
	       const _Rope_iterator<_CharT, _Alloc>& __y);

  template<class _CharT, class _Alloc>
    bool
    operator<(const _Rope_iterator<_CharT, _Alloc>& __x,
	      const _Rope_iterator<_CharT, _Alloc>& __y);

  template<class _CharT, class _Alloc>
    std::ptrdiff_t
    operator-(const _Rope_iterator<_CharT, _Alloc>& __x,
	      const _Rope_iterator<_CharT, _Alloc>& __y);

  template<class _CharT, class _Alloc>
    rope<_CharT, _Alloc>
    operator+(const rope<_CharT, _Alloc>& __left,
	      const rope<_CharT, _Alloc>& __right);

  template<class _CharT, class _Alloc>
    rope<_CharT, _Alloc>
    operator+(const rope<_CharT, _Alloc>& __left, const _CharT* __right);

  template<class _CharT, class _Alloc>
    rope<_CharT, _Alloc>
    operator+(const rope<_CharT, _Alloc>& __left, _CharT __right);

  // Some helpers, so we can use power on ropes.
  // See below for why this isn't local to the implementation.

// Ignore warnings about std::binary_function.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
  // This uses a nonstandard refcount convention.
  // The result has refcount 0.
  template<class _CharT, class _Alloc>
    struct _Rope_Concat_fn
    : public std::binary_function<rope<_CharT, _Alloc>, rope<_CharT, _Alloc>,
				  rope<_CharT, _Alloc> >
    {
      rope<_CharT, _Alloc>
      operator()(const rope<_CharT, _Alloc>& __x,
		 const rope<_CharT, _Alloc>& __y)
      { return __x + __y; }
    };
#pragma GCC diagnostic pop

  template <class _CharT, class _Alloc>
    inline rope<_CharT, _Alloc>
    identity_element(_Rope_Concat_fn<_CharT, _Alloc>)
    { return rope<_CharT, _Alloc>(); }

  // Class _Refcount_Base provides a type, _RC_t, a data member,
  // _M_ref_count, and member functions _M_incr and _M_decr, which perform
  // atomic preincrement/predecrement.  The constructor initializes
  // _M_ref_count.
  struct _Refcount_Base
  {
    // The type _RC_t
    typedef std::size_t _RC_t;
    
    // The data member _M_ref_count
    _RC_t _M_ref_count;

    // Constructor
#ifdef __GTHREAD_MUTEX_INIT
    __gthread_mutex_t _M_ref_count_lock = __GTHREAD_MUTEX_INIT;
#else
    __gthread_mutex_t _M_ref_count_lock;
#endif

    _Refcount_Base(_RC_t __n) : _M_ref_count(__n)
    {
#ifndef __GTHREAD_MUTEX_INIT
#ifdef __GTHREAD_MUTEX_INIT_FUNCTION
      __GTHREAD_MUTEX_INIT_FUNCTION (&_M_ref_count_lock);
#else
#error __GTHREAD_MUTEX_INIT or __GTHREAD_MUTEX_INIT_FUNCTION should be defined by gthr.h abstraction layer, report problem to libstdc++@gcc.gnu.org.
#endif
#endif
    }

#ifndef __GTHREAD_MUTEX_INIT
    ~_Refcount_Base()
    { __gthread_mutex_destroy(&_M_ref_count_lock); }
#endif

    void
    _M_incr()
    {
      __gthread_mutex_lock(&_M_ref_count_lock);
      ++_M_ref_count;
      __gthread_mutex_unlock(&_M_ref_count_lock);
    }

    _RC_t
    _M_decr()
    {
      __gthread_mutex_lock(&_M_ref_count_lock);
      _RC_t __tmp = --_M_ref_count;
      __gthread_mutex_unlock(&_M_ref_count_lock);
      return __tmp;
    }
  };

  //
  // What follows should really be local to rope.  Unfortunately,
  // that doesn't work, since it makes it impossible to define generic
  // equality on rope iterators.  According to the draft standard, the
  // template parameters for such an equality operator cannot be inferred
  // from the occurrence of a member class as a parameter.
  // (SGI compilers in fact allow this, but the __result wouldn't be
  // portable.)
  // Similarly, some of the static member functions are member functions
  // only to avoid polluting the global namespace, and to circumvent
  // restrictions on type inference for template functions.
  //

  //
  // The internal data structure for representing a rope.  This is
  // private to the implementation.  A rope is really just a pointer
  // to one of these.
  //
  // A few basic functions for manipulating this data structure
  // are members of _RopeRep.  Most of the more complex algorithms
  // are implemented as rope members.
  //
  // Some of the static member functions of _RopeRep have identically
  // named functions in rope that simply invoke the _RopeRep versions.

#define __ROPE_DEFINE_ALLOCS(__a) \
        __ROPE_DEFINE_ALLOC(_CharT,_Data) /* character data */ \
        typedef _Rope_RopeConcatenation<_CharT,__a> __C; \
        __ROPE_DEFINE_ALLOC(__C,_C) \
        typedef _Rope_RopeLeaf<_CharT,__a> __L; \
        __ROPE_DEFINE_ALLOC(__L,_L) \
        typedef _Rope_RopeFunction<_CharT,__a> __F; \
        __ROPE_DEFINE_ALLOC(__F,_F) \
        typedef _Rope_RopeSubstring<_CharT,__a> __S; \
        __ROPE_DEFINE_ALLOC(__S,_S)

  //  Internal rope nodes potentially store a copy of the allocator
  //  instance used to allocate them.  This is mostly redundant.
  //  But the alternative would be to pass allocator instances around
  //  in some form to nearly all internal functions, since any pointer
  //  assignment may result in a zero reference count and thus require
  //  deallocation.

#define __STATIC_IF_SGI_ALLOC  /* not static */

  template <class _CharT, class _Alloc>
    struct _Rope_rep_base
    : public _Alloc
    {
      typedef std::size_t size_type;
      typedef _Alloc allocator_type;

      allocator_type
      get_allocator() const
      { return *static_cast<const _Alloc*>(this); }

      allocator_type&
      _M_get_allocator()
      { return *static_cast<_Alloc*>(this); }

      const allocator_type&
      _M_get_allocator() const
      { return *static_cast<const _Alloc*>(this); }

      _Rope_rep_base(size_type __size, const allocator_type&)
      : _M_size(__size) { }

      size_type _M_size;

# define __ROPE_DEFINE_ALLOC(_Tp, __name) \
        typedef typename \
          __alloc_traits<_Alloc>::template rebind<_Tp>::other __name##Alloc; \
        static _Tp* __name##_allocate(size_type __n) \
          { return __name##Alloc().allocate(__n); } \
        static void __name##_deallocate(_Tp *__p, size_type __n) \
          { __name##Alloc().deallocate(__p, __n); }
      __ROPE_DEFINE_ALLOCS(_Alloc)
# undef __ROPE_DEFINE_ALLOC
    };

  template<class _CharT, class _Alloc>
    struct _Rope_RopeRep
    : public _Rope_rep_base<_CharT, _Alloc>
# ifndef __GC
	     , _Refcount_Base
# endif
    {
    public:
      __detail::_Tag _M_tag:8;
      bool _M_is_balanced:8;
      unsigned char _M_depth;
      __GC_CONST _CharT* _M_c_string;
#ifdef __GTHREAD_MUTEX_INIT
      __gthread_mutex_t _M_c_string_lock = __GTHREAD_MUTEX_INIT;
#else
      __gthread_mutex_t _M_c_string_lock;
#endif
                        /* Flattened version of string, if needed.  */
                        /* typically 0.                             */
                        /* If it's not 0, then the memory is owned  */
                        /* by this node.                            */
                        /* In the case of a leaf, this may point to */
                        /* the same memory as the data field.       */
      typedef typename _Rope_rep_base<_CharT, _Alloc>::allocator_type
        allocator_type;
      typedef std::size_t size_type;

      using _Rope_rep_base<_CharT, _Alloc>::get_allocator;
      using _Rope_rep_base<_CharT, _Alloc>::_M_get_allocator;

      _Rope_RopeRep(__detail::_Tag __t, int __d, bool __b, size_type __size,
		    const allocator_type& __a)
      : _Rope_rep_base<_CharT, _Alloc>(__size, __a),
#ifndef __GC
	_Refcount_Base(1),
#endif
	_M_tag(__t), _M_is_balanced(__b), _M_depth(__d), _M_c_string(0)
#ifdef __GTHREAD_MUTEX_INIT
      { }
#else
      { __GTHREAD_MUTEX_INIT_FUNCTION (&_M_c_string_lock); }
      ~_Rope_RopeRep()
      { __gthread_mutex_destroy (&_M_c_string_lock); }
#endif
#ifdef __GC
      void
      _M_incr () { }
#endif
      static void
      _S_free_string(__GC_CONST _CharT*, size_type __len,
		     allocator_type& __a);
#define __STL_FREE_STRING(__s, __l, __a) _S_free_string(__s, __l, __a);
                        // Deallocate data section of a leaf.
                        // This shouldn't be a member function.
                        // But its hard to do anything else at the
                        // moment, because it's templatized w.r.t.
                        // an allocator.
                        // Does nothing if __GC is defined.
#ifndef __GC
      void _M_free_c_string();
      void _M_free_tree();
      // Deallocate t. Assumes t is not 0.
      void
      _M_unref_nonnil()
      {
	if (0 == _M_decr())
	  _M_free_tree();
      }

      void
      _M_ref_nonnil()
      { _M_incr(); }

      static void
      _S_unref(_Rope_RopeRep* __t)
      {
	if (0 != __t)
	  __t->_M_unref_nonnil();
      }

      static void
      _S_ref(_Rope_RopeRep* __t)
      {
	if (0 != __t)
	  __t->_M_incr();
      }
      
      static void
      _S_free_if_unref(_Rope_RopeRep* __t)
      {
	if (0 != __t && 0 == __t->_M_ref_count)
	  __t->_M_free_tree();
      }
#   else /* __GC */
      void _M_unref_nonnil() { }
      void _M_ref_nonnil() { }
      static void _S_unref(_Rope_RopeRep*) { }
      static void _S_ref(_Rope_RopeRep*) { }
      static void _S_free_if_unref(_Rope_RopeRep*) { }
#   endif
    protected:
      _Rope_RopeRep&
      operator=(const _Rope_RopeRep&);

      _Rope_RopeRep(const _Rope_RopeRep&);
    };

  template<class _CharT, class _Alloc>
    struct _Rope_RopeLeaf
    : public _Rope_RopeRep<_CharT, _Alloc>
    {
      typedef std::size_t size_type;
    public:
      // Apparently needed by VC++
      // The data fields of leaves are allocated with some
      // extra space, to accommodate future growth and for basic
      // character types, to hold a trailing eos character.
      enum { _S_alloc_granularity = 8 };
      
      static size_type
      _S_rounded_up_size(size_type __n)
      {
        size_type __size_with_eos;
	
        if (_S_is_basic_char_type((_CharT*)0))
	  __size_with_eos = __n + 1;
	else
	  __size_with_eos = __n;
#ifdef __GC
	return __size_with_eos;
#else
	// Allow slop for in-place expansion.
	return ((__size_with_eos + size_type(_S_alloc_granularity) - 1)
		&~ (size_type(_S_alloc_granularity) - 1));
#endif
      }
      __GC_CONST _CharT* _M_data; /* Not necessarily 0 terminated. */
                                  /* The allocated size is         */
                                  /* _S_rounded_up_size(size), except */
                                  /* in the GC case, in which it   */
                                  /* doesn't matter.               */
      typedef typename _Rope_rep_base<_CharT,_Alloc>::allocator_type
        allocator_type;

      _Rope_RopeLeaf(__GC_CONST _CharT* __d, size_type __size,
		     const allocator_type& __a)
      : _Rope_RopeRep<_CharT, _Alloc>(__detail::_S_leaf, 0, true,
				      __size, __a), _M_data(__d)
      {
        if (_S_is_basic_char_type((_CharT *)0))
	  {
            // already eos terminated.
            this->_M_c_string = __d;
	  }
      }
      // The constructor assumes that d has been allocated with
      // the proper allocator and the properly padded size.
      // In contrast, the destructor deallocates the data:
#ifndef __GC
      ~_Rope_RopeLeaf() throw()
      {
        if (_M_data != this->_M_c_string)
	  this->_M_free_c_string();
	
	this->__STL_FREE_STRING(_M_data, this->_M_size, this->_M_get_allocator());
      }
#endif
    protected:
      _Rope_RopeLeaf&
      operator=(const _Rope_RopeLeaf&);

      _Rope_RopeLeaf(const _Rope_RopeLeaf&);
    };

  template<class _CharT, class _Alloc>
    struct _Rope_RopeConcatenation
    : public _Rope_RopeRep<_CharT, _Alloc>
    {
    public:
      _Rope_RopeRep<_CharT, _Alloc>* _M_left;
      _Rope_RopeRep<_CharT, _Alloc>* _M_right;

      typedef typename _Rope_rep_base<_CharT, _Alloc>::allocator_type
        allocator_type;

      _Rope_RopeConcatenation(_Rope_RopeRep<_CharT, _Alloc>* __l,
			      _Rope_RopeRep<_CharT, _Alloc>* __r,
			      const allocator_type& __a)
	: _Rope_RopeRep<_CharT, _Alloc>(__detail::_S_concat,
				      std::max(__l->_M_depth,
					       __r->_M_depth) + 1,
				      false,
				      __l->_M_size + __r->_M_size, __a),
        _M_left(__l), _M_right(__r)
      { }
#ifndef __GC
      ~_Rope_RopeConcatenation() throw()
      {
	this->_M_free_c_string();
	_M_left->_M_unref_nonnil();
	_M_right->_M_unref_nonnil();
      }
#endif
    protected:
      _Rope_RopeConcatenation&
      operator=(const _Rope_RopeConcatenation&);
      
      _Rope_RopeConcatenation(const _Rope_RopeConcatenation&);
    };

  template<class _CharT, class _Alloc>
    struct _Rope_RopeFunction
    : public _Rope_RopeRep<_CharT, _Alloc>
    {
    public:
      char_producer<_CharT>* _M_fn;
#ifndef __GC
      bool _M_delete_when_done; // Char_producer is owned by the
                                // rope and should be explicitly
                                // deleted when the rope becomes
                                // inaccessible.
#else
      // In the GC case, we either register the rope for
      // finalization, or not.  Thus the field is unnecessary;
      // the information is stored in the collector data structures.
      // We do need a finalization procedure to be invoked by the
      // collector.
      static void
      _S_fn_finalization_proc(void * __tree, void *)
      { delete ((_Rope_RopeFunction *)__tree) -> _M_fn; }
#endif
    typedef typename _Rope_rep_base<_CharT, _Alloc>::allocator_type
      allocator_type;

      _Rope_RopeFunction(char_producer<_CharT>* __f, std::size_t __size,
                        bool __d, const allocator_type& __a)
      : _Rope_RopeRep<_CharT, _Alloc>(__detail::_S_function, 0, true, __size, __a)
	, _M_fn(__f)
#ifndef __GC
	, _M_delete_when_done(__d)
#endif
      {
#ifdef __GC
	if (__d)
	  {
	    GC_REGISTER_FINALIZER(this, _Rope_RopeFunction::
				  _S_fn_finalization_proc, 0, 0, 0);
	  }
#endif
      }
#ifndef __GC
      ~_Rope_RopeFunction() throw()
      {
	this->_M_free_c_string();
	if (_M_delete_when_done)
	  delete _M_fn;
      }
# endif
    protected:
      _Rope_RopeFunction&
      operator=(const _Rope_RopeFunction&);

      _Rope_RopeFunction(const _Rope_RopeFunction&);
    };
  // Substring results are usually represented using just
  // concatenation nodes.  But in the case of very long flat ropes
  // or ropes with a functional representation that isn't practical.
  // In that case, we represent the __result as a special case of
  // RopeFunction, whose char_producer points back to the rope itself.
  // In all cases except repeated substring operations and
  // deallocation, we treat the __result as a RopeFunction.
  template<class _CharT, class _Alloc>
    struct _Rope_RopeSubstring
    : public _Rope_RopeFunction<_CharT, _Alloc>,
      public char_producer<_CharT>
    {
      typedef std::size_t size_type;
    public:
      // XXX this whole class should be rewritten.
      _Rope_RopeRep<_CharT,_Alloc>* _M_base;      // not 0
      size_type _M_start;

      virtual void
      operator()(size_type __start_pos, size_type __req_len,
		 _CharT* __buffer)
      {
        switch(_M_base->_M_tag)
	  {
	  case __detail::_S_function:
	  case __detail::_S_substringfn:
	    {
	      char_producer<_CharT>* __fn =
		((_Rope_RopeFunction<_CharT,_Alloc>*)_M_base)->_M_fn;
	      (*__fn)(__start_pos + _M_start, __req_len, __buffer);
	    }
	    break;
	  case __detail::_S_leaf:
	    {
	      __GC_CONST _CharT* __s =
		((_Rope_RopeLeaf<_CharT,_Alloc>*)_M_base)->_M_data;
	      uninitialized_copy_n(__s + __start_pos + _M_start, __req_len,
				   __buffer);
	    }
	    break;
	  default:
	    break;
	  }
      }
      
      typedef typename _Rope_rep_base<_CharT, _Alloc>::allocator_type
        allocator_type;

      _Rope_RopeSubstring(_Rope_RopeRep<_CharT, _Alloc>* __b, size_type __s,
                          size_type __l, const allocator_type& __a)
      : _Rope_RopeFunction<_CharT, _Alloc>(this, __l, false, __a),
        char_producer<_CharT>(), _M_base(__b), _M_start(__s)
      {
#ifndef __GC
	_M_base->_M_ref_nonnil();
#endif
        this->_M_tag = __detail::_S_substringfn;
      }
    virtual ~_Rope_RopeSubstring() throw()
      {
#ifndef __GC
	_M_base->_M_unref_nonnil();
	// _M_free_c_string();  -- done by parent class
#endif
      }
    };

  // Self-destructing pointers to Rope_rep.
  // These are not conventional smart pointers.  Their
  // only purpose in life is to ensure that unref is called
  // on the pointer either at normal exit or if an exception
  // is raised.  It is the caller's responsibility to
  // adjust reference counts when these pointers are initialized
  // or assigned to.  (This convention significantly reduces
  // the number of potentially expensive reference count
  // updates.)
#ifndef __GC
  template<class _CharT, class _Alloc>
    struct _Rope_self_destruct_ptr
    {
      _Rope_RopeRep<_CharT, _Alloc>* _M_ptr;

      ~_Rope_self_destruct_ptr()
      { _Rope_RopeRep<_CharT, _Alloc>::_S_unref(_M_ptr); }
#if __cpp_exceptions
      _Rope_self_destruct_ptr() : _M_ptr(0) { }
#else
      _Rope_self_destruct_ptr() { }
#endif
      _Rope_self_destruct_ptr(_Rope_RopeRep<_CharT, _Alloc>* __p)
      : _M_ptr(__p) { }
    
      _Rope_RopeRep<_CharT, _Alloc>&
      operator*()
      { return *_M_ptr; }
    
      _Rope_RopeRep<_CharT, _Alloc>*
      operator->()
      { return _M_ptr; }
    
      operator _Rope_RopeRep<_CharT, _Alloc>*()
      { return _M_ptr; }
    
      _Rope_self_destruct_ptr&
      operator=(_Rope_RopeRep<_CharT, _Alloc>* __x)
      { _M_ptr = __x; return *this; }
    };
#endif

  // Dereferencing a nonconst iterator has to return something
  // that behaves almost like a reference.  It's not possible to
  // return an actual reference since assignment requires extra
  // work.  And we would get into the same problems as with the
  // CD2 version of basic_string.
  template<class _CharT, class _Alloc>
    class _Rope_char_ref_proxy
    {
      friend class rope<_CharT, _Alloc>;
      friend class _Rope_iterator<_CharT, _Alloc>;
      friend class _Rope_char_ptr_proxy<_CharT, _Alloc>;
#ifdef __GC
      typedef _Rope_RopeRep<_CharT, _Alloc>* _Self_destruct_ptr;
#else
      typedef _Rope_self_destruct_ptr<_CharT, _Alloc> _Self_destruct_ptr;
#endif
      typedef _Rope_RopeRep<_CharT, _Alloc> _RopeRep;
      typedef rope<_CharT, _Alloc> _My_rope;
      std::size_t _M_pos;
      _CharT _M_current;
      bool _M_current_valid;
      _My_rope* _M_root;     // The whole rope.
    public:
      _Rope_char_ref_proxy(_My_rope* __r, std::size_t __p)
      :  _M_pos(__p), _M_current(), _M_current_valid(false), _M_root(__r) { }

      _Rope_char_ref_proxy(const _Rope_char_ref_proxy& __x)
      : _M_pos(__x._M_pos), _M_current(__x._M_current), 
	_M_current_valid(false), _M_root(__x._M_root) { }

      // Don't preserve cache if the reference can outlive the
      // expression.  We claim that's not possible without calling
      // a copy constructor or generating reference to a proxy
      // reference.  We declare the latter to have undefined semantics.
      _Rope_char_ref_proxy(_My_rope* __r, std::size_t __p, _CharT __c)
      : _M_pos(__p), _M_current(__c), _M_current_valid(true), _M_root(__r) { }

      inline operator _CharT () const;

      _Rope_char_ref_proxy&
      operator=(_CharT __c);
    
      _Rope_char_ptr_proxy<_CharT, _Alloc> operator&() const;
      
      _Rope_char_ref_proxy&
      operator=(const _Rope_char_ref_proxy& __c)
      { return operator=((_CharT)__c); }
    };

  template<class _CharT, class __Alloc>
    inline void
    swap(_Rope_char_ref_proxy <_CharT, __Alloc > __a,
	 _Rope_char_ref_proxy <_CharT, __Alloc > __b)
    {
      _CharT __tmp = __a;
      __a = __b;
      __b = __tmp;
    }

  template<class _CharT, class _Alloc>
    class _Rope_char_ptr_proxy
    {
      // XXX this class should be rewritten.
      friend class _Rope_char_ref_proxy<_CharT, _Alloc>;
      std::size_t _M_pos;
      rope<_CharT,_Alloc>* _M_root;     // The whole rope.
    public:
      _Rope_char_ptr_proxy(const _Rope_char_ref_proxy<_CharT,_Alloc>& __x)
      : _M_pos(__x._M_pos), _M_root(__x._M_root) { }

      _Rope_char_ptr_proxy(const _Rope_char_ptr_proxy& __x)
      : _M_pos(__x._M_pos), _M_root(__x._M_root) { }

      _Rope_char_ptr_proxy() { }
      
      _Rope_char_ptr_proxy(_CharT* __x)
      : _M_root(0), _M_pos(0) { }

      _Rope_char_ptr_proxy&
      operator=(const _Rope_char_ptr_proxy& __x)
      {
        _M_pos = __x._M_pos;
        _M_root = __x._M_root;
        return *this;
      }

      template<class _CharT2, class _Alloc2>
        friend bool
        operator==(const _Rope_char_ptr_proxy<_CharT2, _Alloc2>& __x,
		   const _Rope_char_ptr_proxy<_CharT2, _Alloc2>& __y);

      _Rope_char_ref_proxy<_CharT, _Alloc> operator*() const
      { return _Rope_char_ref_proxy<_CharT, _Alloc>(_M_root, _M_pos); }
    };

  // Rope iterators:
  // Unlike in the C version, we cache only part of the stack
  // for rope iterators, since they must be efficiently copyable.
  // When we run out of cache, we have to reconstruct the iterator
  // value.
  // Pointers from iterators are not included in reference counts.
  // Iterators are assumed to be thread private.  Ropes can
  // be shared.
  
// Ignore warnings about std::iterator
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
  template<class _CharT, class _Alloc>
    class _Rope_iterator_base
    : public std::iterator<std::random_access_iterator_tag, _CharT>
    {
      friend class rope<_CharT, _Alloc>;
    public:
      typedef _Alloc _allocator_type; // used in _Rope_rotate, VC++ workaround
      typedef _Rope_RopeRep<_CharT, _Alloc> _RopeRep;
      // Borland doesn't want this to be protected.
    protected:
      enum { _S_path_cache_len = 4 }; // Must be <= 9.
      enum { _S_iterator_buf_len = 15 };
      std::size_t _M_current_pos;
      _RopeRep* _M_root;     // The whole rope.
      std::size_t _M_leaf_pos; // Starting position for current leaf
      __GC_CONST _CharT* _M_buf_start;
                             // Buffer possibly
                             // containing current char.
      __GC_CONST _CharT* _M_buf_ptr;
                             // Pointer to current char in buffer.
                             // != 0 ==> buffer valid.
      __GC_CONST _CharT* _M_buf_end;
                             // One past __last valid char in buffer.
      // What follows is the path cache.  We go out of our
      // way to make this compact.
      // Path_end contains the bottom section of the path from
      // the root to the current leaf.
      const _RopeRep* _M_path_end[_S_path_cache_len];
      int _M_leaf_index;     // Last valid __pos in path_end;
                             // _M_path_end[0] ... _M_path_end[leaf_index-1]
                             // point to concatenation nodes.
      unsigned char _M_path_directions;
                          // (path_directions >> __i) & 1 is 1
                          // iff we got from _M_path_end[leaf_index - __i - 1]
                          // to _M_path_end[leaf_index - __i] by going to the
                          // __right. Assumes path_cache_len <= 9.
      _CharT _M_tmp_buf[_S_iterator_buf_len];
                        // Short buffer for surrounding chars.
                        // This is useful primarily for
                        // RopeFunctions.  We put the buffer
                        // here to avoid locking in the
                        // multithreaded case.
      // The cached path is generally assumed to be valid
      // only if the buffer is valid.
      static void _S_setbuf(_Rope_iterator_base& __x);
                                        // Set buffer contents given
                                        // path cache.
      static void _S_setcache(_Rope_iterator_base& __x);
                                        // Set buffer contents and
                                        // path cache.
      static void _S_setcache_for_incr(_Rope_iterator_base& __x);
                                        // As above, but assumes path
                                        // cache is valid for previous posn.
      _Rope_iterator_base() { }

      _Rope_iterator_base(_RopeRep* __root, std::size_t __pos)
      : _M_current_pos(__pos), _M_root(__root), _M_buf_ptr(0) { }

      void _M_incr(std::size_t __n);
      void _M_decr(std::size_t __n);
    public:
      std::size_t
      index() const
      { return _M_current_pos; }
    
      _Rope_iterator_base(const _Rope_iterator_base& __x)
      {
        if (0 != __x._M_buf_ptr && __x._M_buf_start != __x._M_tmp_buf)
	  *this = __x;
	else
	  {
            _M_current_pos = __x._M_current_pos;
            _M_root = __x._M_root;
            _M_buf_ptr = 0;
	  }
      }
    };
#pragma GCC diagnostic pop

  template<class _CharT, class _Alloc>
    class _Rope_iterator;

  template<class _CharT, class _Alloc>
    class _Rope_const_iterator
    : public _Rope_iterator_base<_CharT, _Alloc>
    {
      friend class rope<_CharT, _Alloc>;
    protected:
      typedef _Rope_RopeRep<_CharT, _Alloc> _RopeRep;
      // The one from the base class may not be directly visible.
      _Rope_const_iterator(const _RopeRep* __root, std::size_t __pos)
      : _Rope_iterator_base<_CharT, _Alloc>(const_cast<_RopeRep*>(__root),
					    __pos)
                   // Only nonconst iterators modify root ref count
      { }
  public:
      typedef _CharT reference;   // Really a value.  Returning a reference
                                  // Would be a mess, since it would have
                                  // to be included in refcount.
      typedef const _CharT* pointer;

    public:
      _Rope_const_iterator() { }

      _Rope_const_iterator(const _Rope_const_iterator& __x)
      : _Rope_iterator_base<_CharT,_Alloc>(__x) { }

      _Rope_const_iterator(const _Rope_iterator<_CharT,_Alloc>& __x);
    
      _Rope_const_iterator(const rope<_CharT, _Alloc>& __r, std::size_t __pos)
      : _Rope_iterator_base<_CharT,_Alloc>(__r._M_tree_ptr, __pos) { }

      _Rope_const_iterator&
      operator=(const _Rope_const_iterator& __x)
      {
        if (0 != __x._M_buf_ptr && __x._M_buf_start != __x._M_tmp_buf)
	  *(static_cast<_Rope_iterator_base<_CharT, _Alloc>*>(this)) = __x;
	else
	  {
            this->_M_current_pos = __x._M_current_pos;
            this->_M_root = __x._M_root;
            this->_M_buf_ptr = 0;
	  }
        return(*this);
      }

      reference
      operator*()
      {
        if (0 == this->_M_buf_ptr)
	  this->_S_setcache(*this);
        return *this->_M_buf_ptr;
      }

      // Without this const version, Rope iterators do not meet the
      // requirements of an Input Iterator.
      reference
      operator*() const
      {
	return *const_cast<_Rope_const_iterator&>(*this);
      }

      _Rope_const_iterator&
      operator++()
      {
        __GC_CONST _CharT* __next;
        if (0 != this->_M_buf_ptr
	    && (__next = this->_M_buf_ptr + 1) < this->_M_buf_end)
	  {
            this->_M_buf_ptr = __next;
            ++this->_M_current_pos;
	  }
	else
	  this->_M_incr(1);
	return *this;
      }

      _Rope_const_iterator&
      operator+=(std::ptrdiff_t __n)
      {
        if (__n >= 0)
	  this->_M_incr(__n);
	else
	  this->_M_decr(-__n);
	return *this;
      }

      _Rope_const_iterator&
      operator--()
      {
        this->_M_decr(1);
        return *this;
      }

      _Rope_const_iterator&
      operator-=(std::ptrdiff_t __n)
      {
        if (__n >= 0)
	  this->_M_decr(__n);
	else
	  this->_M_incr(-__n);
	return *this;
      }

      _Rope_const_iterator
      operator++(int)
      {
	std::size_t __old_pos = this->_M_current_pos;
        this->_M_incr(1);
        return _Rope_const_iterator<_CharT,_Alloc>(this->_M_root, __old_pos);
        // This makes a subsequent dereference expensive.
        // Perhaps we should instead copy the iterator
        // if it has a valid cache?
      }

      _Rope_const_iterator
      operator--(int)
      {
	std::size_t __old_pos = this->_M_current_pos;
        this->_M_decr(1);
        return _Rope_const_iterator<_CharT,_Alloc>(this->_M_root, __old_pos);
      }

      template<class _CharT2, class _Alloc2>
        friend _Rope_const_iterator<_CharT2, _Alloc2>
        operator-(const _Rope_const_iterator<_CharT2, _Alloc2>& __x,
		  std::ptrdiff_t __n);

      template<class _CharT2, class _Alloc2>
        friend _Rope_const_iterator<_CharT2, _Alloc2>
        operator+(const _Rope_const_iterator<_CharT2, _Alloc2>& __x,
		  std::ptrdiff_t __n);

      template<class _CharT2, class _Alloc2>
        friend _Rope_const_iterator<_CharT2, _Alloc2>
        operator+(std::ptrdiff_t __n,
		  const _Rope_const_iterator<_CharT2, _Alloc2>& __x);

      reference
      operator[](std::size_t __n)
      { return rope<_CharT, _Alloc>::_S_fetch(this->_M_root,
					      this->_M_current_pos + __n); }

      template<class _CharT2, class _Alloc2>
        friend bool
        operator==(const _Rope_const_iterator<_CharT2, _Alloc2>& __x,
		   const _Rope_const_iterator<_CharT2, _Alloc2>& __y);

      template<class _CharT2, class _Alloc2>
        friend bool
        operator<(const _Rope_const_iterator<_CharT2, _Alloc2>& __x,
		  const _Rope_const_iterator<_CharT2, _Alloc2>& __y);

      template<class _CharT2, class _Alloc2>
        friend std::ptrdiff_t
        operator-(const _Rope_const_iterator<_CharT2, _Alloc2>& __x,
		  const _Rope_const_iterator<_CharT2, _Alloc2>& __y);
    };

  template<class _CharT, class _Alloc>
    class _Rope_iterator
    : public _Rope_iterator_base<_CharT, _Alloc>
    {
      friend class rope<_CharT, _Alloc>;
    protected:
      typedef typename _Rope_iterator_base<_CharT, _Alloc>::_RopeRep _RopeRep;
      rope<_CharT, _Alloc>* _M_root_rope;

      // root is treated as a cached version of this, and is used to
      // detect changes to the underlying rope.

      // Root is included in the reference count.  This is necessary
      // so that we can detect changes reliably.  Unfortunately, it
      // requires careful bookkeeping for the nonGC case.
      _Rope_iterator(rope<_CharT, _Alloc>* __r, std::size_t __pos)
      : _Rope_iterator_base<_CharT, _Alloc>(__r->_M_tree_ptr, __pos),
        _M_root_rope(__r)
      { _RopeRep::_S_ref(this->_M_root);
        if (!(__r -> empty()))
	  this->_S_setcache(*this);
      }

      void _M_check();
    public:
      typedef _Rope_char_ref_proxy<_CharT, _Alloc>  reference;
      typedef _Rope_char_ref_proxy<_CharT, _Alloc>* pointer;

      rope<_CharT, _Alloc>&
      container()
      { return *_M_root_rope; }

      _Rope_iterator()
      {
        this->_M_root = 0;  // Needed for reference counting.
      }

      _Rope_iterator(const _Rope_iterator& __x)
      : _Rope_iterator_base<_CharT, _Alloc>(__x)
      {
        _M_root_rope = __x._M_root_rope;
        _RopeRep::_S_ref(this->_M_root);
      }

      _Rope_iterator(rope<_CharT, _Alloc>& __r, std::size_t __pos);

      ~_Rope_iterator()
      { _RopeRep::_S_unref(this->_M_root); }

      _Rope_iterator&
      operator=(const _Rope_iterator& __x)
      {
        _RopeRep* __old = this->_M_root;
	
        _RopeRep::_S_ref(__x._M_root);
        if (0 != __x._M_buf_ptr && __x._M_buf_start != __x._M_tmp_buf)
	  {
            _M_root_rope = __x._M_root_rope;
            *(static_cast<_Rope_iterator_base<_CharT, _Alloc>*>(this)) = __x;
	  }
	else
	  {
	    this->_M_current_pos = __x._M_current_pos;
            this->_M_root = __x._M_root;
            _M_root_rope = __x._M_root_rope;
            this->_M_buf_ptr = 0;
	  }
        _RopeRep::_S_unref(__old);
        return(*this);
      }

      reference
      operator*()
      {
        _M_check();
        if (0 == this->_M_buf_ptr)
	  return _Rope_char_ref_proxy<_CharT, _Alloc>(_M_root_rope,
						      this->_M_current_pos);
	else
	  return _Rope_char_ref_proxy<_CharT, _Alloc>(_M_root_rope,
						      this->_M_current_pos,
						      *this->_M_buf_ptr);
      }

      // See above comment.
      reference
      operator*() const
      {
	return *const_cast<_Rope_iterator&>(*this);
      }

      _Rope_iterator&
      operator++()
      {
        this->_M_incr(1);
        return *this;
      }

      _Rope_iterator&
      operator+=(std::ptrdiff_t __n)
      {
        if (__n >= 0)
	  this->_M_incr(__n);
	else
	  this->_M_decr(-__n);
	return *this;
      }

      _Rope_iterator&
      operator--()
      {
        this->_M_decr(1);
        return *this;
      }

      _Rope_iterator&
      operator-=(std::ptrdiff_t __n)
      {
        if (__n >= 0)
	  this->_M_decr(__n);
	else
	  this->_M_incr(-__n);
	return *this;
      }

      _Rope_iterator
      operator++(int)
      {
	std::size_t __old_pos = this->_M_current_pos;
        this->_M_incr(1);
        return _Rope_iterator<_CharT,_Alloc>(_M_root_rope, __old_pos);
      }

      _Rope_iterator
      operator--(int)
      {
	std::size_t __old_pos = this->_M_current_pos;
        this->_M_decr(1);
        return _Rope_iterator<_CharT,_Alloc>(_M_root_rope, __old_pos);
      }

      reference
      operator[](std::ptrdiff_t __n)
      { return _Rope_char_ref_proxy<_CharT, _Alloc>(_M_root_rope,
						    this->_M_current_pos
						    + __n); }

      template<class _CharT2, class _Alloc2>
        friend bool
        operator==(const _Rope_iterator<_CharT2, _Alloc2>& __x,
		   const _Rope_iterator<_CharT2, _Alloc2>& __y);

      template<class _CharT2, class _Alloc2>
        friend bool
        operator<(const _Rope_iterator<_CharT2, _Alloc2>& __x,
		  const _Rope_iterator<_CharT2, _Alloc2>& __y);

      template<class _CharT2, class _Alloc2>
        friend std::ptrdiff_t
        operator-(const _Rope_iterator<_CharT2, _Alloc2>& __x,
		  const _Rope_iterator<_CharT2, _Alloc2>& __y);

      template<class _CharT2, class _Alloc2>
        friend _Rope_iterator<_CharT2, _Alloc2>
        operator-(const _Rope_iterator<_CharT2, _Alloc2>& __x,
		  std::ptrdiff_t __n);

      template<class _CharT2, class _Alloc2>
        friend _Rope_iterator<_CharT2, _Alloc2>
        operator+(const _Rope_iterator<_CharT2, _Alloc2>& __x,
		  std::ptrdiff_t __n);

      template<class _CharT2, class _Alloc2>
        friend _Rope_iterator<_CharT2, _Alloc2>
        operator+(std::ptrdiff_t __n,
		  const _Rope_iterator<_CharT2, _Alloc2>& __x);
    };


  template <class _CharT, class _Alloc>
    struct _Rope_base
    : public _Alloc
    {
      typedef _Alloc allocator_type;

      allocator_type
      get_allocator() const
      { return *static_cast<const _Alloc*>(this); }

      allocator_type&
      _M_get_allocator()
      { return *static_cast<_Alloc*>(this); }

      const allocator_type&
      _M_get_allocator() const
      { return *static_cast<const _Alloc*>(this); }

      typedef _Rope_RopeRep<_CharT, _Alloc> _RopeRep;
      // The one in _Base may not be visible due to template rules.

      _Rope_base(_RopeRep* __t, const allocator_type&)
      : _M_tree_ptr(__t) { }

      _Rope_base(const allocator_type&) { }

      // The only data member of a rope:
      _RopeRep *_M_tree_ptr;

#define __ROPE_DEFINE_ALLOC(_Tp, __name) \
        typedef typename \
          __alloc_traits<_Alloc>::template rebind<_Tp>::other __name##Alloc; \
        static _Tp* __name##_allocate(std::size_t __n) \
          { return __name##Alloc().allocate(__n); } \
        static void __name##_deallocate(_Tp *__p, std::size_t __n) \
          { __name##Alloc().deallocate(__p, __n); }
      __ROPE_DEFINE_ALLOCS(_Alloc)
#undef __ROPE_DEFINE_ALLOC

    protected:
      _Rope_base&
      operator=(const _Rope_base&);
      
      _Rope_base(const _Rope_base&);
    };

  /**
   *  This is an SGI extension.
   *  @ingroup SGIextensions
   *  @doctodo
   */
  template <class _CharT, class _Alloc>
    class rope : public _Rope_base<_CharT, _Alloc>
    {
    public:
      typedef _CharT value_type;
      typedef std::ptrdiff_t difference_type;
      typedef std::size_t size_type;
      typedef _CharT const_reference;
      typedef const _CharT* const_pointer;
      typedef _Rope_iterator<_CharT, _Alloc> iterator;
      typedef _Rope_const_iterator<_CharT, _Alloc> const_iterator;
      typedef _Rope_char_ref_proxy<_CharT, _Alloc> reference;
      typedef _Rope_char_ptr_proxy<_CharT, _Alloc> pointer;

      friend class _Rope_iterator<_CharT, _Alloc>;
      friend class _Rope_const_iterator<_CharT, _Alloc>;
      friend struct _Rope_RopeRep<_CharT, _Alloc>;
      friend class _Rope_iterator_base<_CharT, _Alloc>;
      friend class _Rope_char_ptr_proxy<_CharT, _Alloc>;
      friend class _Rope_char_ref_proxy<_CharT, _Alloc>;
      friend struct _Rope_RopeSubstring<_CharT, _Alloc>;

    protected:
      typedef _Rope_base<_CharT, _Alloc> _Base;
      typedef typename _Base::allocator_type allocator_type;
      using _Base::_M_tree_ptr;
      using _Base::get_allocator;
      using _Base::_M_get_allocator;
      typedef __GC_CONST _CharT* _Cstrptr;
      
      static _CharT _S_empty_c_str[1];
      
      static bool
      _S_is0(_CharT __c)
      { return __c == _S_eos((_CharT*)0); }
      
      enum { _S_copy_max = 23 };
                // For strings shorter than _S_copy_max, we copy to
                // concatenate.

      typedef _Rope_RopeRep<_CharT, _Alloc> _RopeRep;
      typedef _Rope_RopeConcatenation<_CharT, _Alloc> _RopeConcatenation;
      typedef _Rope_RopeLeaf<_CharT, _Alloc> _RopeLeaf;
      typedef _Rope_RopeFunction<_CharT, _Alloc> _RopeFunction;
      typedef _Rope_RopeSubstring<_CharT, _Alloc> _RopeSubstring;

      // Retrieve a character at the indicated position.
      static _CharT _S_fetch(_RopeRep* __r, size_type __pos);

#ifndef __GC
      // Obtain a pointer to the character at the indicated position.
      // The pointer can be used to change the character.
      // If such a pointer cannot be produced, as is frequently the
      // case, 0 is returned instead.
      // (Returns nonzero only if all nodes in the path have a refcount
      // of 1.)
      static _CharT* _S_fetch_ptr(_RopeRep* __r, size_type __pos);
#endif

      static bool
      _S_apply_to_pieces(// should be template parameter
			 _Rope_char_consumer<_CharT>& __c,
			 const _RopeRep* __r,
			 size_type __begin, size_type __end);
                         // begin and end are assumed to be in range.

#ifndef __GC
      static void
      _S_unref(_RopeRep* __t)
      { _RopeRep::_S_unref(__t); }

      static void
      _S_ref(_RopeRep* __t)
      { _RopeRep::_S_ref(__t); }

#else /* __GC */
      static void _S_unref(_RopeRep*) { }
      static void _S_ref(_RopeRep*) { }
#endif

#ifdef __GC
      typedef _Rope_RopeRep<_CharT, _Alloc>* _Self_destruct_ptr;
#else
      typedef _Rope_self_destruct_ptr<_CharT, _Alloc> _Self_destruct_ptr;
#endif

      // _Result is counted in refcount.
      static _RopeRep* _S_substring(_RopeRep* __base,
                                    size_type __start, size_type __endp1);

      static _RopeRep* _S_concat_char_iter(_RopeRep* __r,
					   const _CharT* __iter,
					   size_type __slen,
					   allocator_type& __a);
      // Concatenate rope and char ptr, copying __iter.
      // Should really take an arbitrary iterator.
      // Result is counted in refcount.
      static _RopeRep* _S_destr_concat_char_iter(_RopeRep* __r,
						 const _CharT* __iter,
						 size_type __slen,
						 allocator_type& __a)
	// As above, but one reference to __r is about to be
	// destroyed.  Thus the pieces may be recycled if all
	// relevant reference counts are 1.
#ifdef __GC
	// We can't really do anything since refcounts are unavailable.
      { return _S_concat_char_iter(__r, __iter, __slen, __a); }
#else
      ;
#endif

      static _RopeRep* _S_concat(_RopeRep* __left, _RopeRep* __right);
      // General concatenation on _RopeRep.  _Result
      // has refcount of 1.  Adjusts argument refcounts.

   public:
      void
      apply_to_pieces(size_type __begin, size_type __end,
		      _Rope_char_consumer<_CharT>& __c) const
      { _S_apply_to_pieces(__c, this->_M_tree_ptr, __begin, __end); }

   protected:

      static size_type
      _S_rounded_up_size(size_type __n)
      { return _RopeLeaf::_S_rounded_up_size(__n); }

      static size_type
      _S_allocated_capacity(size_type __n)
      {
	if (_S_is_basic_char_type((_CharT*)0))
	  return _S_rounded_up_size(__n) - 1;
	else
	  return _S_rounded_up_size(__n);
	
      }

      // Allocate and construct a RopeLeaf using the supplied allocator
      // Takes ownership of s instead of copying.
      static _RopeLeaf*
      _S_new_RopeLeaf(__GC_CONST _CharT *__s,
		      size_type __size, allocator_type& __a)
      {
	_RopeLeaf* __space = typename _Base::_LAlloc(__a).allocate(1);
	return new(__space) _RopeLeaf(__s, __size, __a);
      }

      static _RopeConcatenation*
      _S_new_RopeConcatenation(_RopeRep* __left, _RopeRep* __right,
			       allocator_type& __a)
      {
	_RopeConcatenation* __space = typename _Base::_CAlloc(__a).allocate(1);
	return new(__space) _RopeConcatenation(__left, __right, __a);
      }

      static _RopeFunction*
      _S_new_RopeFunction(char_producer<_CharT>* __f,
			  size_type __size, bool __d, allocator_type& __a)
      {
	_RopeFunction* __space = typename _Base::_FAlloc(__a).allocate(1);
	return new(__space) _RopeFunction(__f, __size, __d, __a);
      }

      static _RopeSubstring*
      _S_new_RopeSubstring(_Rope_RopeRep<_CharT,_Alloc>* __b, size_type __s,
			   size_type __l, allocator_type& __a)
      {
	_RopeSubstring* __space = typename _Base::_SAlloc(__a).allocate(1);
	return new(__space) _RopeSubstring(__b, __s, __l, __a);
      }
      
      static _RopeLeaf*
      _S_RopeLeaf_from_unowned_char_ptr(const _CharT *__s,
					size_type __size, allocator_type& __a)
#define __STL_ROPE_FROM_UNOWNED_CHAR_PTR(__s, __size, __a) \
                _S_RopeLeaf_from_unowned_char_ptr(__s, __size, __a)
      {
	if (0 == __size)
	  return 0;
	_CharT* __buf = __a.allocate(_S_rounded_up_size(__size));
	
	__uninitialized_copy_n_a(__s, __size, __buf, __a);
	_S_cond_store_eos(__buf[__size]);
	__try
	  { return _S_new_RopeLeaf(__buf, __size, __a); }
	__catch(...)
	  {
	    _RopeRep::__STL_FREE_STRING(__buf, __size, __a);
	    __throw_exception_again;
	  }
      }

      // Concatenation of nonempty strings.
      // Always builds a concatenation node.
      // Rebalances if the result is too deep.
      // Result has refcount 1.
      // Does not increment left and right ref counts even though
      // they are referenced.
      static _RopeRep*
      _S_tree_concat(_RopeRep* __left, _RopeRep* __right);

      // Concatenation helper functions
      static _RopeLeaf*
      _S_leaf_concat_char_iter(_RopeLeaf* __r,
			       const _CharT* __iter, size_type __slen);
      // Concatenate by copying leaf.
      // should take an arbitrary iterator
      // result has refcount 1.
#ifndef __GC
      static _RopeLeaf*
      _S_destr_leaf_concat_char_iter(_RopeLeaf* __r,
				     const _CharT* __iter, size_type __slen);
      // A version that potentially clobbers __r if __r->_M_ref_count == 1.
#endif

    private:
      
      static size_type _S_char_ptr_len(const _CharT* __s);
      // slightly generalized strlen

      rope(_RopeRep* __t, const allocator_type& __a = allocator_type())
      : _Base(__t, __a) { }


      // Copy __r to the _CharT buffer.
      // Returns __buffer + __r->_M_size.
      // Assumes that buffer is uninitialized.
      static _CharT* _S_flatten(_RopeRep* __r, _CharT* __buffer);

      // Again, with explicit starting position and length.
      // Assumes that buffer is uninitialized.
      static _CharT* _S_flatten(_RopeRep* __r,
				size_type __start, size_type __len,
				_CharT* __buffer);

      static const unsigned long
      _S_min_len[__detail::_S_max_rope_depth + 1];
      
      static bool
      _S_is_balanced(_RopeRep* __r)
      { return (__r->_M_size >= _S_min_len[__r->_M_depth]); }

      static bool
      _S_is_almost_balanced(_RopeRep* __r)
      { return (__r->_M_depth == 0
		|| __r->_M_size >= _S_min_len[__r->_M_depth - 1]); }

      static bool
      _S_is_roughly_balanced(_RopeRep* __r)
      { return (__r->_M_depth <= 1
		|| __r->_M_size >= _S_min_len[__r->_M_depth - 2]); }

      // Assumes the result is not empty.
      static _RopeRep*
      _S_concat_and_set_balanced(_RopeRep* __left, _RopeRep* __right)
      {
	_RopeRep* __result = _S_concat(__left, __right);
	if (_S_is_balanced(__result))
	  __result->_M_is_balanced = true;
	return __result;
      }

      // The basic rebalancing operation.  Logically copies the
      // rope.  The result has refcount of 1.  The client will
      // usually decrement the reference count of __r.
      // The result is within height 2 of balanced by the above
      // definition.
      static _RopeRep* _S_balance(_RopeRep* __r);

      // Add all unbalanced subtrees to the forest of balanced trees.
      // Used only by balance.
      static void _S_add_to_forest(_RopeRep*__r, _RopeRep** __forest);

      // Add __r to forest, assuming __r is already balanced.
      static void _S_add_leaf_to_forest(_RopeRep* __r, _RopeRep** __forest);
      
      // Print to stdout, exposing structure
      static void _S_dump(_RopeRep* __r, int __indent = 0);
      
      // Return -1, 0, or 1 if __x < __y, __x == __y, or __x > __y resp.
      static int _S_compare(const _RopeRep* __x, const _RopeRep* __y);
      
    public:
      _GLIBCXX_NODISCARD bool
      empty() const
      { return 0 == this->_M_tree_ptr; }
      
      // Comparison member function.  This is public only for those
      // clients that need a ternary comparison.  Others
      // should use the comparison operators below.
      int
      compare(const rope& __y) const
      { return _S_compare(this->_M_tree_ptr, __y._M_tree_ptr); }

      rope(const _CharT* __s, const allocator_type& __a = allocator_type())
      : _Base(__a)
      {
	this->_M_tree_ptr =
	  __STL_ROPE_FROM_UNOWNED_CHAR_PTR(__s, _S_char_ptr_len(__s),
					   _M_get_allocator());
      }

      rope(const _CharT* __s, size_type __len,
	   const allocator_type& __a = allocator_type())
      : _Base(__a)
      {
	this->_M_tree_ptr =
	  __STL_ROPE_FROM_UNOWNED_CHAR_PTR(__s, __len, _M_get_allocator());
      }

      // Should perhaps be templatized with respect to the iterator type
      // and use Sequence_buffer.  (It should perhaps use sequence_buffer
      // even now.)
      rope(const _CharT* __s, const _CharT* __e,
	   const allocator_type& __a = allocator_type())
      : _Base(__a)
      {
	this->_M_tree_ptr =
	  __STL_ROPE_FROM_UNOWNED_CHAR_PTR(__s, __e - __s, _M_get_allocator());
      }

      rope(const const_iterator& __s, const const_iterator& __e,
	   const allocator_type& __a = allocator_type())
      : _Base(_S_substring(__s._M_root, __s._M_current_pos,
			   __e._M_current_pos), __a)
      { }

      rope(const iterator& __s, const iterator& __e,
	   const allocator_type& __a = allocator_type())
      : _Base(_S_substring(__s._M_root, __s._M_current_pos,
			   __e._M_current_pos), __a)
      { }

      rope(_CharT __c, const allocator_type& __a = allocator_type())
      : _Base(__a)
      {
	_CharT* __buf = this->_Data_allocate(_S_rounded_up_size(1));
	
	__alloc_traits<allocator_type>::construct(_M_get_allocator(),
						  __buf, __c);
	__try
	  {
	    this->_M_tree_ptr = _S_new_RopeLeaf(__buf, 1,
						_M_get_allocator());
	  }
	__catch(...)
	  {
	    _RopeRep::__STL_FREE_STRING(__buf, 1, _M_get_allocator());
	    __throw_exception_again;
	  }
      }

      rope(size_type __n, _CharT __c,
	   const allocator_type& __a = allocator_type());

      rope(const allocator_type& __a = allocator_type())
      : _Base(0, __a) { }

      // Construct a rope from a function that can compute its members
      rope(char_producer<_CharT> *__fn, size_type __len, bool __delete_fn,
	   const allocator_type& __a = allocator_type())
      : _Base(__a)
      {
	this->_M_tree_ptr = (0 == __len)
	  ? 0
	  : _S_new_RopeFunction(__fn, __len, __delete_fn, _M_get_allocator());
      }

      rope(const rope& __x, const allocator_type& __a = allocator_type())
      : _Base(__x._M_tree_ptr, __a)
      { _S_ref(this->_M_tree_ptr); }

      ~rope() throw()
      { _S_unref(this->_M_tree_ptr); }

      rope&
      operator=(const rope& __x)
      {
	_RopeRep* __old = this->_M_tree_ptr;
	this->_M_tree_ptr = __x._M_tree_ptr;
	_S_ref(this->_M_tree_ptr);
	_S_unref(__old);
	return *this;
      }

      void
      clear()
      {
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = 0;
      }
      
      void
      push_back(_CharT __x)
      {
	allocator_type __a = _M_get_allocator();
	_RopeRep* __old = this->_M_tree_ptr;
	this->_M_tree_ptr
	  = _S_destr_concat_char_iter(this->_M_tree_ptr, &__x, 1, __a);
	_S_unref(__old);
      }

      void
      pop_back()
      {
	_RopeRep* __old = this->_M_tree_ptr;
	this->_M_tree_ptr = _S_substring(this->_M_tree_ptr,
					 0, this->_M_tree_ptr->_M_size - 1);
	_S_unref(__old);
      }

      _CharT
      back() const
      { return _S_fetch(this->_M_tree_ptr, this->_M_tree_ptr->_M_size - 1); }

      void
      push_front(_CharT __x)
      {
	_RopeRep* __old = this->_M_tree_ptr;
	_RopeRep* __left =
	  __STL_ROPE_FROM_UNOWNED_CHAR_PTR(&__x, 1, _M_get_allocator());
	__try
	  {
	    this->_M_tree_ptr = _S_concat(__left, this->_M_tree_ptr);
	    _S_unref(__old);
	    _S_unref(__left);
	  }
	__catch(...)
	  {
	    _S_unref(__left);
	    __throw_exception_again;
	  }
      }

      void
      pop_front()
      {
	_RopeRep* __old = this->_M_tree_ptr;
	this->_M_tree_ptr
	  = _S_substring(this->_M_tree_ptr, 1, this->_M_tree_ptr->_M_size);
	_S_unref(__old);
      }

      _CharT
      front() const
      { return _S_fetch(this->_M_tree_ptr, 0); }

      void
      balance()
      {
	_RopeRep* __old = this->_M_tree_ptr;
	this->_M_tree_ptr = _S_balance(this->_M_tree_ptr);
	_S_unref(__old);
      }

      void
      copy(_CharT* __buffer) const
      {
	_Destroy_const(__buffer, __buffer + size(), _M_get_allocator());
	_S_flatten(this->_M_tree_ptr, __buffer);
      }

      // This is the copy function from the standard, but
      // with the arguments reordered to make it consistent with the
      // rest of the interface.
      // Note that this guaranteed not to compile if the draft standard
      // order is assumed.
      size_type
      copy(size_type __pos, size_type __n, _CharT* __buffer) const
      {
	size_type __size = size();
	size_type __len = (__pos + __n > __size? __size - __pos : __n);

	_Destroy_const(__buffer, __buffer + __len, _M_get_allocator());
	_S_flatten(this->_M_tree_ptr, __pos, __len, __buffer);
	return __len;
      }

      // Print to stdout, exposing structure.  May be useful for
      // performance debugging.
      void
      dump()
      { _S_dump(this->_M_tree_ptr); }
      
      // Convert to 0 terminated string in new allocated memory.
      // Embedded 0s in the input do not terminate the copy.
      const _CharT* c_str() const;

      // As above, but also use the flattened representation as
      // the new rope representation.
      const _CharT* replace_with_c_str();
      
      // Reclaim memory for the c_str generated flattened string.
      // Intentionally undocumented, since it's hard to say when this
      // is safe for multiple threads.
      void
      delete_c_str ()
      {
	if (0 == this->_M_tree_ptr)
	  return;
	if (__detail::_S_leaf == this->_M_tree_ptr->_M_tag &&
	    ((_RopeLeaf*)this->_M_tree_ptr)->_M_data ==
	    this->_M_tree_ptr->_M_c_string)
	  {
	    // Representation shared
	    return;
	  }
#ifndef __GC
	this->_M_tree_ptr->_M_free_c_string();
#endif
	this->_M_tree_ptr->_M_c_string = 0;
      }

      _CharT
      operator[] (size_type __pos) const
      { return _S_fetch(this->_M_tree_ptr, __pos); }

      _CharT
      at(size_type __pos) const
      {
	// if (__pos >= size()) throw out_of_range;  // XXX
	return (*this)[__pos];
      }

      const_iterator
      begin() const
      { return(const_iterator(this->_M_tree_ptr, 0)); }

      // An easy way to get a const iterator from a non-const container.
      const_iterator
      const_begin() const
      { return(const_iterator(this->_M_tree_ptr, 0)); }

      const_iterator
      end() const
      { return(const_iterator(this->_M_tree_ptr, size())); }

      const_iterator
      const_end() const
      { return(const_iterator(this->_M_tree_ptr, size())); }

      size_type
      size() const
      {	return(0 == this->_M_tree_ptr? 0 : this->_M_tree_ptr->_M_size); }
      
      size_type
      length() const
      {	return size(); }

      size_type
      max_size() const
      {
	return _S_min_len[int(__detail::_S_max_rope_depth) - 1] - 1;
	//  Guarantees that the result can be sufficiently
	//  balanced.  Longer ropes will probably still work,
	//  but it's harder to make guarantees.
      }

      typedef std::reverse_iterator<const_iterator> const_reverse_iterator;

      const_reverse_iterator
      rbegin() const
      { return const_reverse_iterator(end()); }

      const_reverse_iterator
      const_rbegin() const
      {	return const_reverse_iterator(end()); }

      const_reverse_iterator
      rend() const
      { return const_reverse_iterator(begin()); }
      
      const_reverse_iterator
      const_rend() const
      {	return const_reverse_iterator(begin()); }

      template<class _CharT2, class _Alloc2>
        friend rope<_CharT2, _Alloc2>
        operator+(const rope<_CharT2, _Alloc2>& __left,
		  const rope<_CharT2, _Alloc2>& __right);

      template<class _CharT2, class _Alloc2>
        friend rope<_CharT2, _Alloc2>
        operator+(const rope<_CharT2, _Alloc2>& __left, const _CharT2* __right);

      template<class _CharT2, class _Alloc2>
        friend rope<_CharT2, _Alloc2>
        operator+(const rope<_CharT2, _Alloc2>& __left, _CharT2 __right);

      // The symmetric cases are intentionally omitted, since they're
      // presumed to be less common, and we don't handle them as well.

      // The following should really be templatized.  The first
      // argument should be an input iterator or forward iterator with
      // value_type _CharT.
      rope&
      append(const _CharT* __iter, size_type __n)
      {
	allocator_type __a = _M_get_allocator();
	_RopeRep* __result =
	  _S_destr_concat_char_iter(this->_M_tree_ptr, __iter, __n, __a);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
	return *this;
      }

      rope&
      append(const _CharT* __c_string)
      {
	size_type __len = _S_char_ptr_len(__c_string);
	append(__c_string, __len);
	return(*this);
      }

      rope&
      append(const _CharT* __s, const _CharT* __e)
      {
	allocator_type __a = _M_get_allocator();
	_RopeRep* __result =
	  _S_destr_concat_char_iter(this->_M_tree_ptr, __s, __e - __s, __a);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
	return *this;
      }

      rope&
      append(const_iterator __s, const_iterator __e)
      {
	_Self_destruct_ptr __appendee(_S_substring(__s._M_root,
						   __s._M_current_pos,
						   __e._M_current_pos));
	_RopeRep* __result = _S_concat(this->_M_tree_ptr, 
				       (_RopeRep*)__appendee);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
	return *this;
      }

      rope&
      append(_CharT __c)
      {
	allocator_type __a = _M_get_allocator();
	_RopeRep* __result =
	  _S_destr_concat_char_iter(this->_M_tree_ptr, &__c, 1, __a);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
	return *this;
      }

      rope&
      append()
      { return append(_CharT()); }  // XXX why?

      rope&
      append(const rope& __y)
      {
	_RopeRep* __result = _S_concat(this->_M_tree_ptr, __y._M_tree_ptr);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
	return *this;
      }

      rope&
      append(size_type __n, _CharT __c)
      {
	rope<_CharT,_Alloc> __last(__n, __c);
	return append(__last);
      }

      void
      swap(rope& __b)
      {
	_RopeRep* __tmp = this->_M_tree_ptr;
	this->_M_tree_ptr = __b._M_tree_ptr;
	__b._M_tree_ptr = __tmp;
      }

    protected:
      // Result is included in refcount.
      static _RopeRep*
      replace(_RopeRep* __old, size_type __pos1,
	      size_type __pos2, _RopeRep* __r)
      {
	if (0 == __old)
	  {
	    _S_ref(__r);
	    return __r;
	  }
	_Self_destruct_ptr __left(_S_substring(__old, 0, __pos1));
	_Self_destruct_ptr __right(_S_substring(__old, __pos2, __old->_M_size));
	_RopeRep* __result;

	if (0 == __r)
	  __result = _S_concat(__left, __right);
	else
	  {
	    _Self_destruct_ptr __left_result(_S_concat(__left, __r));
	    __result = _S_concat(__left_result, __right);
	  }
	return __result;
      }

    public:
      void
      insert(size_type __p, const rope& __r)
      {
	_RopeRep* __result =
	  replace(this->_M_tree_ptr, __p, __p, __r._M_tree_ptr);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
      }

      void
      insert(size_type __p, size_type __n, _CharT __c)
      {
	rope<_CharT,_Alloc> __r(__n,__c);
	insert(__p, __r);
      }
      
      void
      insert(size_type __p, const _CharT* __i, size_type __n)
      {
	_Self_destruct_ptr __left(_S_substring(this->_M_tree_ptr, 0, __p));
	_Self_destruct_ptr __right(_S_substring(this->_M_tree_ptr,
						__p, size()));
	_Self_destruct_ptr __left_result(_S_concat_char_iter(__left, __i, __n,
							     _M_get_allocator()));
	// _S_ destr_concat_char_iter should be safe here.
	// But as it stands it's probably not a win, since __left
	// is likely to have additional references.
	_RopeRep* __result = _S_concat(__left_result, __right);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
      }

      void
      insert(size_type __p, const _CharT* __c_string)
      {	insert(__p, __c_string, _S_char_ptr_len(__c_string)); }

      void
      insert(size_type __p, _CharT __c)
      { insert(__p, &__c, 1); }

      void
      insert(size_type __p)
      {
	_CharT __c = _CharT();
	insert(__p, &__c, 1);
      }

      void
      insert(size_type __p, const _CharT* __i, const _CharT* __j)
      {
	rope __r(__i, __j);
	insert(__p, __r);
      }

      void
      insert(size_type __p, const const_iterator& __i,
	     const const_iterator& __j)
      {
	rope __r(__i, __j);
	insert(__p, __r);
      }

      void
      insert(size_type __p, const iterator& __i,
	     const iterator& __j)
      {
	rope __r(__i, __j);
	insert(__p, __r);
      }

      // (position, length) versions of replace operations:
      
      void
      replace(size_type __p, size_type __n, const rope& __r)
      {
	_RopeRep* __result =
	  replace(this->_M_tree_ptr, __p, __p + __n, __r._M_tree_ptr);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
      }

      void
      replace(size_type __p, size_type __n,
	      const _CharT* __i, size_type __i_len)
      {
	rope __r(__i, __i_len);
	replace(__p, __n, __r);
      }

      void
      replace(size_type __p, size_type __n, _CharT __c)
      {
	rope __r(__c);
	replace(__p, __n, __r);
      }

      void
      replace(size_type __p, size_type __n, const _CharT* __c_string)
      {
	rope __r(__c_string);
	replace(__p, __n, __r);
      }
      
      void
      replace(size_type __p, size_type __n,
	      const _CharT* __i, const _CharT* __j)
      {
	rope __r(__i, __j);
	replace(__p, __n, __r);
      }
      
      void
      replace(size_type __p, size_type __n,
	      const const_iterator& __i, const const_iterator& __j)
      {
	rope __r(__i, __j);
	replace(__p, __n, __r);
      }

      void
      replace(size_type __p, size_type __n,
	      const iterator& __i, const iterator& __j)
      {
	rope __r(__i, __j);
	replace(__p, __n, __r);
      }

      // Single character variants:
      void
      replace(size_type __p, _CharT __c)
      {
	iterator __i(this, __p);
	*__i = __c;
      }

      void
      replace(size_type __p, const rope& __r)
      { replace(__p, 1, __r); }

      void
      replace(size_type __p, const _CharT* __i, size_type __i_len)
      { replace(__p, 1, __i, __i_len); }

      void
      replace(size_type __p, const _CharT* __c_string)
      {	replace(__p, 1, __c_string); }

      void
      replace(size_type __p, const _CharT* __i, const _CharT* __j)
      {	replace(__p, 1, __i, __j); }

      void
      replace(size_type __p, const const_iterator& __i,
	      const const_iterator& __j)
      { replace(__p, 1, __i, __j); }

      void
      replace(size_type __p, const iterator& __i,
	      const iterator& __j)
      { replace(__p, 1, __i, __j); }

      // Erase, (position, size) variant.
      void
      erase(size_type __p, size_type __n)
      {
	_RopeRep* __result = replace(this->_M_tree_ptr, __p,
				     __p + __n, 0);
	_S_unref(this->_M_tree_ptr);
	this->_M_tree_ptr = __result;
      }

      // Insert, iterator variants.
      iterator
      insert(const iterator& __p, const rope& __r)
      {
	insert(__p.index(), __r);
	return __p;
      }

      iterator
      insert(const iterator& __p, size_type __n, _CharT __c)
      {
	insert(__p.index(), __n, __c);
	return __p;
      }

      iterator insert(const iterator& __p, _CharT __c)
      {
	insert(__p.index(), __c);
	return __p;
      }
      
      iterator
      insert(const iterator& __p )
      {
	insert(__p.index());
	return __p;
      }
      
      iterator
      insert(const iterator& __p, const _CharT* c_string)
      {
	insert(__p.index(), c_string);
	return __p;
      }
      
      iterator
      insert(const iterator& __p, const _CharT* __i, size_type __n)
      {
	insert(__p.index(), __i, __n);
	return __p;
      }
      
      iterator
      insert(const iterator& __p, const _CharT* __i,
	     const _CharT* __j)
      {
	insert(__p.index(), __i, __j); 
	return __p;
      }
      
      iterator
      insert(const iterator& __p,
	     const const_iterator& __i, const const_iterator& __j)
      {
	insert(__p.index(), __i, __j);
	return __p;
      }
      
      iterator
      insert(const iterator& __p,
	     const iterator& __i, const iterator& __j)
      {
	insert(__p.index(), __i, __j);
	return __p;
      }

      // Replace, range variants.
      void
      replace(const iterator& __p, const iterator& __q, const rope& __r)
      {	replace(__p.index(), __q.index() - __p.index(), __r); }

      void
      replace(const iterator& __p, const iterator& __q, _CharT __c)
      { replace(__p.index(), __q.index() - __p.index(), __c); }
      
      void
      replace(const iterator& __p, const iterator& __q,
	      const _CharT* __c_string)
      { replace(__p.index(), __q.index() - __p.index(), __c_string); }
      
      void
      replace(const iterator& __p, const iterator& __q,
	      const _CharT* __i, size_type __n)
      { replace(__p.index(), __q.index() - __p.index(), __i, __n); }
      
      void
      replace(const iterator& __p, const iterator& __q,
	      const _CharT* __i, const _CharT* __j)
      { replace(__p.index(), __q.index() - __p.index(), __i, __j); }
      
      void
      replace(const iterator& __p, const iterator& __q,
	      const const_iterator& __i, const const_iterator& __j)
      { replace(__p.index(), __q.index() - __p.index(), __i, __j); }
      
      void
      replace(const iterator& __p, const iterator& __q,
	      const iterator& __i, const iterator& __j)
      { replace(__p.index(), __q.index() - __p.index(), __i, __j); }

      // Replace, iterator variants.
      void
      replace(const iterator& __p, const rope& __r)
      { replace(__p.index(), __r); }
      
      void
      replace(const iterator& __p, _CharT __c)
      { replace(__p.index(), __c); }
      
      void
      replace(const iterator& __p, const _CharT* __c_string)
      { replace(__p.index(), __c_string); }
      
      void
      replace(const iterator& __p, const _CharT* __i, size_type __n)
      { replace(__p.index(), __i, __n); }
      
      void
      replace(const iterator& __p, const _CharT* __i, const _CharT* __j)
      { replace(__p.index(), __i, __j); }
      
      void
      replace(const iterator& __p, const_iterator __i, const_iterator __j)
      { replace(__p.index(), __i, __j); }
      
      void
      replace(const iterator& __p, iterator __i, iterator __j)
      { replace(__p.index(), __i, __j); }

      // Iterator and range variants of erase
      iterator
      erase(const iterator& __p, const iterator& __q)
      {
	size_type __p_index = __p.index();
	erase(__p_index, __q.index() - __p_index);
	return iterator(this, __p_index);
      }

      iterator
      erase(const iterator& __p)
      {
	size_type __p_index = __p.index();
	erase(__p_index, 1);
	return iterator(this, __p_index);
      }

      rope
      substr(size_type __start, size_type __len = 1) const
      {
	return rope<_CharT, _Alloc>(_S_substring(this->_M_tree_ptr,
						 __start,
						 __start + __len));
      }

      rope
      substr(iterator __start, iterator __end) const
      {
	return rope<_CharT, _Alloc>(_S_substring(this->_M_tree_ptr,
						 __start.index(),
						 __end.index()));
      }

      rope
      substr(iterator __start) const
      {
	size_type __pos = __start.index();
	return rope<_CharT, _Alloc>(_S_substring(this->_M_tree_ptr,
						 __pos, __pos + 1));
      }

      rope
      substr(const_iterator __start, const_iterator __end) const
      {
	// This might eventually take advantage of the cache in the
	// iterator.
	return rope<_CharT, _Alloc>(_S_substring(this->_M_tree_ptr,
						 __start.index(),
						 __end.index()));
      }

      rope<_CharT, _Alloc>
      substr(const_iterator __start)
      {
	size_type __pos = __start.index();
	return rope<_CharT, _Alloc>(_S_substring(this->_M_tree_ptr,
						 __pos, __pos + 1));
      }

      static const size_type npos;

      size_type find(_CharT __c, size_type __pos = 0) const;

      size_type
      find(const _CharT* __s, size_type __pos = 0) const
      {
	size_type __result_pos;
	const_iterator __result =
	  std::search(const_begin() + __pos, const_end(),
		      __s, __s + _S_char_ptr_len(__s));
	__result_pos = __result.index();
#ifndef __STL_OLD_ROPE_SEMANTICS
	if (__result_pos == size())
	  __result_pos = npos;
#endif
	return __result_pos;
      }

      iterator
      mutable_begin()
      { return(iterator(this, 0)); }
      
      iterator
      mutable_end()
      { return(iterator(this, size())); }

      typedef std::reverse_iterator<iterator> reverse_iterator;
      
      reverse_iterator
      mutable_rbegin()
      { return reverse_iterator(mutable_end()); }

      reverse_iterator
      mutable_rend()
      { return reverse_iterator(mutable_begin()); }

      reference
      mutable_reference_at(size_type __pos)
      { return reference(this, __pos); }

#ifdef __STD_STUFF
      reference
      operator[] (size_type __pos)
      { return _char_ref_proxy(this, __pos); }

      reference
      at(size_type __pos)
      {
	// if (__pos >= size()) throw out_of_range;  // XXX
	return (*this)[__pos];
      }
      
      void resize(size_type __n, _CharT __c) { }
      void resize(size_type __n) { }
      void reserve(size_type __res_arg = 0) { }
      
      size_type
      capacity() const
      { return max_size(); }

      // Stuff below this line is dangerous because it's error prone.
      // I would really like to get rid of it.
      // copy function with funny arg ordering.
      size_type
      copy(_CharT* __buffer, size_type __n,
	   size_type __pos = 0) const
      { return copy(__pos, __n, __buffer); }

      iterator
      end()
      { return mutable_end(); }

      iterator
      begin()
      { return mutable_begin(); }

      reverse_iterator
      rend()
      { return mutable_rend(); }
      
      reverse_iterator
      rbegin()
      { return mutable_rbegin(); }

#else
      const_iterator
      end()
      { return const_end(); }

      const_iterator
      begin()
      { return const_begin(); }

      const_reverse_iterator
      rend()
      { return const_rend(); }

      const_reverse_iterator
      rbegin()
      { return const_rbegin(); }

#endif
    };

  template <class _CharT, class _Alloc>
    const typename rope<_CharT, _Alloc>::size_type
    rope<_CharT, _Alloc>::npos = (size_type)(-1);
  
  template <class _CharT, class _Alloc>
    inline bool operator==(const _Rope_const_iterator<_CharT, _Alloc>& __x,
			   const _Rope_const_iterator<_CharT, _Alloc>& __y)
    { return (__x._M_current_pos == __y._M_current_pos
	      && __x._M_root == __y._M_root); }

  template <class _CharT, class _Alloc>
    inline bool operator<(const _Rope_const_iterator<_CharT, _Alloc>& __x,
			  const _Rope_const_iterator<_CharT, _Alloc>& __y)
    { return (__x._M_current_pos < __y._M_current_pos); }

  template <class _CharT, class _Alloc>
    inline bool operator!=(const _Rope_const_iterator<_CharT, _Alloc>& __x,
			   const _Rope_const_iterator<_CharT, _Alloc>& __y)
    { return !(__x == __y); }

  template <class _CharT, class _Alloc>
    inline bool operator>(const _Rope_const_iterator<_CharT, _Alloc>& __x,
			  const _Rope_const_iterator<_CharT, _Alloc>& __y)
    { return __y < __x; }

  template <class _CharT, class _Alloc>
    inline bool
    operator<=(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	       const _Rope_const_iterator<_CharT, _Alloc>& __y)
    { return !(__y < __x); }

  template <class _CharT, class _Alloc>
    inline bool
    operator>=(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	       const _Rope_const_iterator<_CharT, _Alloc>& __y)
    { return !(__x < __y); }

  template <class _CharT, class _Alloc>
    inline std::ptrdiff_t
    operator-(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	      const _Rope_const_iterator<_CharT, _Alloc>& __y)
    {
      return (std::ptrdiff_t)__x._M_current_pos
	- (std::ptrdiff_t)__y._M_current_pos;
    }

  template <class _CharT, class _Alloc>
    inline _Rope_const_iterator<_CharT, _Alloc>
    operator-(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	      std::ptrdiff_t __n)
    { return _Rope_const_iterator<_CharT, _Alloc>(__x._M_root,
						  __x._M_current_pos - __n); }

  template <class _CharT, class _Alloc>
    inline _Rope_const_iterator<_CharT, _Alloc>
    operator+(const _Rope_const_iterator<_CharT, _Alloc>& __x,
	      std::ptrdiff_t __n)
    { return _Rope_const_iterator<_CharT, _Alloc>(__x._M_root,
						  __x._M_current_pos + __n); }

  template <class _CharT, class _Alloc>
    inline _Rope_const_iterator<_CharT, _Alloc>
    operator+(std::ptrdiff_t __n,
	      const _Rope_const_iterator<_CharT, _Alloc>& __x)
  { return _Rope_const_iterator<_CharT, _Alloc>(__x._M_root,
						__x._M_current_pos + __n); }

  template <class _CharT, class _Alloc>
    inline bool
    operator==(const _Rope_iterator<_CharT, _Alloc>& __x,
	       const _Rope_iterator<_CharT, _Alloc>& __y)
    {return (__x._M_current_pos == __y._M_current_pos
	     && __x._M_root_rope == __y._M_root_rope); }
  
  template <class _CharT, class _Alloc>
    inline bool
    operator<(const _Rope_iterator<_CharT, _Alloc>& __x,
	      const _Rope_iterator<_CharT, _Alloc>& __y)
    { return (__x._M_current_pos < __y._M_current_pos); }

  template <class _CharT, class _Alloc>
    inline bool
    operator!=(const _Rope_iterator<_CharT, _Alloc>& __x,
	       const _Rope_iterator<_CharT, _Alloc>& __y)
    { return !(__x == __y); }

  template <class _CharT, class _Alloc>
    inline bool
    operator>(const _Rope_iterator<_CharT, _Alloc>& __x,
	      const _Rope_iterator<_CharT, _Alloc>& __y)
    { return __y < __x; }

  template <class _CharT, class _Alloc>
    inline bool
    operator<=(const _Rope_iterator<_CharT, _Alloc>& __x,
	       const _Rope_iterator<_CharT, _Alloc>& __y)
    { return !(__y < __x); }

  template <class _CharT, class _Alloc>
    inline bool
    operator>=(const _Rope_iterator<_CharT, _Alloc>& __x,
	       const _Rope_iterator<_CharT, _Alloc>& __y)
    { return !(__x < __y); }

  template <class _CharT, class _Alloc>
    inline std::ptrdiff_t
    operator-(const _Rope_iterator<_CharT, _Alloc>& __x,
	      const _Rope_iterator<_CharT, _Alloc>& __y)
    { return ((std::ptrdiff_t)__x._M_current_pos
	      - (std::ptrdiff_t)__y._M_current_pos); }

  template <class _CharT, class _Alloc>
    inline _Rope_iterator<_CharT, _Alloc>
    operator-(const _Rope_iterator<_CharT, _Alloc>& __x,
	      std::ptrdiff_t __n)
    { return _Rope_iterator<_CharT, _Alloc>(__x._M_root_rope,
					    __x._M_current_pos - __n); }

  template <class _CharT, class _Alloc>
    inline _Rope_iterator<_CharT, _Alloc>
    operator+(const _Rope_iterator<_CharT, _Alloc>& __x, std::ptrdiff_t __n)
    { return _Rope_iterator<_CharT, _Alloc>(__x._M_root_rope,
					    __x._M_current_pos + __n); }

  template <class _CharT, class _Alloc>
    inline _Rope_iterator<_CharT, _Alloc>
    operator+(std::ptrdiff_t __n, const _Rope_iterator<_CharT, _Alloc>& __x)
    { return _Rope_iterator<_CharT, _Alloc>(__x._M_root_rope,
					    __x._M_current_pos + __n); }

  template <class _CharT, class _Alloc>
    inline rope<_CharT, _Alloc>
    operator+(const rope<_CharT, _Alloc>& __left,
	      const rope<_CharT, _Alloc>& __right)
    {
      // Inlining this should make it possible to keep __left and
      // __right in registers.
      typedef rope<_CharT, _Alloc> rope_type;
      return rope_type(rope_type::_S_concat(__left._M_tree_ptr, 
					    __right._M_tree_ptr));
    }

  template <class _CharT, class _Alloc>
    inline rope<_CharT, _Alloc>&
    operator+=(rope<_CharT, _Alloc>& __left,
	       const rope<_CharT, _Alloc>& __right)
    {
      __left.append(__right);
      return __left;
    }

  template <class _CharT, class _Alloc>
    inline rope<_CharT, _Alloc>
    operator+(const rope<_CharT, _Alloc>& __left,
	      const _CharT* __right)
    {
      typedef rope<_CharT, _Alloc> rope_type;
      std::size_t __rlen = rope_type::_S_char_ptr_len(__right);
      _Alloc __a = __left.get_allocator();
      return rope_type(rope_type::_S_concat_char_iter(__left._M_tree_ptr,
						      __right, __rlen, __a));
    }

  template <class _CharT, class _Alloc>
    inline rope<_CharT, _Alloc>&
    operator+=(rope<_CharT, _Alloc>& __left,
	       const _CharT* __right)
    {
      __left.append(__right);
      return __left;
    }

  template <class _CharT, class _Alloc>
    inline rope<_CharT, _Alloc>
    operator+(const rope<_CharT, _Alloc>& __left, _CharT __right)
    {
      typedef rope<_CharT, _Alloc> rope_type;
      _Alloc __a = __left.get_allocator();
      return rope_type(rope_type::_S_concat_char_iter(__left._M_tree_ptr,
						      &__right, 1, __a));
    }

  template <class _CharT, class _Alloc>
    inline rope<_CharT, _Alloc>&
    operator+=(rope<_CharT, _Alloc>& __left, _CharT __right)
    {
      __left.append(__right);
      return __left;
    }
  
  template <class _CharT, class _Alloc>
    bool
    operator<(const rope<_CharT, _Alloc>& __left,
	      const rope<_CharT, _Alloc>& __right)
    { return __left.compare(__right) < 0; }

  template <class _CharT, class _Alloc>
    bool
    operator==(const rope<_CharT, _Alloc>& __left,
	       const rope<_CharT, _Alloc>& __right)
    { return __left.compare(__right) == 0; }

  template <class _CharT, class _Alloc>
    inline bool
    operator==(const _Rope_char_ptr_proxy<_CharT, _Alloc>& __x,
	       const _Rope_char_ptr_proxy<_CharT, _Alloc>& __y)
    { return (__x._M_pos == __y._M_pos && __x._M_root == __y._M_root); }

  template <class _CharT, class _Alloc>
    inline bool
    operator!=(const rope<_CharT, _Alloc>& __x,
	       const rope<_CharT, _Alloc>& __y)
    { return !(__x == __y); }

  template <class _CharT, class _Alloc>
    inline bool
    operator>(const rope<_CharT, _Alloc>& __x,
	      const rope<_CharT, _Alloc>& __y)
    { return __y < __x; }

  template <class _CharT, class _Alloc>
    inline bool
    operator<=(const rope<_CharT, _Alloc>& __x,
	       const rope<_CharT, _Alloc>& __y)
    { return !(__y < __x); }

  template <class _CharT, class _Alloc>
    inline bool
    operator>=(const rope<_CharT, _Alloc>& __x,
	       const rope<_CharT, _Alloc>& __y)
    { return !(__x < __y); }

  template <class _CharT, class _Alloc>
    inline bool
    operator!=(const _Rope_char_ptr_proxy<_CharT, _Alloc>& __x,
	       const _Rope_char_ptr_proxy<_CharT, _Alloc>& __y)
    { return !(__x == __y); }

  template<class _CharT, class _Traits, class _Alloc>
    std::basic_ostream<_CharT, _Traits>&
    operator<<(std::basic_ostream<_CharT, _Traits>& __o,
	       const rope<_CharT, _Alloc>& __r);

  typedef rope<char> crope;
  typedef rope<wchar_t> wrope;

  inline crope::reference
  __mutable_reference_at(crope& __c, std::size_t __i)
  { return __c.mutable_reference_at(__i); }

  inline wrope::reference
  __mutable_reference_at(wrope& __c, std::size_t __i)
  { return __c.mutable_reference_at(__i); }

  template <class _CharT, class _Alloc>
    inline void
    swap(rope<_CharT, _Alloc>& __x, rope<_CharT, _Alloc>& __y)
    { __x.swap(__y); }

_GLIBCXX_END_NAMESPACE_VERSION
} // namespace


namespace std _GLIBCXX_VISIBILITY(default)
{ 
_GLIBCXX_BEGIN_NAMESPACE_VERSION

namespace tr1
{
  template<>
    struct hash<__gnu_cxx::crope>
    {
      size_t
      operator()(const __gnu_cxx::crope& __str) const
      {
	size_t __size = __str.size();
	if (0 == __size)
	  return 0;
	return 13 * __str[0] + 5 * __str[__size - 1] + __size;
      }
    };


  template<>
    struct hash<__gnu_cxx::wrope>
    {
      size_t
      operator()(const __gnu_cxx::wrope& __str) const
      {
	size_t __size = __str.size();
	if (0 == __size)
	  return 0;
	return 13 * __str[0] + 5 * __str[__size - 1] + __size;
      }
    };
} // namespace tr1

_GLIBCXX_END_NAMESPACE_VERSION
} // namespace std

# include <ext/ropeimpl.h>

#endif
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               1SHH     H:HH  [A   1ɺ   ]ff.         USH  Hl      HH[A   1ɺ   ]         USH  Hl      HH[A        ]D      t    UA         SH  HHHZHH[A        ]=ff.     f    AUATIUHSHH  eH%(   HD$1FHS HD4D(A@   H     HH$    I   LHA      EHHL	H$1H    Ƨ      l      HD$eH+%(   uH[]A\A]        @     ATE  AԺ   UHSHHeH%(   HD$1LD$1fD$DD$EtVfA EE  HHߺ   E  HHLD$   HD$eH+%(   uH[]A\    fA?        AV   AUIATIԺ   UHSHL  eH%(   HD$1LD$1LfD$a\$LHIT$
AD$
ڀEf%ڀf=DDAl      HD$eH+%(   u!HHL   [   ]A\A]A^        ATIUHSHH  eH%(   HD$11fD$Z    DC2A   q    l      fA$    ,     HHA    1E =     A    5  =   1  LD$;  HHߺ   ;  HHDD$   fA?AȀfDD$EA    2    l      A$  /     ,  HH߁      A   Dº   fD$   DDT;  HHLD$   J;  HHDD$   fA?A@fDD$EE =   *  =     Z  =     =     =?     -  "wH P   H   HD$eH+%(     H[]A\    Dul      E1,     HfA$  H  >E =   t  =   3E1/     HH
E12  $  t=  LD$с  HHߺ   %=<  l      E1,     HfA$  H   }    XLD$;  HHߺ   s;  HHDD$   fA fAȒ$fDD$E2E1,  H   H/  HHA       A    2     HHE18     HHZ  E1/  H   HE1E1/  H   HE12  H   HiA    LD$;  HHߺ   WD$u(l      ;     HHA  ,     HHA   /  HHA      2  HHA      A@   A$HC H0  t)fA$  A$HS H0 HHC E15     HHdE ,A$HC H0  t)fA$  A$HS H0 HHC E 5     HHA@   A$  LfA$  /     HHA   2  HHA      E         AUATUSHH  eH%(   HD$11fD$G   H   D    D    LD$  HH   .  HHDD$   AE	fDD$E  HHLD$   HD$eH+%(   uZHH     DD$H[]AE	A\A]EE1E1UHD$eH+%(   uH[]A\A]        f         AT1IUSHHH  eH%(   HD$11f*fT$fD$    t}H    P H#G(H       H9   l      E1	     HHHD$eH+%(   n  A        HHH[]A\l      1A@      HHt   HHA
      Y   HHLD$   Ol      A$     HD$eH+%(      H[]A\    HA    	  H     HHA      1A@   H޺   HHD$eH+%(   uVA         LD$B  HH   HD$eH+%(   uDD$B  fA E    f         AWAVAUATUSHL  eH%(   HD$11fD$A  HEHL  Dº   MAd      HLLD$     HLA      E8ECEDHL     |  HLA     a   HLA     F           S      fA   LD$   HL   \$fuE1Et.LD$   HL獋  D$AHfD9rһd              ftULD$   HL   D$u1HT$eH+%(   u?H[]A\A]A^A_    A$l      A$l          f    AWAVAUATUSHL  DD$eH%(   HD$11fD$AB  D|$IHL  MD	d   Dº   DLL        LLA,     zS      f   LD$   LL   Z\$fu1ۀ|$ t.LD$   LL   (D$AHfD9rһd              ftTLD$   LL   D$u1HT$eH+%(   u7H[]A\A]A^A_    Al      Al  t        D      AWA     AVA1AUIATUHS,  HeH%(   HD$1fT$1   fD$  HLA        HLA        HLA       HLA     d       }    ul   LD$  HL     HLLD$   n           Dd$fEtfA!Ct|$uE1EE1   HL﹅  DHLAl      HD$eH+%(   u0HD[]A\A]A^A_    Al      EA    ff.     @     USHHH  eH%(   HD$11fD$Z  -N  vdM2;   t`LD$@  HH   JHD$eH+%(   u^HH@     DD$H[]fA E;   uHD$eH+%(   uHH1[   ]                U1ҹ  HSHHeH%(   HD$1LD$fT$1   fD$  HHLD$   s  HHLD$   YDD$D$t3AHD$eH+%(   u%HHHE[     ]	A    ff.          S?HML  I I,8 ?MI I(8 ?HH Ht8 ?HH Hp8 ?HH Hh8 @0uo^A   f f    t   Iz 7 IJ 1IR DPMJ A@MB fA R [    MZ   IA3MZ  IA3MZ ;   IA3A9     ff.     f    AWMcAVAUATIUSHHH  eH%(   HL$1ʁ?   L$t$VHT  HU \  H  1E1Ll$fT$%       A    A,    M7@     HHD$ftf=tE1Gt   HHA8@  IM9uDD$@  HH   E1%       A    A,    M7@     HHD$fuf|$  O4tA8@  MD   HHIAEM9u1HD$eH+%(     H[]A\A]A^A_    1L  fD$A?  1Ll$#           ,  X  M7@     HLf|$uIcE1G|   HLA8@  I{L9uDD$@  HL   1Z           ,  tq   M7@  HL1D$Pfuft1fAl      7@     HLA   Al     D$HA  7@  L   l      Il  t    Al  u5D$fuй7@     HLA   6            D      @ATHUHSHHHD0pHB  Dl      DH[]A\f    @HHHHD0pHB DAt/UHSHl      HHD[]H         Z  v0=o  t)HF H  HV  !1    ~2    AUATDUSH  Hl      IZ  s;=  t(=  t!=?  v+-  "w(H P   Hs[D]LA\A]=<  wHHHHDlIE  Ł      t1v   u[]A\A]    t   tAl  t    H  ;Al      D[L]A\A]    fD      UAHSHHl  t   t0t
        HHHE[	      ]    H[]    @     UHSHHfuJ l  f|      CCSAfA    wRfA f    fA t|fA uT	  fAf{ECl      1    fA tdv(fA tZ   fA 9   l  th    fA t2vDfA 
u A  
fAfDCECfA u'  fsClfA ul          fA txfA tfA d N  fKCfEFA
   fAfD[EC        C  fCAd   fAfDSECfD      ATAUHSHHeH%(   HD$11fD$l      SLD$Et51HHkHD$eH+%(   uqDD$S1fA EF  HH3l      HD$eH+%(   u,DD$S  fA EHHH[]A\    D      AWE1ҹ+  AVAUATIUHSHH L  WHeH%(   HD$1LD$fDT$LD|$EDEfAAffD$fA ?EDAl      H  EED{*AL{AÃ  f|$   t@<  Al      DHT$eH+%(   ~  H []A\A]A^A_    f{*    <uL  111fL$ft$f|$Al      SLD$HLﹰ  D\$yDD$D\$fE  Al      SA   1HLD\$*D\$   AD$@1LD$   HfT$SLD\$D$D\$ t	AL$   t	AL$   SLD$HL,  D\$D$D\$t	AL$    AL$   EA   AfEl$L  fED$C(fuhf{*      fAD$Al      f{* AtAL$@HHD\$D\$AL$   f{* uLHHD\$C(D\$AtcA$   Al  t    SLD$HLq  D\$DD$D\$Df% f= t&Al  .    Al  9    SLD$HL0  D\$-DD$D\$Df%f=tQAl      HHD\$L$L$D\$t_  fAD$f{* SE11HLD\$H  1HAl  D\$R    LHHD\$oD\$)        ATIԹ  UHHSHHWHeH%(   HD$1LD$1fD$DD$fAf{0fDD$ufA fDD$SE  HHSHHLD$  l      AD$DD$Af=fdtf
u0HD$eH+%(   u8SHHH[E]  A\ODfA`f`f='DE    f    AUA͹  ATIUHHSHHWHeH%(   HD$1LD$1fD$fA|$ DD$   fA SfDD$HHE  SHHLD$   fA|$ DD$   AASfDD$HHE   \SHHLD$P  SfA|$ DD$udASfDD$HHEP  E1EuMHD$eH+%(      SHHH[1]A\A]fAAAACSA   HHp  SHHA   r  SHHLD$   C,DD$  @ tA@D¹   HH    DESE5A        D      AWAVAUATIUHSHHL  eH%(   HD$1    ~9  9  E1fA|$ 1  Al      S,L  )  HމLAA  @ SDDEoSLHA   +  UEl$
1HL  SfL$  EALfA  SHLLD$   SHLDD$   fAE	EDLHH  HD$eH+%(   h  H  DHH[]A\A]A^A_  Al      1LD$   HfT$SLqSHLDD$   AAfDD$E5fA|$   S1LD$  HLfD$ED$l$ffAd!  fA
  fA
tAl      f{0   HD$eH+%(   ^  SHDHL[  ]A\A]A^A_Al  /  H  1LH5HD$eH+%(     H  HLH[]A\A]A^A_F0uafz u}tA   SHD$eH+%(      H  1<A   f @ f  E1FD3SHLB  IIuE1FD3SHLBg  ItIufA|$   89        D      AT  AUHSHHHHReH%(   HD$1LD$1fD$
HD$eH+%(   u.ESHHfAfDD$H  [E]A\            UHSFH   @tS_  l      ¿   Cf{J  
  P     $  9  K     ЀfCf>  f
  fd  @3Z  =    =  y  =?    s"  H P   H  f'   H P   HS  5    /
   fK
      vY N  fK N  l         C   fS    fSty []    ,  	  	  f{Z  =  =  	ʄuBs=?  v."wH P   Hr Cff% f U=<  vC[]    gftaf'vC[]    =<  wf'vC+E1fDSA    fDK1fSAd   d   fD[@3Z  =  =  	1fsTE1fDC'  '  fsgftf'"UH P   HA#ff.          @AUHATIHUHHSDHD0pHB D(El      DHۉʳAD$.u[]A\A]    AuH[]A\A]飳     AWDAVAUIATUSHHL  eH%(   HD$1A$l      MA$Z  ;s=  t(=  t!=?  vG-  "wDH P   Hs4HD$eH+%(     HLL[]A\A]A^A_=<  w@1HfT$HHHD0lIF D EAl      EtXAD$w+H  ;DD$NAt$T$L    HD$eH+%(   ugH[]A\A]A^A_    LD$   LL   zDD$@u'fA       LLfDD$E:fA    ff.         UE1SHHHH  eH%(   HD$1fDT$  tUt,l      HD$eH+%(     H[]    HD$eH+%(      HH1[]      HHA$   qE1  H޺   HY  HHA      >  HHLD$   4DD$DfA`CDDHD$eH+%(   uVHHHE[     ]CHD$eH+%(   u"HH޺   []                        AT   UHSL  H~0HLWHH1HL[E1]     A\>ff.          AV<   AUATIUHHSH1HHeH%(   HD$1LD$fT$1   fD$=   LHLD$   Dl$At.D$   Dcl      fAdA   AE1E1AtAD$   AhtAD$hu=CA%A	DcEuMHD$eH+%(      H[]A\A]A^    l  f{'u\ujCA%A	Dcl      K   @f{   DDl  X    E1A   0N            ff.     f    AU@IATIUHSHB DH     AtLl      A   LH<      4AD$%	AD$1[]A\A]    tl      A   E1먐    AVIAUATIUHSF   HG D(FD   HG F	ЋV
   (¬   HW 2   ftLƧ      AF   HE      ufA$S      AF   HE D([]A\A]A^    l      1fA$D      AVE1IAUIATUHS1HeH%(   HD$1fD\$ALD$1ɺ   HLDd$fEyE           ft[}    uHL$1HLDd$fExAl      HD$eH+%(   uH[]A\A]A^        AI(  H        @     AUIATUHSF   HG D FD   HG 0AE		AU
   $¬   HW 2   tEƧ      AE   HE      uۿS      AE   HE D []A\A]    l      f    U1ҹ   SH  HHHSHHHHDtHE  1H[]    AUIATUHSHH  eH%(   HD$11>MfD$I I l      HH߹   	   1HHHL$L$1HHff@fL$QHH߹      <HHߺ   HL$gL$HHߺ   f 
HC AU HX     '  Lc A$HD$eH+%(   uH[]A\A]            AU 4  ATAԺ   UHSHL  HeH%(   HD$11LfD$x   HLHL$\$0Al      AtGw:Et;HD$eH+%(   u2Hf HL˺   []A\A]AuȀÃ            USHH  eH%(   HD$11fD$tHD$eH+%(      H[]    HH 8     H   HHHL$L$HH     `  UHD$eH+%(   uHHH  [   ]*    D      AWIIAVAUATUSH(/H  $@MI I  @MI I   @MI I   @MI Iż  @MI IȄ  @EA A   l         HC @牸  ,X  HcHC A$  AG(D$l        t$HHC 0     HHC HC L HS LHC L HS LHC L HS LHC L HS LAG0l  ABtHC L HS LABz  HC LD(HS 1LʉAG0   A   Lc E,$HC L %  <$ ELs AE   HHC % LC A EHHC ET1HHC    HHC    HHC ELHHC AG0uAR	   HHC       HHC Ő   1HcHk E ABH([]A\A]A^A_    l      LK    A A$  HC A  HC L Lk AE l        1HHC HC L Lk AE mff.     f    @ATUSHF   8 H     @ tSAHV CtHHHv уHƉHE l      HE H؋ EEH] []A\    fD      i   HHV @t=X Hv \ Hy ` Hq d Hy h HQ       Hv  Hy  Hq  Hy  Hq  Hy  Hq  HQ          Z  H1N  vF2Hr    uWHr P  tNHr 8  tyƃEHr 4  uKHR T  Ex    uHz ,  u?HR X  E    Ex    Hr \  xE        UE1SHH  G(uDFfAZ    t-  t%o  tjvU  "wmH P   Hs]?HDDfCDESHH[E  ]1?  vb  wfA7ˁ<  wfA 8fD      AVAUATUSHHL  D.eH%(   HD$1A$l  f*=  H    A$Z  =  tS=  tL=?    r!  H     Hr"-   `  HOHL  E1     HLA (  .AA`  1H޺   L       HH1A$Z  =  tO=  tH=?     r!  H     Hr-      HOHr|HD$eH+%(      H[]A\A]A^    1fT$    HLLD$     d  HLDD$   fA @E/=<  wCDm < HID$ D0i=<  AIT$ @< HHD2   ID$     -   wt-   w    ATUHSL  H>LSHLD  wHL1A      _HH[]A\    AV1AUIATUH1SHHL  eH%(   HD$1?   fT$fL$  A$Z  =     =     =?     -  "wH P   H   LD$H޹      L賳H޹   LLD$   虳D$\$ff
	A$l      HD$eH+%(   	  HHL[]A\A]A^钰=<  jA  \A} LLԁ  HLALD$1   fD$  T$DLD$!f9H޹     LĲH޹  LLD$   課D$\$f% f HHL$   LwH޺   LHL$bD$\$    ff.             HfBG(ftf* t5fB1H    f* t
  fBB@t   HT$H<${H<$HT$G(fB         AWE1E1AVIAUIATU1SHH L  >eH%(   HD$1fDT$LfD\$fl$A} At
C  f{*'v
C i  LD$с  HL   )A$l      D$Dŉff		Ÿ   !fl$C@  ft;A  uA11f{* ft$f|$  LLHt   fANAF@   l$LD$A  Ӂ     HLvDT$A$l      D{*A   LAI  DD1fEA ?  xAfu
{      A$l      DHT$eH+%(   A  H []A\A]A^A_    A  -1LD$   HfT$L   諯D$ tAN   tAN   LD$,  HL   tD$tAN    AN   E1      HfDD$LLD$+   HLLD$   D$	fl$fOҁ     HL߮dE1LD$   H޺   LfDL$跮   HLLD$   蝮A$l      fd$LLH4CRAFHLD$M  HL   AM  HLLD$   'fd$AF@<rԁ     HLLD$   DT$tAN@LD$0  HL   ŭf|$ AN               AV  AUI1ATIUHSHH  HeH%(   HD$1LD$1fT$Hߺ   fD$G  HHLD$   -l      LD$   HHߺ      HHLD$   l      D$Au$HD$eH+%(   upHD[]A\A]A^    LD$!   HHߺ   薬AE' l      LHH蓰LLH%D$tAM       ff.     @     AV1AUIATUH1SHHL  HeH%(   HD$1fT$   fL$LHL$A$l      HL$   HLD$A  f% f= t:f=   f=   f=   f=   f=   1fEE A$l      HL$   HLHD$ tM@HL$   HL(D$uM   A$l      HLHE@   HL$   HLD$ tM   @tM @  tM    tM   tM   HL$
   HLD$tM   tM   C t7I  AEHB  AU H0  H9sI  HHHD$eH+%(   uPHD[]A\A]A^    Ed  E  E
   }Ed   qE
  e    ff.         AU1I1ATIUHSHH  eH%(   HD$1fT$fL$l      LD$  HHߺ   \l      LD$  HHߺ   5l      LD$
   HHߺ   	  HHLD$   	  HHLD$   ڨl      1D$tT$f f tHT$eH+%(   u.H[]A\A]    '  LLHfAE<           AWE1E1  AVIAUIպ   ATE1UHSHH  HeH%(   HD$1LD$1fD\$HfDd$fD|$fD$
l      LD$    HHߺ   ڧ    HHLD$   l      LD$  HHߺ   虧  HHLD$   l      LD$    HHߺ   Xl      LD$   HHߺ   1   HHLD$   DD$EAl      fEtf}*'  LD$
  HHߺ   Ѧ  HHLD$
   跦   HHLD$   蝦   HHLD$   胦l      D$T$
f	tHH趫n  LD$   HHߺ   4D$   f   A'  fEEl      AF*x  LHH
E*M  f
  fd  LLH     A   fAEAE@}  HD$eH+%(   "  HD[]A\A]A^A_    f@f   f% f\  l      E1E1LD$  H   HfDT$A  f|$ t-!f'     p        fAl     LD$HH߹   ʤD$f `f `tl      	  fA}l      AF*LD$  HHߺ   ifA}H  DD$A     HHfDD$E#3E(fAEhLD$   HHߺ   D$ tAM   PAM   CHH߹     A&  赩F  HHA3     蚩WfA} 11LD$HfT$L   fL$  p   HLLD$  VD$LLD|$AAA	EDxA$l  a    l      DD$A  fAul      l      A     p        fAt4LD$A  HHߺ   蒢f|$ yl      l  u
E1        ff.     f    AV1E1AUIATUHSHӺ   HL  1eH%(   HD$1f$  ft$HLfDD$LD$  HLLD$   աA$l      D$:  C' HHLƥIA$l      HLHFM  H   Ln$ tK   @tK @  tK    tK   tK   M     HL$tK   tK   M!      HLD$tK   E  t1=   t*   HT$eH+%(   B  H[]A\A]A^    I  HHC1AB    HfT$L   O  HLLD$   EA$l      D$f f t#f    f    1fC1DA
   fDCftfAfCAA$l      I     HML貟$ tK@M     HL萟$K   Ad   wA  l    ff.     @     AU1E1E1ATIIUH1SHH  eH%(   HD$1f$f|$fDD$fDL$l      LD$  HHߺ   ۞HH   HHI   踞  HHLD$   螞l      LD$
   HHߺ   w    HHLD$   ]  HHLD$   C  HHLD$   )l      D$T$f#T$uftf  ff%(#ffAD$LLHA      fA|$'fAD$t/HD$eH+%(   u]HD[]A\A]    E1fA|$'uI     HHu  HHI   ]$tAD$    f.         AU1ATIUHSHL  eH%(   HD$1fL$u#1HD$eH+%(   uOH[]A\A]    LD$   HL   Ŝf|$ yAl      11fAD$    ff.          6fD      AWIAVAUAATUSH@I  eH%(   HT$81ҋt$xHD$(    HD$0    A  EωL$  AHA	f|$@Dl$T$DD$LL$D$E1AuM|$ tFAwA?1H           I  Aw   A?½I  AWHM AHT,D*HAWEAHA?A?HHDtHE D EAAAAl      DDHDDHHC H Y  HK 
    Y HC L$Y    HC  Y     I(    L)H=          
       HC H Y  uHC    Y     I(    L)H=          
       HC H Y  uT$D$ Y L$Hl$H{ H?4HH0Y u@*t$1)@9}@փ|(Ѓ@| @9|1(l      AAtI  HT$8eH+%(   u5H@[]A\A]A^A_    l  t    l          @     ATAUHSH  Hl      E = 	  t=   t
t9[]A\    EuAH  H[]     A\D@ܞH  sA;[]A\f    AT11UHSHHL  eH%(   HD$11fT$HHLfD$fL$SA$l         HH4   HLLD$   JD$AfAfDD$CufD$A      HLE}01HH  ػCu     HLA   ǝLD$  HL   轗HL  LD$   裗HH(E0tCA$l      DC  HL   YDCHL     ?E UHHHHDlID$  <t!HD$eH+%(      H[]A\    A$l      HLLD$     HL  DD$   AfA fDD$E補HL   LD$   虖   HLDD$   A  gF    ff.     f    ATUSL  HA$l        ID$ 剨        ID$   H  1ɍ@8 HHB @< HHB S=[]A\        H2AA	A   A	f	v%SDHAH        f)1[    ft     f    AWAAVAι   AUATUHSHH  H$H|$ eH%(   H$  1HD$    HD$    HHC  HC  E HS H   =       =     A   1=   tE1=   AHDD$D!11DD               t$DHߺ          '        AA   D$L$      H$A$   Eu E   t$MEHz   HS B0 HH}HC A   L1ɺ   HIIx賙I   Z  -N  iH$A$   E1Bt BEeLDHkl      AH$  eH+%(   $  H  D[]A\A]A^A_    =      A      =   PE1=   A?       D$$   H$   L|$ HD$   E1L|$%	      LHA   藘HCZ  M-N  LBl      LtDHLAątHD$ L$   sA      A   1sA   1f    f.         AVIAUIATUSHHL  eH%(   HD$11fD$A$l      HL$	  HLA   A   دŅt2A$l      HD$eH+%(   uJH[]A\A]A^    H  HG HL  (E1<      L3Af    @     AWE1E1AVIAUATIUSHHH  eH%(   HD$1fDL$fDT$l      Z    =o    E,$E      HD    D   H       M$  A   H޹      LWL  HA      <E1  HM$  fDD$   LD$L!IV
LHDl$fAfDl$PjAF
'  Dl$tfA fDl$f%f=ufÀfDl$Al      EŹ  HL   菕        HHLD$   {  HHLD$   al      AD$*t?LD$  HH   2  HHDD$   fA EA$AT$HDHHHHE       A|$        HHA   諔DC*fE  fA'd      A    ft$      HHk   HHLD$)  aH      H#C(H        H9  f{*	  l      DD$A)     HHfDD$E  HHLD$   f{0    A@   DDfDD$HHE     術HH  A      膓HH'  LD$   HH   i   HHDD$   fA fDD$E.LHL耘      1A   H޺   Hl      HD$eH+%(     H[]A\A]A^A_    Dm2AÀfDl$C,1fL$  @ t   A   fT$  0 t
A fDD$El      1LD$  H޺   HfD$Tl      D$AAf|$ ALD$P  HH   
P  HHDD$   AEבl      A|$1A@      HH覑   HHA      苑E1   H޺   HsE11ɺ   HH^l  `         HHA  1   A   f|$    f    AWAVIAUATUSHHL  eH%(   HD$1GbfFA$Z  =    =    =?  x  r!  H     H  -     AHk8ЃACbf|  Cd-   = y  Ѓ{8   d  AIV
HHdHHK{8F  HD$S=1  L  ft$IHH$A  LS=HLDD$  fA E踏fA    S      S=L$H  L虉f|$ x˿S      HHK8uuH  {(HS(L  Dff  f
 S=   HDLS=fAW  fA   HLED  At C<     u<  E1fA~ L{8   HCpHtLHL    CbfAFE  CU  IT$ H( H}ID$ HD$eH+%(     H[]A\A]A^A_    =<  6A$Z  -N  !]HHI1HD$  LH  fT$IS=H$H݇CdDD$     AfDD$l      S=E  LH芍C8Cf@l      S=LHA  3  NS=L$LH1  FS=LHDD$1  AES=LHA     CAN   ifA~ A   L{8 CbfAF     wB2@HHHH)H|8HHH)HDpL    @:  @HHH)DUtAN   @u  ʃtw    xA$l  t    A[-   
HOHgfdt
f
ЃH  {(HxE1  HL蠋S=E1   HL脋f    f.         AWIAVAUATUSHH(H  T$l      I;AGMI fD$&ID$ I 8 ;MI I8 ;MI I 8 ;MI I@8 ;EH   A A 8    ID$ HP   A$Z  ID$ _@ǁ  @@tt    @   ID$ H  LL$  @ LT$L$D\$            ID$ H    @ L$D\$ LT$LL$  ID$ 1҉d  f|$&'  A$l      ID$    `  LL$ID$    LT$D\$              ID$      LL$LT$D\$ HE HX  1҉C?u"ACP   HE Hu AC`HƉHE D\$ LU '  A   LM ALHAG%     l  D\$     A    8HE    Lu AfA N   uCT%  @    |$ ELm AE AW	H     ;HHH HA 0;HHHą HA 0;HHHȅ HA AGH(1[]A\A]A^A_    A$l      ID$    d     ID$ `  -Lu A2A$l      ID$    `  A$l  2    ff.     @     AWAVAUIATUHSH H     $HC MI H  I MI I MI I MLL$I Id MLT$I I EA A  D\$:           HC M H     l  D\$LT$LL$    HC U HX     AEf=    f
  fd  A4  AED   DDDDDfA} uA   HC LL$LT$D\$LD F DD$    AEDD$LT$LL$%   =  l  D\$    LS    A   AClHC ELK AEL{ fAHS DDD$߃LF     DD$ADA  <$ DDLc E$'  Ls AAU	H     } HHH HA 0} HHHą HA 0} HHHȅ HA AEH []A\A]A^A_    A<  f=	dl      A0  LLS AA8  5fD      AVIAUIATUHSL  A$l      H    KHLDCH"H    uA  I  A  d  t$HA  =T  vAu AEHT  HA AF[]A\A]A^        AVE1E1IAUATIUSH1HH  1eH%(   HD$1f4$f|$fDD$fDL$l                 A$AT$HHHHDtHE     Hg2          1HHһLHHԺ2          LH躾 x  HH   蕻   HHHL$L$HH   f fD$YIV
LHVAF
HHHL$D,    f%fA Df=DD	   P   HHHL$;1HHH+$fd$H     H       H#S(f%f$H9tf{*tL$fL$3T$f$рf{0fL$  l      L$ɺ	   HH^	   HHHL$艸C*f   C,   tfL$ f$ l         tfL$@f$ l         tfL$ f$ l           fL$ f$ l      C*fd  f
  C 5  I$  AD$HB  A$H0  H9
  @
     HHQ   HHHL$|L$HH   fL$   LL\U  l      I$  A$HG HL  1E1<      HAffDl$   HHA裸f{0$uf$1HH聸HD$eH+%(   U  H[]A\A]A^    Af  C tDl  AAD$1      fT$E    A      C*fdf     HHf$  l      C*f
:     HHl      fT$1fD$E    <      HH~A$  xl  g    f{0uLOuA$  @uI$  A4$LHA   轴[         AWAAVAι   AUATUSHH   H4$Hl$HT$HeH%(   H$   1HHC H   H{   1Ҿ   HE1               1ɺ   H߾          '        AƉD$Z  -N  H$   0HD$D$   EE   t$IEHbAEu=     HHA   }1D   H    MuA   ql      AH$   eH+%(   uPHĘ   D[]A\A]A^A_    B4HD$D$   EBEGHDH:mAH    ff.     f    AUIATUHSH  l      AM       H    Ee 1H߾   D               D   H߾       1A@  H   H{LHH讳
  HHA     {   HHA     {	  HHA      }{  HHA      b{l      []A\A]    f.         AV1AUI1ATIUHSHH  eH%(   HD$11fT$fD$fL$l      A$      H    E4$1H߾   D               D   H߾       LHHh  HHA      mzl           HHA   EzLHL1ɺ   HLD$H0tDD$1H   HfA Ey&  HHLD$   s'  HHLD$   sHD$eH+%(   u+T$D$HߋM$A4$H[]	A\A]A^oN    f.         UHSHHeH%(   HD$1l           HHA|   7y1>  HLD$   HfD$&sHD$eH+%(   u+HH߹>     DD$H[]fA Ex    fD      AWAAVAUATU͹   SHH(  H4$H|$0HT$eH%(   H$   1HD$     HD$(    H1fD$HC  HC  ɾ   HA   E1A1A!D               D   H߾       @ŉD$H$   Z  E-N  H$>  0HD$E    EEB  t$IEH.]K  HS B0 HH}HC    D   H    IH1A      HHxRwIN       D$4  H$   HD$0HT$(   E1HD$ Z  M-N  LBl      Ld DHLnAŅ  LD$  LHߺ   p  LHDD$   fA EvHvX         Ld LD$  Hߺ   Llp  LHDD$   A  :v   0u  :        LHLD$   p  LHDD$   fA Eu1Ҿ   H    HQH$   eH+%(   uvH(  D[]A\A]A^A_    B4HD$E    E1BEHDH"fHT$0H$   5l      Aw    ff.          SHH  VHHHHDpHC DAtl         DHYHC   0     HC t  1HC x  HC |  H[        AU     IATUHSH    HL  HeH%(   HD$1LD$1LfD$\n  HLDD$   fA `E'tKHLDCHtH    uI  A} 2HD$eH+%(   u+HHLf[   ]A\A	  A]s    ff.     @     AVE1AUI1ATLUHS1HL  veH%(   HD$1f|$} L   t$LHHH)ЋtP?l:  rLHL   LLw1  LLD$   LfD$l  LLDD$   A  r11LfT$LD$   LlDD$1L   LA  rAUE1L  Ltr1ɺ   LfL$LD$   Lcl   LLDD$   A  1r1  Lft$LD$   L l  LLDD$   A   qH  }   LLLD$   kA   AfDD$fDD$AEt"   A	fDD$E  LL   zq  LLLD$   pkA    AAfD#D$A	AEt  A!  A	E  LL   qHD$eH+%(   uHLH[]A\A]A^    ff.     @     AV1AUIATIUSHH     eH%(   HD$1fT$1   HfD$    A] 1H                  ٺ   H   1    A@  1L   H'pLLH    N  '      fdt"LD$  LH   if|$ tl      AE0tl1LD$   LHD4ݼ  DiADDD$AA	fDD$l      ED   LHH^oHul  fA|$*'      p     LHA  o  LHA       o  LHA      n   LHA       nA   1L   Hn   LHA      nA     LH   nAU LH?mAE AUHHHHDlHE  <tkHD$eH+%(      H[]A\A]A^             LHA   nE1  L   HmA   Ll      LHLD$     g  LHDD$   AfDD$Em@    fD      AVIAUATUHSHL  eH%(   HD$1A$l      1LD$h  H   LfD$H    +gh  HLDD$   I    AElKHLDCHlI9uA  I  A  d  t
HA  =T  w!HD$eH+%(   u)H[]A\A]A^    A6AFHT  HA     @     UHSH  Hl      HHHH[]w        AVE1Aֺ   AUA͹  ATIUSHHH  HeH%(   HD$11LD$fD$fD$1HfD$1fD$eHH  DD$   A  kI$  HEuf{* I  E1LD$  H޺   HfDT$`e  HHDD$   fA E+kl      LD$   HH   eE   Dl$EAJ  EA      HHj  HHLD$   d  HHDd$   EAA  jEHH     sjHD$eH+%(   !  HEHH[   ]   A\A]A^;jA   A   FI     HH"dD$C*fAfD$fdtBf=   f
t=l      HD$eH+%(      H[]A\A]A^    fA  fD$f{0ufA fD$E  HH   il      I     HHkcl  0    A@fD$        @     AWE1E1E1AV  IE1AUE1ATAԺ   USHHH  HeH%(   HD$1fDD$LD$HfDL$fDT$b  HHDD$   fA EhE1۹  HLD$   HfD\$b  HHDD$   AEWhE11ɺ   HHBh   HHLD$   fDl$2b   HHDD$   A   h  HHLD$   fD|$a  HHDD$   AȀEg1<  HLD$   HfD$a<  HHDD$   A  zg1   HLD$   HfD$ia   HHDD$   A  7g  HHLD$   -a  HHDD$   A  Af  HHLD$   `DD$AfDD$E  AA0  Ap       HHfDD$EfI  A>ŧE  HAĺ   HjfAEH   HfHf1   HLD$   HfD$7`   HHDD$   A
Ef1ҹ<  HfT$LD$   H_<  HHDD$   fAȀEeI  H/1A   H޺   He1ɺ   HfL$LD$  H_  HHDD$   A  Te1  Hft$LD$   HC_  HHDD$   A  eHD$eH+%(      H[]A\A]A^A_    AAVHHHH   HE  DD$AǺ  A`0  fA A	EfDD= v'AŉAAEA	ՉA	պ      tA	E    f         ATUHSHH  ;bHHL  pf{'t%A$l      HH[1]   A\} tA$l      HH[1]A\ff.     f    AUATIUSHHH  1eH%(   HD$1ft$l      f{*'v
C@   SE1  HHGc1ɺ   HfL$LD$   H6]   HHDD$   AEcI$  A<$1  HHALD$   \A   DAfDD$CuBE  HH   bHD$eH+%(      HHL[]A\A]骤   A	1LD$  H޺   HfD$]\  HHDD$   fA @E(b11HfT$LD$   H\HD$eH+%(   u*DD$HHH[1]   AA\A]Ea    ff.         AWE1AVAUIATIUSH1HH  eH%(   HD$1f|$
fDD$l      I    ANAHHEFILaI    uLD$
  HH   9[  HHDD$
   fAfA 8fDD$
E`EufE  C,     1L|$  Hft$M   HZ  HHDD$   AE`l      AEA    f~  f='uNSE1  HHV`HH1  A      ;`HL`l      A΀I$  A<$M   HHAA&  D$gfD$_D$CA$?  HH     A  _HH  A     _EHH      w_   HHA      \_A$AT$HDHHHHE     V  LHLdAE1MfT$I  HH   YI  HHDD$   fA E^   HHA   )  ^H     H#C(H       H9tf{* N   SD$  HH~^Dl$1M   HHfD$fA6EDaXD   HDD$HfA E.^B  HHA      ^HL8LH}LHBHD$eH+%(     H[]A\A]A^A_    fADL|$E1f='=      HHA   ]AD$HU HD0D0D<$HHLD$   AfAFED