diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index 17f97a0f9890..51070cd3963e 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -18,38 +18,6 @@ inline void FormatCompareAndSwap(std::ostream& out) { - out << R"XXX( -template -_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch(_Fn& __cuda_cas, int __success_memorder, int __failure_memorder, _Sco) { - bool __res = false; - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __res = __cuda_cas(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __res = __cuda_cas(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) - return __res; -} -)XXX"; - // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -61,8 +29,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch(_Fn& __cuda_ constexpr auto asm_intrinsic_format_128 = R"XXX( template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -78,12 +47,11 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( mov.b128 {{%0, %1}}, _d; }} )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; }})XXX"; - - constexpr auto asm_intrinsic_format = R"XXX( + constexpr auto asm_intrinsic_format = R"XXX( template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __cuda_atomic_operand_{0}{1}, {6}) -{{ asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; constexpr Operand supported_types[] = { Operand::Bit, @@ -135,7 +103,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -147,7 +115,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -156,46 +124,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( } } - out << "\n" - << R"XXX( -template -struct __cuda_atomic_bind_compare_exchange { - _Type* __ptr; - _Type* __exp; - _Type* __des; - - template - _CCCL_DEVICE_API bool operator()(_Atomic_Memorder) { - return __cuda_atomic_compare_exchange(__ptr, *__exp, *__exp, *__des, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template -_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch(_Type* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); - bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); -} -template -_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch(_Type volatile* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); - bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); -} -)XXX"; + out << "\n"; } #endif // COMPARED_AND_SWAP_H diff --git a/libcudacxx/codegen/generators/definitions.h b/libcudacxx/codegen/generators/definitions.h index fb9c034df046..d1886103e8c6 100644 --- a/libcudacxx/codegen/generators/definitions.h +++ b/libcudacxx/codegen/generators/definitions.h @@ -153,6 +153,18 @@ inline std::string semantic_tag(Semantic sem) return sem_map[sem]; } +inline std::string ptx_semantic_tag(Semantic sem) +{ + static std::map sem_map = { + std::pair{Semantic::Relaxed, "__cuda_atomic_ptx_order_relaxed"}, + std::pair{Semantic::Release, "__cuda_atomic_ptx_order_release"}, + std::pair{Semantic::Acquire, "__cuda_atomic_ptx_order_acquire"}, + std::pair{Semantic::Acq_Rel, "__cuda_atomic_ptx_order_acq_rel"}, + std::pair{Semantic::Volatile, "__cuda_atomic_order_volatile"}, + }; + return sem_map[sem]; +} + enum class Scope { Thread, diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index 7ef5ab230f0e..d6cdaa2cc5ad 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -18,36 +18,6 @@ inline void FormatExchange(std::ostream& out) { - out << R"XXX( -template -_CCCL_DEVICE_API void __cuda_atomic_exchange_order_dispatch(_Fn& __cuda_exch, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_exch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_exch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_exch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; - // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -59,8 +29,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange_order_dispatch(_Fn& __cuda_exch, in constexpr auto asm_intrinsic_format_128 = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -76,12 +47,11 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( }} )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); }})XXX"; - - constexpr auto asm_intrinsic_format = R"XXX( + constexpr auto asm_intrinsic_format = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) -{{ asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; constexpr Operand supported_types[] = { Operand::Bit, @@ -133,7 +103,7 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -145,7 +115,7 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -154,44 +124,7 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } } - out << "\n" - << R"XXX( -template -struct __cuda_atomic_bind_exchange { - _Type* __ptr; - _Type* __old; - _Type* __new; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_exchange(__ptr, *__old, *__new, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template -_CCCL_DEVICE_API void __atomic_exchange_cuda(_Type* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_exchange_cuda(_Type volatile* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); -} -)XXX"; + out << "\n"; } #endif // EXCHANGE_H diff --git a/libcudacxx/codegen/generators/fence.h b/libcudacxx/codegen/generators/fence.h index 1c6ae3495f1c..928c7c8391e1 100644 --- a/libcudacxx/codegen/generators/fence.h +++ b/libcudacxx/codegen/generators/fence.h @@ -77,16 +77,32 @@ _CCCL_DEVICE_API inline void __cuda_atomic_fence({0}, {2}) } out << "\n" << R"XXX( +template +_CCCL_DEVICE_API void +__cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco __scope) +{ + if (__order.__was_seq_cst) + { + ::cuda::std::__cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); + } +} + +template +_CCCL_DEVICE_API void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) +{} + template -_CCCL_DEVICE_API void __cuda_atomic_thread_fence(int __memorder, _Sco) { +_CCCL_DEVICE_API void __cuda_atomic_thread_fence( + __cuda_atomic_ptx_backend, memory_order __order, _Sco) { + [[maybe_unused]] const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; + case __ATOMIC_SEQ_CST: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } @@ -97,7 +113,7 @@ _CCCL_DEVICE_API void __cuda_atomic_thread_fence(int __memorder, _Sco) { case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_membar(_Sco{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index be6232c9455d..a7fcd0c4fd76 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -17,13 +17,13 @@ #include "definitions.h" -inline std::string fetch_op_skip_v(std::string fetch_op) +inline std::string fetch_op_transform(std::string fetch_op) { if (fetch_op == "add") { - return "constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip;"; + return "\n __op = __op * __atomic_ptr_skip_t<_Type>::__skip;"; } - return "constexpr auto __skip_v = 1;"; + return {}; } inline void FormatFetchOps(std::ostream& out) @@ -50,37 +50,6 @@ inline void FormatFetchOps(std::ostream& out) std::pair{std::string{"and"}, std::pair{bitwise_types, std::string{"bitwise"}}}, }; - // Memory order dispatcher - out << R"XXX( -template -_CCCL_DEVICE_API void __cuda_atomic_fetch_order_dispatch(_Fn& __cuda_fetch, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_fetch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_fetch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_fetch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; - // Argument ID Reference // 0 - Atomic Operation // 1 - Operand Type @@ -93,59 +62,9 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_order_dispatch(_Fn& __cuda_fetch, int constexpr auto asm_intrinsic_format = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_fetch_{0}( - _Type* __ptr, _Type& __dst, _Type __op, {5}, __cuda_atomic_operand_{1}{2}, {7}) -{{ asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; - - // 0 - Atomic Operation - // 1 - Operand type constraint - // 2 - Pointer op skip_v - constexpr auto fetch_bind_invoke = R"XXX( -template -struct __cuda_atomic_bind_fetch_{0} {{ - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) {{ - __cuda_atomic_fetch_{0}(__ptr, *__dst, *__op, _Atomic_Memorder{{}}, _Tag{{}}, _Sco{{}}); - }} -}}; -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_{0}_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{{ - {2} - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_{1}<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_{1}<_Type>::__tag; - _Type __dst{{}}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} - __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}}; - __cuda_atomic_fetch_order_dispatch(__bound_{0}, __memorder, _Sco{{}}); - return __dst; -}} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_{0}_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{{ - {2} - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_{1}<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_{1}<_Type>::__tag; - _Type __dst{{}}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} - __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}}; - __cuda_atomic_fetch_order_dispatch(__bound_{0}, __memorder, _Sco{{}}); - return __dst; -}} -)XXX"; - - constexpr size_t supported_sizes[] = { + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, {5} __order, __cuda_atomic_operand_{1}{2}, {7}) +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {7}{{}}); asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; + constexpr size_t supported_sizes[] = { 32, 64, }; @@ -192,28 +111,15 @@ template -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} -template -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} -)XXX"; } #endif // FETCH_OPS_H diff --git a/libcudacxx/codegen/generators/header.h b/libcudacxx/codegen/generators/header.h index 41a9b5cdf723..62d0a41a1bf0 100644 --- a/libcudacxx/codegen/generators/header.h +++ b/libcudacxx/codegen/generators/header.h @@ -44,15 +44,11 @@ inline void FormatHeader(std::ostream& out) #include #include -#include -#include -#include - #include #include #include -#include -#include +#include +#include #include diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index c883eea5fb50..dbc5d38ef6ab 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -38,32 +38,6 @@ inline std::string scope_ld_st(Semantic sem, Scope sco) inline void FormatLoad(std::ostream& out) { - out << R"XXX( -template -_CCCL_DEVICE_API void __cuda_atomic_load_order_dispatch(_Fn &__cuda_load, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; - // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -77,8 +51,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load_order_dispatch(_Fn &__cuda_load, int __ constexpr auto asm_intrinsic_format_128 = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -95,16 +70,17 @@ _CCCL_DEVICE_API void __cuda_atomic_load( constexpr auto asm_intrinsic_format = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) -{{ asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); uint16_t __tmp; asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); }})XXX"; constexpr size_t supported_sizes[] = { @@ -170,7 +146,7 @@ _CCCL_DEVICE_API void __cuda_atomic_load( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -184,7 +160,7 @@ _CCCL_DEVICE_API void __cuda_atomic_load( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -198,7 +174,7 @@ _CCCL_DEVICE_API void __cuda_atomic_load( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -210,68 +186,11 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } } } - out << "\n" - << R"XXX( -template -struct __cuda_atomic_bind_load { - const _Type* __ptr; - _Type* __dst; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_load(__ptr, *__dst, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_DEVICE_API void __atomic_load_cuda(const _Type* __ptr, _Type& __dst, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - const __proxy_t* __ptr_proxy = reinterpret_cast(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_load_cuda(const _Type volatile* __ptr, _Type& __dst, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - const __proxy_t* __ptr_proxy = reinterpret_cast(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); -} -)XXX"; + out << "\n"; } inline void FormatStore(std::ostream& out) { - out << R"XXX( -template -_CCCL_DEVICE_API void __cuda_atomic_store_order_dispatch(_Fn &__cuda_store, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_RELEASE: __cuda_store(__cuda_atomic_order_release{}); break; - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_RELEASE: [[fallthrough]]; - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -285,8 +204,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store_order_dispatch(_Fn &__cuda_store, int constexpr auto asm_intrinsic_format_128 = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -303,13 +223,14 @@ _CCCL_DEVICE_API void __cuda_atomic_store( constexpr auto asm_intrinsic_format = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) -{{ asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); const uint16_t __tmp = static_cast(__val); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__tmp) : "memory"); }})XXX"; @@ -374,7 +295,7 @@ _CCCL_DEVICE_API void __cuda_atomic_store( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -388,7 +309,7 @@ _CCCL_DEVICE_API void __cuda_atomic_store( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -402,7 +323,7 @@ _CCCL_DEVICE_API void __cuda_atomic_store( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -414,41 +335,7 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } } } - out << "\n" - << R"XXX( -template -struct __cuda_atomic_bind_store { - _Type* __ptr; - _Type* __val; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_store(__ptr, *__val, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_DEVICE_API void __atomic_store_cuda(_Type* __ptr, _Type& __val, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_store_cuda(volatile _Type* __ptr, _Type& __val, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); -} -)XXX"; + out << "\n"; } #endif // LD_ST_H diff --git a/libcudacxx/include/cuda/__atomic/atomic.h b/libcudacxx/include/cuda/__atomic/atomic.h index 388a85329cb6..c5ecc576134d 100644 --- a/libcudacxx/include/cuda/__atomic/atomic.h +++ b/libcudacxx/include/cuda/__atomic/atomic.h @@ -144,13 +144,16 @@ atomic_thread_fence(memory_order __m, [[maybe_unused]] thread_scope _Scope = thr NV_IS_DEVICE, (switch (_Scope) { case thread_scope::thread_scope_system: - ::cuda::std::__cuda_atomic_thread_fence((int) __m, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_thread_fence( + ::cuda::std::__cuda_atomic_device_backend{}, __m, __thread_scope_system_tag{}); break; case thread_scope::thread_scope_device: - ::cuda::std::__cuda_atomic_thread_fence((int) __m, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_thread_fence( + ::cuda::std::__cuda_atomic_device_backend{}, __m, __thread_scope_device_tag{}); break; case thread_scope::thread_scope_block: - ::cuda::std::__cuda_atomic_thread_fence((int) __m, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_thread_fence( + ::cuda::std::__cuda_atomic_device_backend{}, __m, __thread_scope_block_tag{}); break; // Atomics scoped to themselves do not require fencing case thread_scope::thread_scope_thread: diff --git a/libcudacxx/include/cuda/std/__atomic/functions.h b/libcudacxx/include/cuda/std/__atomic/functions.h index b8de1c701762..c69220d36dec 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions.h +++ b/libcudacxx/include/cuda/std/__atomic/functions.h @@ -21,13 +21,7 @@ # pragma system_header #endif // no system header +#include #include -// Device atomics -#include -#include - -// Host atomics -#include - #endif // __CUDA_STD___ATOMIC_FUNCTIONS_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/backend.h b/libcudacxx/include/cuda/std/__atomic/functions/backend.h new file mode 100644 index 000000000000..a2476a286328 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/backend.h @@ -0,0 +1,327 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +enum class __cuda_atomic_order +{ + _relaxed, + _release, + _acquire, + _acq_rel, + _seq_cst, + _volatile, +}; + +template <__cuda_atomic_order _Order> +using __cuda_atomic_order_tag _CCCL_NODEBUG = integral_constant<__cuda_atomic_order, _Order>; + +using __cuda_atomic_order_relaxed _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_relaxed>; +using __cuda_atomic_order_release _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_release>; +using __cuda_atomic_order_acquire _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_acquire>; +using __cuda_atomic_order_acq_rel _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_acq_rel>; +using __cuda_atomic_order_seq_cst _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_seq_cst>; +using __cuda_atomic_order_volatile _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_volatile>; + +template +struct __cuda_atomic_ptx_order : _Order +{ + bool __was_seq_cst; + + _CCCL_HOST_DEVICE_API constexpr explicit __cuda_atomic_ptx_order(bool __was_seq_cst_) noexcept + : __was_seq_cst(__was_seq_cst_) + {} +}; + +using __cuda_atomic_ptx_order_relaxed _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_relaxed>; +using __cuda_atomic_ptx_order_release _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_release>; +using __cuda_atomic_ptx_order_acquire _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_acquire>; +using __cuda_atomic_ptx_order_acq_rel _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_acq_rel>; + +struct __cuda_atomic_operation_load +{}; + +struct __cuda_atomic_operation_store +{}; + +struct __cuda_atomic_operation_rmw +{}; + +struct __cuda_atomic_runtime_cas_order +{ + memory_order __success; + memory_order __failure; +}; + +template +struct __cuda_atomic_cas_order +{ + using __success _CCCL_NODEBUG = _Success; + using __failure _CCCL_NODEBUG = _Failure; +}; + +struct __cuda_atomic_cas_strong +{}; + +struct __cuda_atomic_cas_weak : __cuda_atomic_cas_strong +{}; + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __cuda_atomic_cas_is_weak(__cuda_atomic_cas_strong) +{ + return false; +} + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __cuda_atomic_cas_is_weak(__cuda_atomic_cas_weak) +{ + return true; +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __cuda_atomic_initial_load_order(_Order) +{ + if constexpr (is_same_v<_Order, __cuda_atomic_order_volatile>) + { + return __cuda_atomic_order_volatile{}; + } + else if constexpr (is_same_v<_Order, memory_order> || is_same_v<_Order, __cuda_atomic_runtime_cas_order>) + { + return memory_order_relaxed; + } + else + { + return __cuda_atomic_order_relaxed{}; + } +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __cuda_atomic_initial_load_order(__cuda_atomic_ptx_order<_Order>) +{ + return __cuda_atomic_ptx_order_relaxed{false}; +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +__cuda_atomic_compare_exchange_initial_load_order(__cuda_atomic_cas_order<_Success, _Failure>) +{ + return _Failure{}; +} + +// Compare-exchange may fail after the initial load without issuing a CAS, so that load must satisfy the failure order. +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __cuda_atomic_compare_exchange_initial_load_order(_Order __order) +{ + if constexpr (is_same_v<_Order, __cuda_atomic_runtime_cas_order>) + { + return __order.__failure; + } + else if constexpr (is_same_v<_Order, memory_order>) + { + return __order == memory_order_release + ? memory_order_relaxed + : (__order == memory_order_acq_rel ? memory_order_acquire : __order); + } + else if constexpr (is_same_v<_Order, __cuda_atomic_order_acquire> || is_same_v<_Order, __cuda_atomic_order_acq_rel>) + { + return __cuda_atomic_order_acquire{}; + } + else + { + return ::cuda::std::__cuda_atomic_initial_load_order(__order); + } +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +__cuda_atomic_compare_exchange_initial_load_order(__cuda_atomic_ptx_order<_Order> __order) +{ + if constexpr (is_same_v<_Order, __cuda_atomic_order_acquire> || is_same_v<_Order, __cuda_atomic_order_acq_rel>) + { + return __cuda_atomic_ptx_order_acquire{__order.__was_seq_cst}; + } + else + { + return __cuda_atomic_ptx_order_relaxed{false}; + } +} + +template +using __cuda_atomic_mmio_tag = integral_constant; + +using __cuda_atomic_mmio_enable = __cuda_atomic_mmio_tag; +using __cuda_atomic_mmio_disable = __cuda_atomic_mmio_tag; + +enum class __cuda_atomic_operand +{ + _f, + _s, + _u, + _b, +}; + +template <__cuda_atomic_operand _Op, size_t _Size> +struct __cuda_atomic_operand_tag +{ + static constexpr auto __op = _Op; + static constexpr auto __size = _Size; +}; + +using __cuda_atomic_operand_f8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 8>; +using __cuda_atomic_operand_s8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 8>; +using __cuda_atomic_operand_u8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 8>; +using __cuda_atomic_operand_b8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 8>; +using __cuda_atomic_operand_f16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 16>; +using __cuda_atomic_operand_s16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 16>; +using __cuda_atomic_operand_u16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 16>; +using __cuda_atomic_operand_b16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 16>; +using __cuda_atomic_operand_f32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 32>; +using __cuda_atomic_operand_s32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 32>; +using __cuda_atomic_operand_u32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 32>; +using __cuda_atomic_operand_b32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 32>; +using __cuda_atomic_operand_f64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 64>; +using __cuda_atomic_operand_s64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 64>; +using __cuda_atomic_operand_u64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 64>; +using __cuda_atomic_operand_b64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 64>; +using __cuda_atomic_operand_f128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 128>; +using __cuda_atomic_operand_s128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 128>; +using __cuda_atomic_operand_u128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 128>; +using __cuda_atomic_operand_b128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 128>; + +template +struct __cuda_atomic_operand_deduction +{ + using __type = _AtomicType; + using __tag = _OpTag; +}; + +struct _CCCL_ALIGNAS(16) __cuda_atomic_longlong2 +{ + uint64_t __x; + uint64_t __y; + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __cuda_atomic_longlong2 + operator&(__cuda_atomic_longlong2 __lhs, __cuda_atomic_longlong2 __rhs) noexcept + { + return {__lhs.__x & __rhs.__x, __lhs.__y & __rhs.__y}; + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __cuda_atomic_longlong2 + operator|(__cuda_atomic_longlong2 __lhs, __cuda_atomic_longlong2 __rhs) noexcept + { + return {__lhs.__x | __rhs.__x, __lhs.__y | __rhs.__y}; + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __cuda_atomic_longlong2 + operator^(__cuda_atomic_longlong2 __lhs, __cuda_atomic_longlong2 __rhs) noexcept + { + return {__lhs.__x ^ __rhs.__x, __lhs.__y ^ __rhs.__y}; + } +}; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_bitwise_impl() noexcept +{ + using __tag = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, sizeof(_Type) * CHAR_BIT>; + if constexpr (sizeof(_Type) == 16) + { + return __cuda_atomic_operand_deduction<__cuda_atomic_longlong2, __tag>{}; + } + else + { + return __cuda_atomic_operand_deduction<__make_nbit_uint_t, __tag>{}; + } +} + +// TODO: Once CUDA 12.0 is no longer supported, factor the repeated decltype below into a common deduction alias. +// CUDA 12.0 cudafe can substitute an unrelated alias for an intermediate alias template in large translation units. +template +using __cuda_atomic_deduce_bitwise_t = typename decltype(__cuda_atomic_deduce_bitwise_impl<_Type>())::__type; + +template +using __cuda_atomic_deduce_bitwise_tag_t = typename decltype(__cuda_atomic_deduce_bitwise_impl<_Type>())::__tag; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_arithmetic_impl() noexcept +{ + constexpr bool __is_floating = is_floating_point_v<_Type> || __is_extended_floating_point_v<_Type>; + constexpr auto __op = + __is_floating ? __cuda_atomic_operand::_f + : (is_signed_v<_Type> && sizeof(_Type) != 8 ? __cuda_atomic_operand::_s : __cuda_atomic_operand::_u); + using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * CHAR_BIT>; + if constexpr (__is_floating || sizeof(_Type) == 16) + { + return __cuda_atomic_operand_deduction<_Type, __tag>{}; + } + else + { + return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; + } +} + +template +using __cuda_atomic_deduce_arithmetic_t = typename decltype(__cuda_atomic_deduce_arithmetic_impl<_Type>())::__type; + +template +using __cuda_atomic_deduce_arithmetic_tag_t = typename decltype(__cuda_atomic_deduce_arithmetic_impl<_Type>())::__tag; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_minmax_impl() noexcept +{ + constexpr bool __is_floating = is_floating_point_v<_Type> || __is_extended_floating_point_v<_Type>; + constexpr auto __op = __is_floating ? __cuda_atomic_operand::_f + : (is_signed_v<_Type> ? __cuda_atomic_operand::_s : __cuda_atomic_operand::_u); + using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * CHAR_BIT>; + if constexpr (__is_floating || sizeof(_Type) == 16) + { + return __cuda_atomic_operand_deduction<_Type, __tag>{}; + } + else + { + return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; + } +} + +template +using __cuda_atomic_deduce_minmax_t = typename decltype(__cuda_atomic_deduce_minmax_impl<_Type>())::__type; + +template +using __cuda_atomic_deduce_minmax_tag_t = typename decltype(__cuda_atomic_deduce_minmax_impl<_Type>())::__tag; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/common.h b/libcudacxx/include/cuda/std/__atomic/functions/common.h index 07ece7c4abeb..ca9b967d0e94 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/common.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/common.h @@ -22,11 +22,15 @@ #endif // no system header #include +#include #include _CCCL_BEGIN_NAMESPACE_CUDA_STD +template +using __unv = remove_volatile_t<_Tp>; + template struct __atomic_ptr_skip { diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h index 49e770d646c1..a377ef9e1093 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h @@ -21,6 +21,7 @@ #endif // no system header #include +#include #include #include #include @@ -52,151 +53,177 @@ _CCCL_DEVICE_API inline bool __cuda_atomic_is_local(const volatile void* __ptr) } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_and(volatile _Type& __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_and(_Type __atom, _Type const& __v) { - __atom = __atom & __v; + return __atom & __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_or(volatile _Type& __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_or(_Type __atom, _Type const& __v) { - __atom = __atom | __v; + return __atom | __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_xor(volatile _Type& __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_xor(_Type __atom, _Type const& __v) { - __atom = __atom ^ __v; + return __atom ^ __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_add(volatile _Type& __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_add(_Type __atom, _Type const& __v) { - __atom = __atom + __v; + return __atom + __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_sub(volatile _Type& __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_sub(_Type __atom, _Type const& __v) { - __atom = __atom - __v; + return __atom - __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_max(volatile _Type& __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_max(_Type __atom, _Type const& __v) { - __atom = __atom < __v ? __v : __atom; + return ::cuda::std::__cuda_atomic_less(__atom, __v) ? __v : __atom; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_min(volatile _Type& __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_min(_Type __atom, _Type const& __v) { - __atom = __v < __atom ? __v : __atom; + return ::cuda::std::__cuda_atomic_less(__v, __atom) ? __v : __atom; } -_CCCL_DEVICE_API inline bool __cuda_atomic_load_weak_if_local(const volatile void* __ptr, void* __ret, size_t __size) +template +_CCCL_DEVICE_API bool +__cuda_atomic_load_weak_if_local(const _Type* __ptr, __unv<_Type>* __ret, [[maybe_unused]] size_t __size) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(__ret, const_cast(__ptr), __size); + __atomic_assign_volatile(__ret, *__ptr); // Required to workaround a compiler bug, see nvbug/4064730 NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } -_CCCL_DEVICE_API inline bool __cuda_atomic_store_weak_if_local(volatile void* __ptr, const void* __val, size_t __size) +template +_CCCL_DEVICE_API bool +__cuda_atomic_store_weak_if_local(_Type* __ptr, const __unv<_Type>* __val, [[maybe_unused]] size_t __size) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(const_cast(__ptr), __val, __size); + __atomic_assign_volatile(__ptr, *__val); return true; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_weak_if_local( - volatile _Type* __ptr, _Type* __expected, const _Type* __desired, bool* __success) + _Type* __ptr, __unv<_Type>* __expected, const __unv<_Type>* __desired, bool* __success) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } - if (__atomic_memcmp(const_cast(__ptr), const_cast(__expected), sizeof(_Type)) == 0) + using _ValueType = __unv<_Type>; + _ValueType __old{}; + __atomic_assign_volatile(&__old, *__ptr); + if (__atomic_memcmp(&__old, __expected, sizeof(_ValueType)) == 0) { - ::cuda::std::memcpy(const_cast<_Type*>(__ptr), const_cast<_Type const*>(__desired), sizeof(_Type)); + __atomic_assign_volatile(__ptr, *__desired); *__success = true; } else { - ::cuda::std::memcpy(const_cast<_Type*>(__expected), const_cast<_Type const*>(__ptr), sizeof(_Type)); - *__success = false; + *__expected = __old; + *__success = false; } NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } template -_CCCL_DEVICE_API bool __cuda_atomic_exchange_weak_if_local(volatile _Type* __ptr, _Type* __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_exchange_weak_if_local(_Type* __ptr, __unv<_Type>* __val, __unv<_Type>* __ret) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(const_cast<_Type*>(__ret), const_cast(__ptr), sizeof(_Type)); - ::cuda::std::memcpy(const_cast<_Type*>(__ptr), const_cast(__val), sizeof(_Type)); + __atomic_assign_volatile(__ret, *__ptr); + __atomic_assign_volatile(__ptr, *__val); NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret, _BOp&& __bop) +_CCCL_DEVICE_API bool +__cuda_atomic_fetch_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret, _BOp&& __bop) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(const_cast<_Type*>(__ret), const_cast(__ptr), sizeof(_Type)); - __bop(*__ptr, __val); + using _ValueType = __unv<_Type>; + _ValueType __old{}; + __atomic_assign_volatile(&__old, *__ptr); + *__ret = __old; + const _ValueType __desired = __bop(__old, __val); + __atomic_assign_volatile(__ptr, __desired); NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_and_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_and_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_and<_Type>); + using _ValueType = __unv<_Type>; + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_and<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_or_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_or_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_or<_Type>); + using _ValueType = __unv<_Type>; + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_or<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_xor_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_xor_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_xor<_Type>); + using _ValueType = __unv<_Type>; + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_xor<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_add_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_add_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_add<_Type>); + using _ValueType = __unv<_Type>; + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_add<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_sub_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_sub_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_sub<_Type>); + using _ValueType = __unv<_Type>; + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_sub<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_max_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_max_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_max<_Type>); + using _ValueType = __unv<_Type>; + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_max<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_min_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_min_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_min<_Type>); + using _ValueType = __unv<_Type>; + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_min<_ValueType>); } #endif // _CCCL_CUDA_COMPILATION() diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h new file mode 100644 index 000000000000..22c03a0f37b1 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h @@ -0,0 +1,400 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() + +extern "C" _CCCL_DEVICE void __cuda_atomic_nvvm_cas_128b_unsupported_before_SM_90(); +extern "C" _CCCL_DEVICE void __cuda_atomic_nvvm_exchange_128b_unsupported_before_SM_90(); + +template +struct __cuda_atomic_nvvm_order; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_relaxed> +{ + static constexpr int __value = __NV_ATOMIC_RELAXED; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_release> +{ + static constexpr int __value = __NV_ATOMIC_RELEASE; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_acquire> +{ + static constexpr int __value = __NV_ATOMIC_ACQUIRE; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_acq_rel> +{ + static constexpr int __value = __NV_ATOMIC_ACQ_REL; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_seq_cst> +{ + static constexpr int __value = __NV_ATOMIC_SEQ_CST; +}; + +template +struct __cuda_atomic_nvvm_scope; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_block_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_BLOCK; +}; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_cluster_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_CLUSTER; +}; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_device_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_DEVICE; +}; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_system_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_SYSTEM; +}; + +template +[[nodiscard]] _CCCL_DEVICE_API __unv<_Type>* __cuda_atomic_nvvm_ptr(_Type* __ptr) +{ + return const_cast<__unv<_Type>*>(__ptr); +} + +template +struct __cuda_atomic_nvvm_failure_order +{ + using type = _Order; +}; + +template <> +struct __cuda_atomic_nvvm_failure_order<__cuda_atomic_order_release> +{ + using type = __cuda_atomic_order_relaxed; +}; + +template <> +struct __cuda_atomic_nvvm_failure_order<__cuda_atomic_order_acq_rel> +{ + using type = __cuda_atomic_order_acquire; +}; + +template +struct __cuda_atomic_nvvm_cas_orders +{ + using __success = _Order; + using __failure = typename __cuda_atomic_nvvm_failure_order<_Order>::type; +}; + +template +struct __cuda_atomic_nvvm_cas_orders<__cuda_atomic_cas_order<_Success, _Failure>> +{ + using __success = _Success; + using __failure = _Failure; +}; + +template +_CCCL_DEVICE_API void __cuda_atomic_load( + __cuda_atomic_nvvm_backend, + const _Type* __ptr, + __unv<_Type>& __dst, + _Order, + _Operand, + _Scope __scope, + __cuda_atomic_mmio_disable) +{ + ::__nv_atomic_load(__cuda_atomic_nvvm_ptr(__ptr), + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); +} + +template +_CCCL_DEVICE_API void __cuda_atomic_store( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type> __val, + _Order, + _Operand, + _Scope __scope, + __cuda_atomic_mmio_disable) +{ + ::__nv_atomic_store(__cuda_atomic_nvvm_ptr(__ptr), + &__val, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); +} + +template = 1000), bool> = false> +[[nodiscard]] _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __cmp, + __unv<_Type> __op, + _Cas, + _Order, + _Operand, + _Scope __scope) +{ + using __orders = __cuda_atomic_nvvm_cas_orders<_Order>; + using __success = typename __orders::__success; + using __failure = typename __orders::__failure; + __dst = __cmp; + + if constexpr (_Operand::__size == 128) + { + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_90, + (return ::__nv_atomic_compare_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__dst, + &__op, + __cuda_atomic_cas_is_weak(_Cas{}), + +__cuda_atomic_nvvm_order<__success>::__value, + +__cuda_atomic_nvvm_order<__failure>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value);), + (__cuda_atomic_nvvm_cas_128b_unsupported_before_SM_90(); return false;)) + } + else + { + return ::__nv_atomic_compare_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__dst, + &__op, + __cuda_atomic_cas_is_weak(_Cas{}), + +__cuda_atomic_nvvm_order<__success>::__value, + +__cuda_atomic_nvvm_order<__failure>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); + } +} + +template +_CCCL_DEVICE_API void __cuda_atomic_exchange( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + _Order __order, + _Operand, + _Scope __scope) +{ + if constexpr (_Operand::__size < 32) + { + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_100, + (::__nv_atomic_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__op, + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value);), + (__dst = __cuda_atomic_fetch_update( + __cuda_atomic_nvvm_backend{}, + __ptr, + __cuda_atomic_op_bind<__unv<_Type>, __cuda_atomic_op_store>{__op}, + __order, + _Operand{}, + __scope);)) + } + else if constexpr (_Operand::__size == 128) + { + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_90, + (::__nv_atomic_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__op, + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value);), + (__cuda_atomic_nvvm_exchange_128b_unsupported_before_SM_90();)) + } + else + { + ::__nv_atomic_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__op, + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); + } +} + +# define _CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(_Name) \ + template = 32) || (_CCCL_PTX_ARCH() >= 1000)) \ + && !(is_integral_v<_Type> && is_signed_v<_Type> && sizeof(_Type) == 8), \ + bool> = false> \ + _CCCL_DEVICE_API void __cuda_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order, _Operand, _Scope) \ + { \ + __dst = ::__nv_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_ptr(__ptr), \ + __op, \ + +__cuda_atomic_nvvm_order<_Order>::__value, \ + +__cuda_atomic_nvvm_scope<_Scope>::__value); \ + } \ + \ + template && is_signed_v<_Type> && sizeof(_Type) == 8 && _Operand::__size == 64, \ + bool> = false> \ + _CCCL_DEVICE_API void __cuda_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order, _Operand, _Scope) \ + { \ + const auto __result = ::__nv_atomic_fetch_##_Name( \ + reinterpret_cast(__cuda_atomic_nvvm_ptr(__ptr)), \ + ::cuda::std::bit_cast(__op), \ + +__cuda_atomic_nvvm_order<_Order>::__value, \ + +__cuda_atomic_nvvm_scope<_Scope>::__value); \ + __dst = ::cuda::std::bit_cast<__unv<_Type>>(__result); \ + } + +# define _CCCL_DEFINE_NVVM_FETCH_OP(_Name, _TypeConstraint) \ + template = 32) || (_CCCL_PTX_ARCH() >= 1000)) \ + && (_TypeConstraint), \ + bool> = false> \ + _CCCL_DEVICE_API void __cuda_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order, _Operand, _Scope) \ + { \ + __dst = ::__nv_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_ptr(__ptr), \ + __op, \ + +__cuda_atomic_nvvm_order<_Order>::__value, \ + +__cuda_atomic_nvvm_scope<_Scope>::__value); \ + } + +_CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(add) +_CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(sub) + +template = false> +_CCCL_DEVICE_API void __cuda_atomic_fetch_sub( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + _Order, + __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, _Size>, + _Scope) +{ + __dst = ::__nv_atomic_fetch_add( + __cuda_atomic_nvvm_ptr(__ptr), + -__op, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); +} + +_CCCL_DEFINE_NVVM_FETCH_OP(and, true) +_CCCL_DEFINE_NVVM_FETCH_OP(or, true) +_CCCL_DEFINE_NVVM_FETCH_OP(xor, true) +_CCCL_DEFINE_NVVM_FETCH_OP(min, is_integral_v<_Type>) +_CCCL_DEFINE_NVVM_FETCH_OP(max, is_integral_v<_Type>) + +# undef _CCCL_DEFINE_NVVM_FETCH_ARITHMETIC +# undef _CCCL_DEFINE_NVVM_FETCH_OP + +template +struct __cuda_atomic_nvvm_fence +{ + template + _CCCL_DEVICE_API void operator()(_Order) const + { + ::__nv_atomic_thread_fence(+__cuda_atomic_nvvm_order<_Order>::__value, +__cuda_atomic_nvvm_scope<_Scope>::__value); + } +}; + +template +_CCCL_DEVICE_API void +__cuda_atomic_thread_fence(__cuda_atomic_nvvm_backend __backend, memory_order __order, _Scope __scope) +{ + (void) __backend; + (void) __scope; + __cuda_atomic_nvvm_fence<_Scope> __fence; + switch (__atomic_order_to_int(__order)) + { + case __ATOMIC_RELAXED: + return; + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __fence(__cuda_atomic_order_acquire{}); + case __ATOMIC_RELEASE: + return __fence(__cuda_atomic_order_release{}); + case __ATOMIC_ACQ_REL: + return __fence(__cuda_atomic_order_acq_rel{}); + case __ATOMIC_SEQ_CST: + return __fence(__cuda_atomic_order_seq_cst{}); + default: + _CCCL_ASSERT(false, "invalid fence memory order"); + } +} + +_CCCL_DEVICE_API void __cuda_atomic_signal_fence(__cuda_atomic_nvvm_backend, memory_order) +{ + asm volatile("" ::: "memory"); +} + +#endif // _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm_backend.h new file mode 100644 index 000000000000..e9c9bed3a5d8 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm_backend.h @@ -0,0 +1,86 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __cuda_atomic_nvvm_backend +{ +private: + template + static constexpr bool __has_builtin_subword_rmw = sizeof(_Type) >= 4 || _CCCL_PTX_ARCH() >= 1000; + +public: + template + _CCCL_HOST_DEVICE_API static constexpr _Order __collapse_cas_order(_Order __order) + { + return __order; + } + + template + _CCCL_DEVICE_API static auto + __with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope, _Args... __args) + -> decltype(__fn(__order, __args..., __scope)) + { + return __fn(__order, __args..., __scope); + } + + template + static constexpr bool __use_direct_bitwise = sizeof(_Type) < 16 && __has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_direct_arithmetic = + is_scalar_v<_Type> && sizeof(_Type) < 16 && __has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_direct_minmax = + is_integral_v<_Type> && sizeof(_Type) < 16 && __has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_fallback_bitwise = sizeof(_Type) == 16 || !__has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_fallback_arithmetic = + is_scalar_v<_Type> && (sizeof(_Type) == 16 || !__has_builtin_subword_rmw<_Type>); + + template + static constexpr bool __use_fallback_minmax = + !is_integral_v<_Type> || (is_scalar_v<_Type> && (sizeof(_Type) == 16 || !__has_builtin_subword_rmw<_Type>) ); + + static constexpr bool __needs_constant_order = true; + static constexpr bool __requires_local_memory_workaround = true; + static constexpr size_t __smallest_cas = 32; + static constexpr size_t __widest_cas = 128; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h new file mode 100644 index 000000000000..a1655e596092 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -0,0 +1,150 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +_CCCL_DEVICE_API auto __cuda_atomic_ptx_backend::__with_transformed_order( + _Operation, _Fn& __fn, _Order __order, _Sco __scope, _Args... __args) -> decltype(__fn(__order, __args..., __scope)) +{ + constexpr bool __is_load = is_same_v<_Operation, __cuda_atomic_operation_load>; + constexpr bool __is_store = is_same_v<_Operation, __cuda_atomic_operation_store>; + constexpr bool __is_rmw = is_same_v<_Operation, __cuda_atomic_operation_rmw>; + static_assert(__is_load || __is_store || __is_rmw, "invalid atomic operation class"); + + [[maybe_unused]] constexpr bool __is_seq_cst = is_same_v<_Order, __cuda_atomic_order_seq_cst>; + + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, + ({ + if constexpr (__is_seq_cst) + { + if constexpr (__is_store) + { + return __fn(__cuda_atomic_ptx_order_relaxed{true}, __args..., __scope); + } + else + { + return __fn(__cuda_atomic_ptx_order_acquire{true}, __args..., __scope); + } + } + else + { + return __fn(__transform_order(__order), __args..., __scope); + } + }), + NV_IS_DEVICE, + ({ + constexpr bool __is_release = is_same_v<_Order, __cuda_atomic_order_release>; + constexpr bool __is_acq_rel = is_same_v<_Order, __cuda_atomic_order_acq_rel>; + constexpr bool __is_acquire = is_same_v<_Order, __cuda_atomic_order_acquire>; + constexpr bool __membar_before = + __is_seq_cst || (__is_store && __is_release) || (__is_rmw && (__is_release || __is_acq_rel)); + constexpr bool __membar_after = (__is_load || __is_rmw) && (__is_acquire || __is_acq_rel || __is_seq_cst); + + if constexpr (__membar_before) + { + ::cuda::std::__cuda_atomic_membar(__scope); + } + if constexpr (__membar_after) + { + if constexpr (is_void_v) + { + __fn(__cuda_atomic_order_volatile{}, __args..., __scope); + ::cuda::std::__cuda_atomic_membar(__scope); + return; + } + else + { + auto __result = __fn(__cuda_atomic_order_volatile{}, __args..., __scope); + ::cuda::std::__cuda_atomic_membar(__scope); + return __result; + } + } + else + { + return __fn(__cuda_atomic_order_volatile{}, __args..., __scope); + } + })) +} + +template , bool> = false> +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_ptx_negate(_Type __value) +{ + using __unsigned_type = make_unsigned_t<_Type>; + const auto __bits = ::cuda::std::bit_cast<__unsigned_type>(__value); + const auto __negated = static_cast<__unsigned_type>(__unsigned_type{} - __bits); + return ::cuda::std::bit_cast<_Type>(__negated); +} + +template , bool> = false> +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_ptx_negate(_Type __value) +{ + return -__value; +} + +template = 32) && (_Operand::__size <= 64), bool> = false> +_CCCL_DEVICE_API void __cuda_atomic_fetch_sub( + __cuda_atomic_ptx_backend __backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + _Order __order, + _Operand, + _Sco __scope) +{ + ::cuda::std::__cuda_atomic_fetch_add( + __backend, __ptr, __dst, ::cuda::std::__cuda_atomic_ptx_negate(__op), __order, _Operand{}, __scope); +} + +#if _CCCL_CUDA_COMPILATION() +_CCCL_DEVICE_API inline void __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend, memory_order) +{ + asm volatile("" ::: "memory"); +} +#endif // _CCCL_CUDA_COMPILATION() + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h new file mode 100644 index 000000000000..e795a0c0ea92 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h @@ -0,0 +1,105 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __cuda_atomic_ptx_backend +{ + template + _CCCL_HOST_DEVICE_API static constexpr auto __transform_order(_Order) + { + if constexpr (is_same_v<_Order, __cuda_atomic_order_relaxed>) + { + return __cuda_atomic_ptx_order_relaxed{false}; + } + else if constexpr (is_same_v<_Order, __cuda_atomic_order_release>) + { + return __cuda_atomic_ptx_order_release{false}; + } + else if constexpr (is_same_v<_Order, __cuda_atomic_order_acquire>) + { + return __cuda_atomic_ptx_order_acquire{false}; + } + else + { + static_assert(is_same_v<_Order, __cuda_atomic_order_acq_rel>, "invalid PTX atomic order"); + return __cuda_atomic_ptx_order_acq_rel{false}; + } + } + + template + _CCCL_HOST_DEVICE_API static constexpr _Order __collapse_cas_order(_Order __order) + { + return __order; + } + + template + _CCCL_HOST_DEVICE_API static constexpr auto __collapse_cas_order(__cuda_atomic_cas_order<_Success, _Failure>) + { + if constexpr (is_same_v<_Success, __cuda_atomic_order_seq_cst> || is_same_v<_Failure, __cuda_atomic_order_seq_cst>) + { + return __cuda_atomic_order_seq_cst{}; + } + else if constexpr (is_same_v<_Success, __cuda_atomic_order_acq_rel> + || (is_same_v<_Success, __cuda_atomic_order_release> + && is_same_v<_Failure, __cuda_atomic_order_acquire>) ) + { + return __cuda_atomic_order_acq_rel{}; + } + else if constexpr (is_same_v<_Success, __cuda_atomic_order_release>) + { + return __cuda_atomic_order_release{}; + } + else if constexpr (is_same_v<_Success, __cuda_atomic_order_acquire> + || is_same_v<_Failure, __cuda_atomic_order_acquire>) + { + return __cuda_atomic_order_acquire{}; + } + else + { + return __cuda_atomic_order_relaxed{}; + } + } + + template + _CCCL_DEVICE_API static auto + __with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope, _Args... __args) + -> decltype(__fn(__order, __args..., __scope)); + + static constexpr bool __needs_constant_order = true; + static constexpr bool __requires_local_memory_workaround = true; + static constexpr size_t __smallest_cas = 32; + static constexpr size_t __widest_cas = 128; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h deleted file mode 100644 index 55a1eea5fbb5..000000000000 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h +++ /dev/null @@ -1,447 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of libcu++, the C++ Standard Library for your entire system, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef __CUDA_STD___ATOMIC_FUNCTIONS_DERIVED_H -#define __CUDA_STD___ATOMIC_FUNCTIONS_DERIVED_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include - -#include - -_CCCL_BEGIN_NAMESPACE_CUDA_STD - -#if _CCCL_CUDA_COMPILATION() - -template -using __cuda_atomic_enable_non_native_arithmetic = - enable_if_t<_Operand::__size <= 16 || _Operand::__op == __cuda_atomic_operand::_f, bool>; - -template -using __cuda_atomic_enable_non_native_bitwise = enable_if_t<_Operand::__size <= 16, bool>; - -template -using __cuda_atomic_enable_native_bitwise = enable_if_t<_Operand::__size >= 32, bool>; - -template = 0> -_CCCL_DEVICE_API bool -__cuda_atomic_compare_exchange(_Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, _Order, _Operand, _Sco) -{ - constexpr uint64_t __alignmask = (sizeof(uint32_t) - 1); - constexpr uint32_t __sizemask = (1 << (sizeof(_Type) * 8)) - 1; - uint32_t* __aligned = (uint32_t*) ((intptr_t) __ptr & (~__alignmask)); // NOLINT(performance-no-int-to-ptr) - const uint8_t __offset = uint32_t((intptr_t) __ptr & __alignmask) * 8; - const uint32_t __valueMask = __sizemask << __offset; - const uint32_t __windowMask = ~__valueMask; - const uint32_t __cmpOffset = __cmp << __offset; - const uint32_t __opOffset = __op << __offset; - - // Algorithm for 8b CAS with 32b intrinsics - // __old = __window[0:32] where [__cmp] resides within some offset. - uint32_t __old; - // Start by loading __old with the current value, this optimizes for early return when __cmp is wrong - NV_IF_TARGET( - NV_PROVIDES_SM_70, - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_relaxed{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});), - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_volatile{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});)) - // Reemit CAS instructions until we succeed or the old value is a mismatch - while (__cmpOffset == (__old & __valueMask)) - { - // Combine the desired value and most recently fetched expected masked portion of the window - const uint32_t __attempt = (__old & __windowMask) | __opOffset; - - if (__cuda_atomic_compare_exchange( - __aligned, __old, __old, __attempt, _Order{}, __cuda_atomic_operand_b32{}, _Sco{})) - { - // CAS was successful - return true; - } - } - __dst = static_cast<_Type>(__old >> __offset); - return false; -} - -// Optimized fetch_update CAS loop with op determined after first load reducing waste. -template = 0> -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_update(_Type* __ptr, const _Fn& __op, _Order, _Operand, _Sco) -{ - constexpr uint64_t __alignmask = (sizeof(uint32_t) - 1); - constexpr uint32_t __sizemask = (1 << (sizeof(_Type) * 8)) - 1; - uint32_t* __aligned = (uint32_t*) ((intptr_t) __ptr & (~__alignmask)); // NOLINT(performance-no-int-to-ptr) - const uint8_t __offset = uint8_t((intptr_t) __ptr & __alignmask) * 8; - const uint32_t __valueMask = __sizemask << __offset; - const uint32_t __windowMask = ~__valueMask; - - // 8/16b fetch update is similar to CAS implementation, but compresses the logic for recalculating the operand - // __old = __window[0:32] where [__cmp] resides within some offset. - uint32_t __old; - NV_IF_TARGET( - NV_PROVIDES_SM_70, - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_relaxed{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});), - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_volatile{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});)) - - // Reemit CAS instructions until we succeed - while (true) - { - // Calculate new desired value from last fetched __old - // Use of the value mask is required due to the possibility of overflow when ops are widened. Possible compiler bug? - const uint32_t __attempt = - ((static_cast(__op(static_cast<_Type>(__old >> __offset))) << __offset) & __valueMask) - | (__old & __windowMask); - - if (__cuda_atomic_compare_exchange( - __aligned, __old, __old, __attempt, _Order{}, __cuda_atomic_operand_b32{}, _Sco{})) - { - // CAS was successful - return static_cast<_Type>(__old >> __offset); - } - } -} - -template class _Op> -struct __cuda_atomic_op_bind -{ - _Type __val; - - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __old) const - { - return _Op<_Type>{}(__val, __old); - } -}; - -template -struct __cuda_atomic_op_store -{ - // Just return first value - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __val, _Type) const - { - return __val; - } -}; - -template -struct __cuda_atomic_op_fetch_min -{ - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __op, _Type __old) const - { - return __op < __old ? __op : __old; - } -}; - -template -struct __cuda_atomic_op_fetch_max -{ - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __op, _Type __old) const - { - return __old < __op ? __op : __old; - } -}; - -// Optimized fetch_update CAS loop with op determined after first load reducing waste. -template = 0> -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_update(_Type* __ptr, const _Fn& __op, _Order, _Operand, _Sco) -{ - _Type __expected = 0; - NV_IF_TARGET( - NV_PROVIDES_SM_70, - (__cuda_atomic_load( - __ptr, - __expected, - __cuda_atomic_order_relaxed{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});), - (__cuda_atomic_load( - __ptr, - __expected, - __cuda_atomic_order_volatile{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});)) - - _Type __desired = __op(__expected); - while (!__cuda_atomic_compare_exchange(__ptr, __expected, __expected, __desired, _Order{}, _Operand{}, _Sco{})) - { - __desired = __op(__expected); - } - return __expected; -} - -template = 0> -_CCCL_DEVICE_API void __cuda_atomic_fetch_add(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) -{ - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::plus>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template = 0> -_CCCL_DEVICE_API void __cuda_atomic_fetch_and(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) -{ - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::bit_and>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template = 0> -_CCCL_DEVICE_API void __cuda_atomic_fetch_xor(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) -{ - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::bit_xor>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template = 0> -_CCCL_DEVICE_API void __cuda_atomic_fetch_or(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) -{ - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::bit_or>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template = 0> -_CCCL_DEVICE_API void __cuda_atomic_fetch_min(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) -{ - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_min>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template = 0> -_CCCL_DEVICE_API void __cuda_atomic_fetch_max(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) -{ - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_max>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template = 0> -_CCCL_DEVICE_API void __cuda_atomic_exchange(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) -{ - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::__cuda_atomic_op_store>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_update(_Tp* __ptr, const _Fn& __op, int __memorder, _Sco) -{ - _Tp __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, _Sco{}); - _Tp __desired = __op(__expected); - while (!__cuda_atomic_compare_exchange_dispatch(__ptr, &__expected, __desired, true, __memorder, __memorder, _Sco{})) - { - __desired = __op(__expected); - } - return __expected; -} -template -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_update(_Tp volatile* __ptr, const _Fn& __op, int __memorder, _Sco) -{ - _Tp __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, _Sco{}); - _Tp __desired = __op(__expected); - while (!__cuda_atomic_compare_exchange_dispatch(__ptr, &__expected, __desired, true, __memorder, __memorder, _Sco{})) - { - __desired = __op(__expected); - } - return __expected; -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Tp __atomic_load_n_cuda(const _Tp* __ptr, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_load_cuda(__ptr, __ret, __memorder, _Sco{}); - return __ret; -} -template -[[nodiscard]] _CCCL_DEVICE_API _Tp __atomic_load_n_cuda(const _Tp volatile* __ptr, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_load_cuda(__ptr, __ret, __memorder, _Sco{}); - return __ret; -} - -template -_CCCL_DEVICE_API void __atomic_store_n_cuda(_Tp* __ptr, _Tp __val, int __memorder, _Sco) -{ - __atomic_store_cuda(__ptr, __val, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_store_n_cuda(_Tp volatile* __ptr, _Tp __val, int __memorder, _Sco) -{ - __atomic_store_cuda(__ptr, __val, __memorder, _Sco{}); -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Tp __atomic_exchange_n_cuda(_Tp* __ptr, _Tp __val, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_exchange_cuda(__ptr, __ret, __val, __memorder, _Sco{}); - return __ret; -} -template -[[nodiscard]] _CCCL_DEVICE_API _Tp __atomic_exchange_n_cuda(_Tp volatile* __ptr, _Tp __val, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_exchange_cuda(__ptr, __ret, __val, __memorder, _Sco{}); - return __ret; -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_add_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::plus>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_add_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::plus>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_and_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_and>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_and_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_and>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_or_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_or>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_or_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_or>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_xor_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_xor>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_xor_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_xor>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_min_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_min>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_min_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_min>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_max_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_max>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Tp __cuda_atomic_fetch_max_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_max>{__val}, __memorder, _Sco{}); -} - -_CCCL_DEVICE_API inline void __cuda_atomic_signal_fence(int) -{ - asm volatile("" ::: "memory"); -} - -#endif // _CCCL_CUDA_COMPILATION() - -_CCCL_END_NAMESPACE_CUDA_STD - -#include - -#endif // __CUDA_STD___ATOMIC_FUNCTIONS_DERIVED_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 0c9e7146ebcb..cab331b147c1 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -27,15 +27,11 @@ #include #include -#include -#include -#include - #include #include #include -#include -#include +#include +#include #include @@ -70,16 +66,32 @@ _CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_system_tag, __cu _CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_system_tag, __cuda_atomic_order_seq_cst) { asm volatile("fence.sc.sys;" ::: "memory"); } +template +_CCCL_DEVICE_API void +__cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco __scope) +{ + if (__order.__was_seq_cst) + { + ::cuda::std::__cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); + } +} + +template +_CCCL_DEVICE_API void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) +{} + template -_CCCL_DEVICE_API void __cuda_atomic_thread_fence(int __memorder, _Sco) { +_CCCL_DEVICE_API void __cuda_atomic_thread_fence( + __cuda_atomic_ptx_backend, memory_order __order, _Sco) { + [[maybe_unused]] const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; + case __ATOMIC_SEQ_CST: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } @@ -90,7 +102,7 @@ _CCCL_DEVICE_API void __cuda_atomic_thread_fence(int __memorder, _Sco) { case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_membar(_Sco{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } @@ -98,918 +110,934 @@ _CCCL_DEVICE_API void __cuda_atomic_thread_fence(int __memorder, _Sco) { ) } -template -_CCCL_DEVICE_API void __cuda_atomic_load_order_dispatch(_Fn &__cuda_load, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1025,8 +1053,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1042,8 +1071,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1059,8 +1089,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1076,8 +1107,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1093,8 +1125,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1110,8 +1143,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1127,8 +1161,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1144,8 +1179,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1161,8 +1197,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1178,8 +1215,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1195,8 +1233,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1212,8 +1251,9 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } template _CCCL_DEVICE_API void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1228,312 +1268,271 @@ _CCCL_DEVICE_API void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } -template -struct __cuda_atomic_bind_load { - const _Type* __ptr; - _Type* __dst; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_load(__ptr, *__dst, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_DEVICE_API void __atomic_load_cuda(const _Type* __ptr, _Type& __dst, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - const __proxy_t* __ptr_proxy = reinterpret_cast(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_load_cuda(const _Type volatile* __ptr, _Type& __dst, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - const __proxy_t* __ptr_proxy = reinterpret_cast(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); -} - -template -_CCCL_DEVICE_API void __cuda_atomic_store_order_dispatch(_Fn &__cuda_store, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_RELEASE: __cuda_store(__cuda_atomic_order_release{}); break; - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_RELEASE: [[fallthrough]]; - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.mmio.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1549,8 +1548,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1566,8 +1566,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1583,8 +1584,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1600,8 +1602,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1617,8 +1620,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1634,8 +1638,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1651,8 +1656,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1668,8 +1674,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1685,8 +1692,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1702,8 +1710,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1719,8 +1728,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1736,8 +1746,9 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } template _CCCL_DEVICE_API void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1752,233 +1763,171 @@ _CCCL_DEVICE_API void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } -template -struct __cuda_atomic_bind_store { - _Type* __ptr; - _Type* __val; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_store(__ptr, *__val, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_DEVICE_API void __atomic_store_cuda(_Type* __ptr, _Type& __val, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_store_cuda(volatile _Type* __ptr, _Type& __val, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); -} - -template -_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch(_Fn& __cuda_cas, int __success_memorder, int __failure_memorder, _Sco) { - bool __res = false; - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __res = __cuda_cas(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __res = __cuda_cas(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) - return __res; -} - template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -1996,8 +1945,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2015,8 +1965,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2034,8 +1985,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2053,8 +2005,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2072,8 +2025,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2091,8 +2045,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2110,8 +2065,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2129,8 +2085,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2148,8 +2105,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2167,8 +2125,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2186,8 +2145,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2205,8 +2165,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2224,8 +2185,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2243,8 +2205,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2262,8 +2225,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2281,8 +2245,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2300,8 +2265,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2319,8 +2285,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2338,8 +2305,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2356,236 +2324,171 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } -template -struct __cuda_atomic_bind_compare_exchange { - _Type* __ptr; - _Type* __exp; - _Type* __des; - - template - _CCCL_DEVICE_API bool operator()(_Atomic_Memorder) { - return __cuda_atomic_compare_exchange(__ptr, *__exp, *__exp, *__des, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template -_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch(_Type* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); - bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); -} -template -_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch(_Type volatile* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); - bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); -} - -template -_CCCL_DEVICE_API void __cuda_atomic_exchange_order_dispatch(_Fn& __cuda_exch, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_exch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_exch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_exch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2603,8 +2506,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2622,8 +2526,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2641,8 +2546,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2660,8 +2566,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2679,8 +2586,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2698,8 +2606,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2717,8 +2626,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2736,8 +2646,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2755,8 +2666,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2774,8 +2686,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2793,8 +2706,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2812,8 +2726,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2831,8 +2746,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2850,8 +2766,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2869,8 +2786,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2888,8 +2806,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2907,8 +2826,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2926,8 +2846,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2945,8 +2866,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } template _CCCL_DEVICE_API void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2963,1870 +2885,1531 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } -template -struct __cuda_atomic_bind_exchange { - _Type* __ptr; - _Type* __old; - _Type* __new; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_exchange(__ptr, *__old, *__new, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template -_CCCL_DEVICE_API void __atomic_exchange_cuda(_Type* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_exchange_cuda(_Type volatile* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); -} - -template -_CCCL_DEVICE_API void __cuda_atomic_fetch_order_dispatch(_Fn& __cuda_fetch, int __memorder, _Sco) { - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_fetch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_fetch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_fetch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } - -template -struct __cuda_atomic_bind_fetch_add { - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_add(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_arithmetic<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_arithmetic<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_add<__proxy_t, __proxy_tag, _Sco> __bound_add{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_add, __memorder, _Sco{}); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_arithmetic<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_arithmetic<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_add<__proxy_t, __proxy_tag, _Sco> __bound_add{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_add, __memorder, _Sco{}); - return __dst; -} + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } - -template -struct __cuda_atomic_bind_fetch_and { - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_and(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_and<__proxy_t, __proxy_tag, _Sco> __bound_and{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_and, __memorder, _Sco{}); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_and<__proxy_t, __proxy_tag, _Sco> __bound_and{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_and, __memorder, _Sco{}); - return __dst; -} + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } - -template -struct __cuda_atomic_bind_fetch_max { - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_max(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_max<__proxy_t, __proxy_tag, _Sco> __bound_max{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_max, __memorder, _Sco{}); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_max<__proxy_t, __proxy_tag, _Sco> __bound_max{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_max, __memorder, _Sco{}); - return __dst; -} + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } - -template -struct __cuda_atomic_bind_fetch_min { - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_min(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_min<__proxy_t, __proxy_tag, _Sco> __bound_min{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_min, __memorder, _Sco{}); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_min<__proxy_t, __proxy_tag, _Sco> __bound_min{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_min, __memorder, _Sco{}); - return __dst; -} + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } - -template -struct __cuda_atomic_bind_fetch_or { - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_or(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_or<__proxy_t, __proxy_tag, _Sco> __bound_or{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_or, __memorder, _Sco{}); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_or<__proxy_t, __proxy_tag, _Sco> __bound_or{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_or, __memorder, _Sco{}); - return __dst; -} + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } - -template -struct __cuda_atomic_bind_fetch_xor { - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_DEVICE_API void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_xor(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); - } -}; -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_xor<__proxy_t, __proxy_tag, _Sco> __bound_xor{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_xor, __memorder, _Sco{}); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_xor<__proxy_t, __proxy_tag, _Sco> __bound_xor{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_xor, __memorder, _Sco{}); - return __dst; -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} -template -[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated_helper.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated_helper.h deleted file mode 100644 index 4217629e38ba..000000000000 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated_helper.h +++ /dev/null @@ -1,184 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of libcu++, the C++ Standard Library for your entire system, -// under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. -// -//===----------------------------------------------------------------------===// - -#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_HELPER_H -#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_HELPER_H - -#include - -#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) -# pragma GCC system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) -# pragma clang system_header -#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) -# pragma system_header -#endif // no system header - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -_CCCL_BEGIN_NAMESPACE_CUDA_STD - -enum class __cuda_atomic_order -{ - _relaxed, - _release, - _acquire, - _acq_rel, - _seq_cst, - _volatile, -}; - -template <__cuda_atomic_order _Order> -using __cuda_atomic_order_tag = integral_constant<__cuda_atomic_order, _Order>; - -using __cuda_atomic_order_relaxed = __cuda_atomic_order_tag<__cuda_atomic_order::_relaxed>; -using __cuda_atomic_order_release = __cuda_atomic_order_tag<__cuda_atomic_order::_release>; -using __cuda_atomic_order_acquire = __cuda_atomic_order_tag<__cuda_atomic_order::_acquire>; -using __cuda_atomic_order_acq_rel = __cuda_atomic_order_tag<__cuda_atomic_order::_acq_rel>; -using __cuda_atomic_order_seq_cst = __cuda_atomic_order_tag<__cuda_atomic_order::_seq_cst>; -using __cuda_atomic_order_volatile = __cuda_atomic_order_tag<__cuda_atomic_order::_volatile>; - -template -using __cuda_atomic_mmio_tag = integral_constant; - -using __cuda_atomic_mmio_enable = __cuda_atomic_mmio_tag; -using __cuda_atomic_mmio_disable = __cuda_atomic_mmio_tag; - -enum class __cuda_atomic_operand -{ - _f, - _s, - _u, - _b, -}; - -template <__cuda_atomic_operand _Op, size_t _Size> -struct __cuda_atomic_operand_tag -{ - static constexpr auto __op = _Op; - static constexpr auto __size = _Size; -}; - -using __cuda_atomic_operand_f8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 8>; -using __cuda_atomic_operand_s8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 8>; -using __cuda_atomic_operand_u8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 8>; -using __cuda_atomic_operand_b8 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 8>; -using __cuda_atomic_operand_f16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 16>; -using __cuda_atomic_operand_s16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 16>; -using __cuda_atomic_operand_u16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 16>; -using __cuda_atomic_operand_b16 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 16>; -using __cuda_atomic_operand_f32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 32>; -using __cuda_atomic_operand_s32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 32>; -using __cuda_atomic_operand_u32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 32>; -using __cuda_atomic_operand_b32 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 32>; -using __cuda_atomic_operand_f64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 64>; -using __cuda_atomic_operand_s64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 64>; -using __cuda_atomic_operand_u64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 64>; -using __cuda_atomic_operand_b64 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 64>; -using __cuda_atomic_operand_f128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, 128>; -using __cuda_atomic_operand_s128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_s, 128>; -using __cuda_atomic_operand_u128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_u, 128>; -using __cuda_atomic_operand_b128 = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, 128>; - -template -struct __cuda_atomic_operand_deduction -{ - using __type = _AtomicType; - using __tag = _OpTag; -}; - -struct _CCCL_ALIGNAS(16) __cuda_atomic_longlong2 -{ - uint64_t __x; - uint64_t __y; -}; - -template -using __cuda_atomic_deduce_bitwise = - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_case<8, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction<__cuda_atomic_longlong2, __cuda_atomic_operand_b128>>>; - -template -using __cuda_atomic_deduce_arithmetic = _If< - is_floating_point_v<_Type>, - _If, - __cuda_atomic_operand_deduction>, - _If, - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>, // There is no - // atom.add.s64 - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>>>; - -template -using __cuda_atomic_deduce_minmax = _If< - is_floating_point_v<_Type>, - _If, - __cuda_atomic_operand_deduction>, - _If, - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>, // atom.min|max.s64 - // supported - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>>>; - -template -using __atomic_enable_if_native_bitwise = enable_if_t<(sizeof(_Type) < 16), bool>; - -template -using __atomic_enable_if_native_arithmetic = enable_if_t && (sizeof(_Type) < 16), bool>; - -template -using __atomic_enable_if_native_minmax = enable_if_t && (sizeof(_Type) < 16), bool>; - -template -using __atomic_enable_if_not_native_bitwise = enable_if_t<(sizeof(_Type) == 16), bool>; - -template -using __atomic_enable_if_not_native_arithmetic = enable_if_t && (sizeof(_Type) == 16), bool>; - -template -using __atomic_enable_if_not_native_minmax = - enable_if_t || (is_scalar_v<_Type> && sizeof(_Type) == 16), bool>; - -_CCCL_END_NAMESPACE_CUDA_STD - -#include - -#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/device_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/device_backend.h new file mode 100644 index 000000000000..07bf2bbcb01e --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/device_backend.h @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_DEVICE_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_DEVICE_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() +# include +#else +# include +#endif + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() +using __cuda_atomic_device_backend = __cuda_atomic_nvvm_backend; +#else +using __cuda_atomic_device_backend = __cuda_atomic_ptx_backend; +#endif + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_DEVICE_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h new file mode 100644 index 000000000000..fdd50df8b4ec --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -0,0 +1,691 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_DISPATCH_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_DISPATCH_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_load_order_dispatch( + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) +{ + if constexpr (!_Backend::__needs_constant_order) + { + __fn(__order, __args..., __scope); + } + else + { + switch (__atomic_order_to_int(__order)) + { + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_relaxed{}, __scope, __args...); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_acquire{}, __scope, __args...); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_seq_cst{}, __scope, __args...); + default: + _CCCL_ASSERT(false, "invalid load memory order"); + } + } +} + +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_store_order_dispatch( + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) +{ + if constexpr (!_Backend::__needs_constant_order) + { + __fn(__order, __args..., __scope); + } + else + { + switch (__atomic_order_to_int(__order)) + { + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_relaxed{}, __scope, __args...); + case __ATOMIC_RELEASE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_release{}, __scope, __args...); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_seq_cst{}, __scope, __args...); + default: + _CCCL_ASSERT(false, "invalid store memory order"); + } + } +} + +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_rmw_order_dispatch( + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) +{ + if constexpr (!_Backend::__needs_constant_order) + { + __fn(__order, __args..., __scope); + } + else + { + switch (__atomic_order_to_int(__order)) + { + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_relaxed{}, __scope, __args...); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acquire{}, __scope, __args...); + case __ATOMIC_RELEASE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_release{}, __scope, __args...); + case __ATOMIC_ACQ_REL: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acq_rel{}, __scope, __args...); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_seq_cst{}, __scope, __args...); + default: + _CCCL_ASSERT(false, "invalid read-modify-write memory order"); + } + } +} + +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_order_dispatch( + _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) +{ + ::cuda::std::__cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); +} + +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_fetch_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) +{ + ::cuda::std::__cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_failure_order_dispatch( + _Backend __backend, _Fn& __fn, int __failure, _Sco __scope, _Args... __args) +{ + switch (__failure) + { + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, + __fn, + __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_relaxed>{}), + __scope, + __args...); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, + __fn, + __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_acquire>{}), + __scope, + __args...); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, + __fn, + __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_seq_cst>{}), + __scope, + __args...); + default: + _CCCL_ASSERT(false, "invalid compare-exchange failure memory order"); + _CCCL_UNREACHABLE(); + } +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( + [[maybe_unused]] _Backend __backend, + _Fn& __fn, + memory_order __success, + memory_order __failure, + _Sco __scope, + _Args... __args) +{ + if constexpr (!_Backend::__needs_constant_order) + { + return __fn(__cuda_atomic_runtime_cas_order{__success, __failure}, __args..., __scope); + } + else + { + const int __failure_order = __atomic_failure_order_to_int(__failure); + switch (__atomic_order_to_int(__success)) + { + case __ATOMIC_RELAXED: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_relaxed>( + __backend, __fn, __failure_order, __scope, __args...); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_acquire>( + __backend, __fn, __failure_order, __scope, __args...); + case __ATOMIC_RELEASE: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_release>( + __backend, __fn, __failure_order, __scope, __args...); + case __ATOMIC_ACQ_REL: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_acq_rel>( + __backend, __fn, __failure_order, __scope, __args...); + case __ATOMIC_SEQ_CST: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_seq_cst>( + __backend, __fn, __failure_order, __scope, __args...); + default: + _CCCL_ASSERT(false, "invalid compare-exchange success memory order"); + _CCCL_UNREACHABLE(); + } + } +} + +template +struct __cuda_atomic_bind_load +{ + _Backend __backend; + const _Type* __ptr; + __unv<_Type>* __dst; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) + { + ::cuda::std::__cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); + } +}; +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_load_dispatch( + _Backend __backend, const _Type* __ptr, __unv<_Type>& __dst, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) + { + return; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{__backend, __ptr_proxy, __dst_proxy}; + __cuda_atomic_load_order_dispatch( + __backend, __bound_load, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) +{ + __unv<_Type> __dst; + ::cuda::std::__cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + return __dst; +} + +template +struct __cuda_atomic_bind_store +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type> __val; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) + { + ::cuda::std::__cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); + } +}; +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __value_type __store = __val; + __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) + { + return; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{__backend, __ptr_proxy, *__val_proxy}; + __cuda_atomic_store_order_dispatch( + __backend, __bound_store, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); +} + +template +struct __cuda_atomic_bind_compare_exchange +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __exp; + __unv<_Type> __cmp; + __unv<_Type> __des; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) + { + return ::cuda::std::__cuda_atomic_compare_exchange( + __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( + _Backend __backend, + _Type* __ptr, + __unv<_Type>* __exp, + __unv<_Type> __des, + _Cas, + memory_order __success, + memory_order __failure, + _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); + __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); +#if _CCCL_CUDA_COMPILATION() + bool __res = false; + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) + { + return __res; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; + return __cuda_atomic_compare_exchange_order_dispatch( + __backend, __bound_compare_swap, __success, __failure, __scope, _Cas{}, __proxy_tag{}); +} + +template +struct __cuda_atomic_bind_exchange +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __old; + __unv<_Type> __new; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); + } +}; +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, memory_order __order, _Sco __scope) +{ + using __value_type _CCCL_NODEBUG = __unv<_Type>; + using __proxy_t _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee _CCCL_NODEBUG = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); + __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) + { + return; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{__backend, __ptr_proxy, __old_proxy, *__new_proxy}; + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_exchange_dispatch(_Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) +{ + using __value_type _CCCL_NODEBUG = __unv<_Type>; + __value_type __old; + ::cuda::std::__cuda_atomic_exchange_dispatch( + __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); + return __old; +} + +template +struct __cuda_atomic_bind_fetch_add +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_add(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_add_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + __op = __op * __atomic_ptr_skip_t<_Type>::__skip; + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_arithmetic_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_add<_Backend, __proxy_pointee> __bound_add{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_and +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_and(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_and_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_and<_Backend, __proxy_pointee> __bound_and{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_max +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_max(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_max_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_max<_Backend, __proxy_pointee> __bound_max{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_min +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_min(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_min_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_min<_Backend, __proxy_pointee> __bound_min{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_or +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_or(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_or_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_or<_Backend, __proxy_pointee> __bound_or{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_xor +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_xor(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_xor_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_xor<_Backend, __proxy_pointee> __bound_xor{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_sub +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type>* __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Order __order, _Operand, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); + } +}; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + constexpr auto __skip = __atomic_ptr_skip_t<__value_type>::__skip; + __op = __op * __skip; + using __proxy_type = __cuda_atomic_deduce_arithmetic_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_type>; + using __proxy_operand = __cuda_atomic_deduce_arithmetic_tag_t<__value_type>; + __value_type __dst{}; + auto* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + auto* __dst_proxy = reinterpret_cast<__proxy_type*>(&__dst); + auto* __op_proxy = reinterpret_cast<__proxy_type*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_sub<_Backend, __proxy_pointee> __bound_fetch_sub{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + ::cuda::std::__cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_sub, __order, __scope, __proxy_operand{}); + return __dst; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_DISPATCH_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic.h b/libcudacxx/include/cuda/std/__atomic/functions/generic.h new file mode 100644 index 000000000000..7d4ecb3415b8 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic.h @@ -0,0 +1,201 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template = false> +_CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange( + _Backend __backend, + _Pointee* __ptr, + __unv<_Pointee>& __dst, + __unv<_Pointee> __cmp, + __unv<_Pointee> __op, + _Cas, + _Order __order, + _Operand, + _Sco __scope) +{ + using _Type = __unv<_Pointee>; + constexpr size_t __rmw_size = _Backend::__smallest_cas; + static_assert(__rmw_size <= _Backend::__widest_cas, "atomic CAS cannot be widened beyond the backend's widest CAS"); + + using __rmw_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, __rmw_size>; + const auto __result = ::cuda::std::__cuda_atomic_rmw( + __backend, + __ptr, + __cuda_atomic_compare_exchange_op<_Type>{__cmp, __op}, + __order, + ::cuda::std::__cuda_atomic_compare_exchange_initial_load_order(__order), + _Operand{}, + __rmw_operand{}, + __scope); + __dst = __result.__old; + return __result.__applied; +} + +template _Backend::__widest_cas), + bool> = false> +_CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange( + _Backend, _Pointee*, __unv<_Pointee>&, __unv<_Pointee>, __unv<_Pointee>, _Cas, _Order, _Operand, _Sco) +{ + static_assert(_Operand::__size < _Backend::__widest_cas, "the backend must provide its widest CAS operation"); + return false; +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_add( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_add>{__op}, __order, _Operand{}, __scope); +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_sub( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_sub>{__op}, __order, _Operand{}, __scope); +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_and( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_and>{__op}, __order, _Operand{}, __scope); +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_or( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_or>{__op}, __order, _Operand{}, __scope); +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_xor( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_xor>{__op}, __order, _Operand{}, __scope); +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_min( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_min>{__op}, __order, _Operand{}, __scope); +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_max( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_max>{__op}, __order, _Operand{}, __scope); +} + +template = false> +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange( + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) +{ + using _ValueType = __unv<_Type>; + __dst = ::cuda::std::__cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_store>{__op}, __order, _Operand{}, __scope); +} +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h new file mode 100644 index 000000000000..234d6b977d76 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h @@ -0,0 +1,331 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_RMW_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_RMW_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +using __cuda_atomic_enable_generic_rmw = enable_if_t<(_Operand::__size <= _Backend::__widest_cas), bool>; + +template +struct __cuda_atomic_rmw_type; + +template <> +struct __cuda_atomic_rmw_type<8> +{ + using type = uint8_t; +}; + +template <> +struct __cuda_atomic_rmw_type<16> +{ + using type = uint16_t; +}; + +template <> +struct __cuda_atomic_rmw_type<32> +{ + using type = uint32_t; +}; + +template <> +struct __cuda_atomic_rmw_type<64> +{ + using type = uint64_t; +}; + +template <> +struct __cuda_atomic_rmw_type<128> +{ + using type = __cuda_atomic_longlong2; +}; + +template +struct __cuda_atomic_rmw_window +{ + using __logical_type = typename __cuda_atomic_rmw_type::type; + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _RmwType __replace(_RmwType __old, _Type __op, uint8_t __offset) + { + constexpr auto __sizemask = (_RmwType{1} << (sizeof(_Type) * 8)) - 1; + const auto __value_mask = __sizemask << __offset; + const auto __op_bits = static_cast<_RmwType>(::cuda::std::bit_cast<__logical_type>(__op)); + return (__old & ~__value_mask) | ((__op_bits << __offset) & __value_mask); + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _Type __extract(_RmwType __old, uint8_t __offset) + { + constexpr auto __sizemask = (_RmwType{1} << (sizeof(_Type) * 8)) - 1; + const auto __old_bits = static_cast<__logical_type>((__old >> __offset) & __sizemask); + return ::cuda::std::bit_cast<_Type>(__old_bits); + } +}; + +template +struct __cuda_atomic_rmw_window<_Type, _RmwType, true> +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API static _RmwType __replace(_RmwType, _Type __op, uint8_t) + { + return ::cuda::std::bit_cast<_RmwType>(__op); + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _Type __extract(_RmwType __old, uint8_t) + { + return ::cuda::std::bit_cast<_Type>(__old); + } +}; + +template +struct __cuda_atomic_rmw_window<_Type, __cuda_atomic_longlong2, false> +{ + static_assert(sizeof(_Type) == sizeof(uint64_t), "only 64-bit atomics can be widened to 128 bits"); + using __logical_type = typename __cuda_atomic_rmw_type::type; + + [[nodiscard]] _CCCL_HOST_DEVICE_API static __cuda_atomic_longlong2 + __replace(__cuda_atomic_longlong2 __old, _Type __op, uint8_t __offset) + { + (__offset == 0 ? __old.__x : __old.__y) = ::cuda::std::bit_cast<__logical_type>(__op); + return __old; + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _Type __extract(__cuda_atomic_longlong2 __old, uint8_t __offset) + { + return ::cuda::std::bit_cast<_Type>(static_cast<__logical_type>(__offset == 0 ? __old.__x : __old.__y)); + } +}; + +template +struct __cuda_atomic_rmw_update +{ + _Type __value; + bool __apply; +}; + +template +struct __cuda_atomic_rmw_result +{ + _Type __old; + bool __applied; +}; + +template +_CCCL_HOST_DEVICE_API __cuda_atomic_rmw_result<__unv<_Pointee>> __cuda_atomic_rmw( + _Backend __backend, + _Pointee* __ptr, + const _Fn& __op, + _Order __order, + _InitialOrder __initial_order, + _Operand, + _RmwOperand, + _Sco __scope) +{ + using _Type = __unv<_Pointee>; + static_assert(_Operand::__op == __cuda_atomic_operand::_b, "generic RMW requires a bitwise operand"); + static_assert(_RmwOperand::__op == __cuda_atomic_operand::_b, "generic RMW requires a bitwise CAS operand"); + static_assert(_Operand::__size <= _RmwOperand::__size, "generic RMW cannot use a narrower CAS operand"); + + using __rmw_type = typename __cuda_atomic_rmw_type<_RmwOperand::__size>::type; + using __rmw_pointee = __copy_cv_t<_Pointee, __rmw_type>; + using __window = __cuda_atomic_rmw_window<_Type, __rmw_type>; + + __rmw_pointee* __aligned; + uint8_t __offset; + if constexpr (sizeof(_Type) == sizeof(__rmw_type)) + { + __aligned = reinterpret_cast<__rmw_pointee*>(__ptr); + __offset = 0; + } + else + { + constexpr uintptr_t __alignmask = sizeof(__rmw_type) - 1; + __aligned = reinterpret_cast<__rmw_pointee*>(reinterpret_cast(__ptr) & ~__alignmask); // NOLINT + __offset = static_cast((reinterpret_cast(__ptr) & __alignmask) * 8); + } + + __rmw_type __old; + __cuda_atomic_load(__backend, __aligned, __old, __initial_order, _RmwOperand{}, __scope, __cuda_atomic_mmio_disable{}); + + while (true) + { + const _Type __logical_old = __window::__extract(__old, __offset); + const __cuda_atomic_rmw_update<_Type> __update = __op(__logical_old); + if (!__update.__apply) + { + return {__logical_old, false}; + } + + const __rmw_type __attempt = __window::__replace(__old, __update.__value, __offset); + if (__cuda_atomic_compare_exchange( + __backend, __aligned, __old, __old, __attempt, __cuda_atomic_cas_weak{}, __order, _RmwOperand{}, __scope)) + { + return {__logical_old, true}; + } + } +} + +template +struct __cuda_atomic_rmw_op +{ + _Fn __op; + + [[nodiscard]] _CCCL_HOST_DEVICE_API __cuda_atomic_rmw_update<_Type> operator()(_Type __old) const + { + return {__op(__old), true}; + } +}; + +template +struct __cuda_atomic_compare_exchange_op +{ + _Type __cmp; + _Type __op; + + [[nodiscard]] _CCCL_HOST_DEVICE_API __cuda_atomic_rmw_update<_Type> operator()(_Type __old) const + { + return {__op, __old == __cmp}; + } +}; + +template class _Op> +struct __cuda_atomic_op_bind +{ + _Type __val; + + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __old) const + { + return _Op<_Type>{}(__val, __old); + } +}; + +template +struct __cuda_atomic_op_store +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __val, _Type) const + { + return __val; + } +}; + +template +struct __cuda_atomic_op_fetch_add +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + if constexpr (is_integral_v<_Type>) + { + using __unsigned_type = make_unsigned_t<_Type>; + const auto __op_bits = ::cuda::std::bit_cast<__unsigned_type>(__op); + const auto __old_bits = ::cuda::std::bit_cast<__unsigned_type>(__old); + return ::cuda::std::bit_cast<_Type>(static_cast<__unsigned_type>(__old_bits + __op_bits)); + } + else + { + return __old + __op; + } + } +}; + +template +struct __cuda_atomic_op_fetch_sub +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + if constexpr (is_integral_v<_Type>) + { + using __unsigned_type = make_unsigned_t<_Type>; + const auto __op_bits = ::cuda::std::bit_cast<__unsigned_type>(__op); + const auto __old_bits = ::cuda::std::bit_cast<__unsigned_type>(__old); + return ::cuda::std::bit_cast<_Type>(static_cast<__unsigned_type>(__old_bits - __op_bits)); + } + else + { + return __old - __op; + } + } +}; + +template +struct __cuda_atomic_op_fetch_min +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + return ::cuda::std::__cuda_atomic_less(__op, __old) ? __op : __old; + } +}; + +template +struct __cuda_atomic_op_fetch_max +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + return ::cuda::std::__cuda_atomic_less(__old, __op) ? __op : __old; + } +}; + +template +_CCCL_HOST_DEVICE_API __unv<_Pointee> +__cuda_atomic_fetch_update(_Backend __backend, _Pointee* __ptr, const _Fn& __op, _Order __order, _Operand, _Sco __scope) +{ + using _Type = __unv<_Pointee>; + static_assert(sizeof(_Type) * 8 == _Operand::__size, "generic RMW requires matching type and operand sizes"); + constexpr size_t __rmw_size = + _Operand::__size < _Backend::__smallest_cas ? _Backend::__smallest_cas : _Operand::__size; + static_assert(__rmw_size <= _Backend::__widest_cas, "generic RMW requires a supported CAS width"); + + using __bitwise_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>; + using __rmw_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, __rmw_size>; + return ::cuda::std::__cuda_atomic_rmw( + __backend, + __ptr, + __cuda_atomic_rmw_op<_Type, _Fn>{__op}, + __order, + ::cuda::std::__cuda_atomic_initial_load_order(__order), + __bitwise_operand{}, + __rmw_operand{}, + __scope) + .__old; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_RMW_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host.h b/libcudacxx/include/cuda/std/__atomic/functions/host.h index 0a20a333ff81..b8ca3782b9f1 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/host.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/host.h @@ -21,12 +21,14 @@ # pragma system_header #endif // no system header +#include #include +#include #include #include #include #include -#include +#include #include @@ -69,166 +71,195 @@ __atomic_alignment_wrapper<_Tp>* __atomic_force_align_host(_Tp* __a) } // Guard ifdef for lock free query in case it is assigned elsewhere (MSVC/CUDA) -inline void __atomic_thread_fence_host(memory_order __order) +inline void __cuda_atomic_thread_fence(__cuda_atomic_host_backend, memory_order __order, __thread_scope_tag) { __atomic_thread_fence(__atomic_order_to_int(__order)); } -inline void __atomic_signal_fence_host(memory_order __order) +inline void __cuda_atomic_signal_fence(__cuda_atomic_host_backend, memory_order __order) { __atomic_signal_fence(__atomic_order_to_int(__order)); } -template -inline void __atomic_store_host(_Tp* __a, _Up __val, memory_order __order) +[[nodiscard]] _CCCL_HOST_API constexpr memory_order __cuda_atomic_failure_order(memory_order __order) { - _LIBCUDACXX_INT128_WARN(_Tp) - __atomic_store(&__atomic_force_align_host(__a)->__atom, &__val, __atomic_order_to_int(__order)); + return __order == memory_order_release + ? memory_order_relaxed + : (__order == memory_order_acq_rel ? memory_order_acquire : __order); } -template -inline auto __atomic_load_host(_Tp* __a, memory_order __order) -> remove_cv_t<_Tp> +template +_CCCL_HOST_API void __cuda_atomic_load( + __cuda_atomic_host_backend, + const _Type* __ptr, + __unv<_Type>& __dst, + memory_order __order, + _Operand, + __thread_scope_tag, + _Mmio) { - _LIBCUDACXX_INT128_WARN(_Tp) - remove_cv_t<_Tp> __ret; - __atomic_load(&__atomic_force_align_host(__a)->__atom, &__ret, __atomic_order_to_int(__order)); - return __ret; + _LIBCUDACXX_INT128_WARN(_Type) + __atomic_load(&__atomic_force_align_host(__ptr)->__atom, &__dst, __atomic_order_to_int(__order)); } -template -inline auto __atomic_exchange_host(_Tp* __a, _Up __val, memory_order __order) -> remove_cv_t<_Tp> +template +_CCCL_HOST_API void __cuda_atomic_store( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __val, + memory_order __order, + _Operand, + __thread_scope_tag, + _Mmio) { - _LIBCUDACXX_INT128_WARN(_Tp) - remove_cv_t<_Tp> __ret; - __atomic_exchange(&__atomic_force_align_host(__a)->__atom, &__val, &__ret, __atomic_order_to_int(__order)); - return __ret; -} - -template -inline bool __atomic_compare_exchange_strong_host( - _Tp* __a, _Up* __expected, _Up __desired, memory_order __success, memory_order __failure) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_compare_exchange( - &__atomic_force_align_host(__a)->__atom, - // This is only alignment wrapped in order to prevent GCC-6 from triggering unused warning - &__atomic_force_align_host(__expected)->__atom, - &__desired, - false, - __atomic_order_to_int(__success), - __atomic_failure_order_to_int(__failure)); + _LIBCUDACXX_INT128_WARN(_Type) + __atomic_store(&__atomic_force_align_host(__ptr)->__atom, &__val, __atomic_order_to_int(__order)); } -template -inline bool __atomic_compare_exchange_weak_host( - _Tp* __a, _Up* __expected, _Up __desired, memory_order __success, memory_order __failure) +template +_CCCL_HOST_API bool __cuda_atomic_compare_exchange( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __cmp, + __unv<_Type> __op, + _Cas __cas, + __cuda_atomic_runtime_cas_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __cmp; return __atomic_compare_exchange( - &__atomic_force_align_host(__a)->__atom, - // This is only alignment wrapped in order to prevent GCC-6 from triggering unused warning - &__atomic_force_align_host(__expected)->__atom, - &__desired, - true, - __atomic_order_to_int(__success), - __atomic_failure_order_to_int(__failure)); + &__atomic_force_align_host(__ptr)->__atom, + // This is only alignment wrapped in order to prevent GCC-6 from triggering an unused warning. + &__atomic_force_align_host(&__dst)->__atom, + &__op, + ::cuda::std::__cuda_atomic_cas_is_weak(__cas), + __atomic_order_to_int(__order.__success), + __atomic_failure_order_to_int(__order.__failure)); } -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_add_host(_Tp* __a, _Td __delta, memory_order __order) +template +_CCCL_HOST_API bool __cuda_atomic_compare_exchange( + __cuda_atomic_host_backend __backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __cmp, + __unv<_Type> __op, + _Cas __cas, + memory_order __order, + _Operand __operand, + __thread_scope_tag __scope) { - _LIBCUDACXX_INT128_WARN(_Tp) - constexpr auto __skip_v = __atomic_ptr_skip_t<_Tp>::__skip; - return __atomic_fetch_add(__a, __delta * __skip_v, __atomic_order_to_int(__order)); + return ::cuda::std::__cuda_atomic_compare_exchange( + __backend, + __ptr, + __dst, + __cmp, + __op, + __cas, + __cuda_atomic_runtime_cas_order{__order, ::cuda::std::__cuda_atomic_failure_order(__order)}, + __operand, + __scope); } -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_add_host(_Tp* __a, _Td __delta, memory_order __order) +template +_CCCL_HOST_API void __cuda_atomic_exchange( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected + __delta; - - while (!__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected + __delta; - } - - return __expected; + _LIBCUDACXX_INT128_WARN(_Type) + __atomic_exchange(&__atomic_force_align_host(__ptr)->__atom, &__op, &__dst, __atomic_order_to_int(__order)); } -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_sub_host(_Tp* __a, _Td __delta, memory_order __order) +template > && (_Operand::__op != __cuda_atomic_operand::_f) + && (_Operand::__size <= 64), + bool> = false> +_CCCL_HOST_API void __cuda_atomic_fetch_add( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) - constexpr auto __skip_v = __atomic_ptr_skip_t<_Tp>::__skip; - return __atomic_fetch_sub(__a, __delta * __skip_v, __atomic_order_to_int(__order)); + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_add(__ptr, __op, __atomic_order_to_int(__order)); } -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_sub_host(_Tp* __a, _Td __delta, memory_order __order) +template > && (_Operand::__op != __cuda_atomic_operand::_f) + && (_Operand::__size <= 64), + bool> = false> +_CCCL_HOST_API void __cuda_atomic_fetch_sub( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected - __delta; - - while (!__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected - __delta; - } - - return __expected; + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_sub(__ptr, __op, __atomic_order_to_int(__order)); } -template -inline remove_cv_t<_Tp> __atomic_fetch_and_host(_Tp* __a, _Td __pattern, memory_order __order) +template = false> +_CCCL_HOST_API void __cuda_atomic_fetch_and( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_fetch_and(__a, __pattern, __atomic_order_to_int(__order)); + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_and(__ptr, __op, __atomic_order_to_int(__order)); } -template -inline remove_cv_t<_Tp> __atomic_fetch_or_host(_Tp* __a, _Td __pattern, memory_order __order) +template = false> +_CCCL_HOST_API void __cuda_atomic_fetch_or( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_fetch_or(__a, __pattern, __atomic_order_to_int(__order)); + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_or(__ptr, __op, __atomic_order_to_int(__order)); } -template -inline remove_cv_t<_Tp> __atomic_fetch_xor_host(_Tp* __a, _Td __pattern, memory_order __order) +template = false> +_CCCL_HOST_API void __cuda_atomic_fetch_xor( + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_fetch_xor(__a, __pattern, __atomic_order_to_int(__order)); -} - -template -inline remove_cv_t<_Tp> __atomic_fetch_max_host(_Tp* __a, _Td __val, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected > __val ? __expected : __val; - - while (__desired == __val && !__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected > __val ? __expected : __val; - } - - return __expected; -} - -template -inline remove_cv_t<_Tp> __atomic_fetch_min_host(_Tp* __a, _Td __val, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected < __val ? __expected : __val; - - while (__desired == __val && !__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected < __val ? __expected : __val; - } - - return __expected; + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_xor(__ptr, __op, __atomic_order_to_int(__order)); } #endif // !_CCCL_COMPILER(NVRTC) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h new file mode 100644 index 000000000000..9057365125d3 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_HOST_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_HOST_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __cuda_atomic_host_backend +{ + static constexpr bool __needs_constant_order = false; + static constexpr bool __requires_local_memory_workaround = false; + static constexpr size_t __smallest_cas = 8; + static constexpr size_t __widest_cas = 128; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_HOST_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/scopes.h b/libcudacxx/include/cuda/std/__atomic/scopes.h index cd65782b6c00..16fe52679af1 100644 --- a/libcudacxx/include/cuda/std/__atomic/scopes.h +++ b/libcudacxx/include/cuda/std/__atomic/scopes.h @@ -42,15 +42,17 @@ enum thread_scope thread_scope_thread = __ATOMIC_THREAD }; -struct __thread_scope_thread_tag +struct __thread_scope_tag {}; -struct __thread_scope_block_tag +struct __thread_scope_thread_tag : __thread_scope_tag {}; -struct __thread_scope_cluster_tag +struct __thread_scope_block_tag : __thread_scope_tag {}; -struct __thread_scope_device_tag +struct __thread_scope_cluster_tag : __thread_scope_tag {}; -struct __thread_scope_system_tag +struct __thread_scope_device_tag : __thread_scope_tag +{}; +struct __thread_scope_system_tag : __thread_scope_tag {}; template diff --git a/libcudacxx/include/cuda/std/__atomic/types/base.h b/libcudacxx/include/cuda/std/__atomic/types/base.h index 538bacb5abc9..f01b62b96938 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/base.h +++ b/libcudacxx/include/cuda/std/__atomic/types/base.h @@ -69,21 +69,26 @@ struct __atomic_storage } }; +#define _CCCL_DISPATCH_ATOMIC_BACKEND(_Fn, ...) \ + NV_DISPATCH_TARGET(NV_IS_DEVICE, \ + (return _Fn(__cuda_atomic_device_backend{}, __VA_ARGS__);), \ + NV_IS_HOST, \ + (return _Fn(__cuda_atomic_host_backend{}, __VA_ARGS__);)) + +#define _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(_Fn, _Scope, ...) \ + NV_DISPATCH_TARGET(NV_IS_DEVICE, \ + (return _Fn(__cuda_atomic_device_backend{}, __VA_ARGS__, _Scope);), \ + NV_IS_HOST, \ + (return _Fn(__cuda_atomic_host_backend{}, __VA_ARGS__, __thread_scope_tag{});)) + _CCCL_HOST_DEVICE_API inline void __atomic_thread_fence_dispatch(memory_order __order) { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__cuda_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order), __thread_scope_system_tag());), - NV_IS_HOST, - (__atomic_thread_fence_host(__order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_thread_fence, __thread_scope_system_tag{}, __order); } _CCCL_HOST_DEVICE_API inline void __atomic_signal_fence_dispatch(memory_order __order) { - NV_DISPATCH_TARGET(NV_IS_DEVICE, - (__cuda_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));), - NV_IS_HOST, - (__atomic_signal_fence_host(__order);)) + _CCCL_DISPATCH_ATOMIC_BACKEND(__cuda_atomic_signal_fence, __order); } template = 0> @@ -95,155 +100,105 @@ _CCCL_HOST_DEVICE_API void __atomic_init_dispatch(_Sto* __a, _Up __val) template = 0> _CCCL_HOST_DEVICE_API void __atomic_store_dispatch(_Sto* __a, _Up __val, memory_order __order, _Sco = {}) { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__atomic_store_n_cuda(__a->get(), __val, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (__atomic_store_host(__a->get(), __val, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_store_dispatch, _Sco{}, __a->get(), __val, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_load_dispatch(const _Sto* __a, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __atomic_load_n_cuda(__a->get(), static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_load_host(__a->get(), __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_load_dispatch, _Sco{}, __a->get(), __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_exchange_dispatch(_Sto* __a, _Up __value, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __atomic_exchange_n_cuda(__a->get(), __value, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_exchange_host(__a->get(), __value, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_exchange_dispatch, _Sco{}, __a->get(), __value, __order); } template = 0> _CCCL_HOST_DEVICE_API bool __atomic_compare_exchange_strong_dispatch( _Sto* __a, _Up* __expected, _Up __val, memory_order __success, memory_order __failure, _Sco = {}) { - bool __result = false; - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__result = __cuda_atomic_compare_exchange_dispatch( - __a->get(), - __expected, - __val, - false, - static_cast<__memory_order_underlying_t>(__success), - static_cast<__memory_order_underlying_t>(__failure), - _Sco{});), - NV_IS_HOST, - (__result = __atomic_compare_exchange_strong_host(__a->get(), __expected, __val, __success, __failure);)) - return __result; + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND( + __cuda_atomic_compare_exchange_dispatch, + _Sco{}, + __a->get(), + __expected, + __val, + __cuda_atomic_cas_strong{}, + __success, + __failure); } template = 0> _CCCL_HOST_DEVICE_API bool __atomic_compare_exchange_weak_dispatch( _Sto* __a, _Up* __expected, _Up __val, memory_order __success, memory_order __failure, _Sco = {}) { - bool __result = false; - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__result = __cuda_atomic_compare_exchange_dispatch( - __a->get(), - __expected, - __val, - true, - static_cast<__memory_order_underlying_t>(__success), - static_cast<__memory_order_underlying_t>(__failure), - _Sco{});), - NV_IS_HOST, - (__result = __atomic_compare_exchange_weak_host(__a->get(), __expected, __val, __success, __failure);)) - return __result; + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND( + __cuda_atomic_compare_exchange_dispatch, + _Sco{}, + __a->get(), + __expected, + __val, + __cuda_atomic_cas_weak{}, + __success, + __failure); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_add_dispatch(_Sto* __a, _Up __delta, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_add_dispatch( - __a->get(), __delta, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_add_host(__a->get(), __delta, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_add_dispatch, _Sco{}, __a->get(), __delta, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_sub_dispatch(_Sto* __a, _Up __delta, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_sub_dispatch( - __a->get(), __delta, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_sub_host(__a->get(), __delta, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_sub_dispatch, _Sco{}, __a->get(), __delta, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_and_dispatch(_Sto* __a, _Up __pattern, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_and_dispatch( - __a->get(), __pattern, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_and_host(__a->get(), __pattern, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_and_dispatch, _Sco{}, __a->get(), __pattern, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_or_dispatch(_Sto* __a, _Up __pattern, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_or_dispatch( - __a->get(), __pattern, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_or_host(__a->get(), __pattern, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_or_dispatch, _Sco{}, __a->get(), __pattern, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_xor_dispatch(_Sto* __a, _Up __pattern, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_xor_dispatch( - __a->get(), __pattern, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_xor_host(__a->get(), __pattern, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_xor_dispatch, _Sco{}, __a->get(), __pattern, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_max_dispatch(_Sto* __a, _Up __val, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_IF_TARGET(NV_IS_DEVICE, - (return __cuda_atomic_fetch_max_dispatch( - __a->get(), __val, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - (return __atomic_fetch_max_host(__a->get(), __val, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_max_dispatch, _Sco{}, __a->get(), __val, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_min_dispatch(_Sto* __a, _Up __val, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_IF_TARGET(NV_IS_DEVICE, - (return __cuda_atomic_fetch_min_dispatch( - __a->get(), __val, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - (return __atomic_fetch_min_host(__a->get(), __val, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_min_dispatch, _Sco{}, __a->get(), __val, __order); } +#undef _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND +#undef _CCCL_DISPATCH_ATOMIC_BACKEND + _CCCL_END_NAMESPACE_CUDA_STD #include diff --git a/libcudacxx/include/cuda/std/__atomic/types/common.h b/libcudacxx/include/cuda/std/__atomic/types/common.h index 6626a8adcb72..6d1fb157a9a5 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/common.h +++ b/libcudacxx/include/cuda/std/__atomic/types/common.h @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include #include @@ -31,6 +33,37 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD +template +_CCCL_HOST_DEVICE_API bool __cuda_atomic_less(_Tp __lhs, _Tp __rhs) +{ + if constexpr (__is_extended_floating_point_v<_Tp> && sizeof(_Tp) == 2) + { +#if _CCCL_HAS_CTK() && _CCCL_CTK_BELOW(12, 2) + // Before CTK 12.2, __hlt is device-only and its bfloat16 overload is unavailable before SM80. +# if _CCCL_HAS_NVBF16() + if constexpr (is_same_v<_Tp, __nv_bfloat16>) + { + // Intentionally unqualified to avoid including . + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_80, (return __hlt(__lhs, __rhs);), (return __bfloat162float(__lhs) < __bfloat162float(__rhs);)) + } + else +# endif // _CCCL_HAS_NVBF16() + { + // Intentionally unqualified to avoid including . + NV_IF_ELSE_TARGET(NV_IS_DEVICE, (return __hlt(__lhs, __rhs);), (return __half2float(__lhs) < __half2float(__rhs);)) + } +#else // ^^^ CTK below 12.2 ^^^ / vvv CTK 12.2 or newer vvv + // Intentionally unqualified to avoid including and . + return __hlt(__lhs, __rhs); +#endif // CTK 12.2 or newer + } + else + { + return __lhs < __rhs; + } +} + enum class __atomic_tag { __atomic_base_tag, diff --git a/libcudacxx/include/cuda/std/__atomic/types/small.h b/libcudacxx/include/cuda/std/__atomic/types/small.h index bd8cf9fa7ce4..7d6df4e56a4c 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/small.h +++ b/libcudacxx/include/cuda/std/__atomic/types/small.h @@ -28,7 +28,6 @@ #include #include #include -#include #include #include @@ -78,30 +77,6 @@ _CCCL_HOST_DEVICE_API _Tp __atomic_small_from_32(__atomic_small_proxy_t<_Tp> __v _CCCL_DIAG_POP -template -_CCCL_HOST_DEVICE_API bool __atomic_small_extended_floating_point_less(_Tp __lhs, _Tp __rhs) -{ -#if _CCCL_HAS_CTK() && _CCCL_CTK_BELOW(12, 2) - // Before CTK 12.2, __hlt is device-only and its bfloat16 overload is unavailable before SM80. -# if _CCCL_HAS_NVBF16() - if constexpr (is_same_v<_Tp, __nv_bfloat16>) - { - // Intentionally unqualified to avoid including . - NV_IF_ELSE_TARGET( - NV_PROVIDES_SM_80, (return __hlt(__lhs, __rhs);), (return __bfloat162float(__lhs) < __bfloat162float(__rhs);)) - } - else -# endif // _CCCL_HAS_NVBF16() - { - // Intentionally unqualified to avoid including . - NV_IF_ELSE_TARGET(NV_IS_DEVICE, (return __hlt(__lhs, __rhs);), (return __half2float(__lhs) < __half2float(__rhs);)) - } -#else // ^^^ CTK below 12.2 ^^^ / vvv CTK 12.2 or newer vvv - // Intentionally unqualified to avoid including and . - return __hlt(__lhs, __rhs); -#endif // CTK 12.2 or newer -} - template struct __atomic_small_storage { @@ -250,7 +225,7 @@ _CCCL_HOST_DEVICE_API auto __atomic_fetch_max_dispatch(_Sto* __a, _Up __val, mem while (true) { const auto __old = __atomic_small_from_32<_Tp>(__expected); - const auto __desired = __atomic_small_extended_floating_point_less(__old, _Tp(__val)) ? _Tp(__val) : __old; + const auto __desired = __cuda_atomic_less(__old, _Tp(__val)) ? _Tp(__val) : __old; if (__atomic_compare_exchange_strong_dispatch( &__a->__a_value, &__expected, __atomic_small_to_32(__desired), __order, __order, _Sco{})) { @@ -278,7 +253,7 @@ _CCCL_HOST_DEVICE_API auto __atomic_fetch_min_dispatch(_Sto* __a, _Up __val, mem while (true) { const auto __old = __atomic_small_from_32<_Tp>(__expected); - const auto __desired = __atomic_small_extended_floating_point_less(_Tp(__val), __old) ? _Tp(__val) : __old; + const auto __desired = __cuda_atomic_less(_Tp(__val), __old) ? _Tp(__val) : __old; if (__atomic_compare_exchange_strong_dispatch( &__a->__a_value, &__expected, __atomic_small_to_32(__desired), __order, __order, _Sco{})) { diff --git a/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt b/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt index df44cd85bc05..e6c0093fa787 100644 --- a/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt +++ b/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt @@ -61,8 +61,42 @@ check_source_compiles( ]] libcudacxx_atomic_codegen_has_int128 ) +check_source_compiles( + CUDA + [[ + #include + + #if defined(__CUDA_ARCH__) \ + && (!_CCCL_CTK_AT_LEAST(13, 5) || !_CCCL_HAS_NV_ATOMIC_BUILTINS()) + # error "the NVVM atomic backend is unavailable" + #endif + + __global__ void test() {} + int main() { test<<<1, 1>>>(); } + ]] + libcudacxx_atomic_codegen_uses_nvvm_backend +) cmake_pop_check_state() +set(atomic_codegen_sass_cas_subword_bitwise_archs) +set(atomic_codegen_sass_native_subword_bitwise_archs) +if (libcudacxx_atomic_codegen_uses_nvvm_backend) + set(atomic_codegen_sass_backend_prefix NVVM) + foreach (arch IN LISTS atomic_codegen_sass_cuda_archs) + if (arch GREATER_EQUAL 100) + list(APPEND atomic_codegen_sass_native_subword_bitwise_archs ${arch}) + else() + list(APPEND atomic_codegen_sass_cas_subword_bitwise_archs ${arch}) + endif() + endforeach() +else() + set(atomic_codegen_sass_backend_prefix PTX) + set( + atomic_codegen_sass_cas_subword_bitwise_archs + ${atomic_codegen_sass_cuda_archs} + ) +endif() + if (NOT libcudacxx_atomic_codegen_has_int128) message( STATUS @@ -83,12 +117,32 @@ endif() # Restrict FileCheck to the wrapper so instructions in outlined helpers cannot # mask an inlining regression. -libcudacxx_codegen_add_sass_tests( - AGGREGATE_TARGET libcudacxx.test.atomics.sass - TARGET_PREFIX atomic_codegen - ARCHITECTURES ${atomic_codegen_sass_cuda_archs} - DUMP_FUNCTIONS atomic_codegen_test - CHECK_PREFIXES ${atomic_codegen_sass_cuda_version_prefix} - TESTS ${libcudacxx_atomic_codegen_tests} - COMPILE_DEFINITIONS _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1 -) +if (atomic_codegen_sass_cas_subword_bitwise_archs) + libcudacxx_codegen_add_sass_tests( + AGGREGATE_TARGET libcudacxx.test.atomics.sass + TARGET_PREFIX atomic_codegen + ARCHITECTURES ${atomic_codegen_sass_cas_subword_bitwise_archs} + DUMP_FUNCTIONS atomic_codegen_test + CHECK_PREFIXES + ${atomic_codegen_sass_cuda_version_prefix} + ${atomic_codegen_sass_backend_prefix} + CAS_SUBWORD_BITWISE + TESTS ${libcudacxx_atomic_codegen_tests} + COMPILE_DEFINITIONS _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1 + ) +endif() + +if (atomic_codegen_sass_native_subword_bitwise_archs) + libcudacxx_codegen_add_sass_tests( + AGGREGATE_TARGET libcudacxx.test.atomics.sass + TARGET_PREFIX atomic_codegen + ARCHITECTURES ${atomic_codegen_sass_native_subword_bitwise_archs} + DUMP_FUNCTIONS atomic_codegen_test + CHECK_PREFIXES + ${atomic_codegen_sass_cuda_version_prefix} + ${atomic_codegen_sass_backend_prefix} + NATIVE_SUBWORD_BITWISE + TESTS ${libcudacxx_atomic_codegen_tests} + COMPILE_DEFINITIONS _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1 + ) +endif() diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu index d08d4fde5438..3be3efe80f1a 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu @@ -31,8 +31,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} -; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} ; SMXX-DAG: {{.*}}[[SASS_CALC]]{{.*}} ; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} ; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} @@ -41,8 +41,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; RELEASE-DAG: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} -; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} +; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.CAS{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu index ef9688720d2b..0b0eceba24e4 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu @@ -47,11 +47,11 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SM90-PLUS-DAG: {{.*}}HSETP2{{.*}} ; BLOCK-DAG: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; NON_BLOCK-DAG: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; RELEASE-DAG: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu index 8f3342cc1fa3..b166a1f2e513 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu @@ -31,8 +31,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} -; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} ; SMXX-DAG: {{.*}}[[SASS_CALC]]{{.*}} ; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} ; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} @@ -42,8 +42,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.CAS[[SASS_SIZE]]{{.*}} -; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} +; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.CAS[[SASS_SIZE]]{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu index bf43a59c126c..ecd4bc32eacf 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu @@ -47,11 +47,11 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}ISETP.NE{{.*}} +; SMXX-DAG: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.[SU]64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} -; SMXX: {{.*}}@{{!?P[0-9]+}} BRA{{.*}} +; SMXX: {{.*}}@[[RETRY_PRED]] BRA{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; SMXX: {{.*}}RET.ABS.NODEC{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu index dfc202d4ef04..65f59fb95bf8 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu @@ -13,8 +13,28 @@ // %PARAM% TYPE type int8_t:uint8_t:int16_t:uint16_t // %PARAM% OP,FILECHECK_PREFIX_OP op add=fetch_add,add:sub=fetch_sub,sub:min=fetch_min,min:max=fetch_max,max // %PARAM% ORDER,FILECHECK_PREFIX_SEQ_CST,FILECHECK_PREFIX_ACQUIRE,FILECHECK_PREFIX_ORDER order relaxed=mor,non_seq_cst,no_acquire,no_membar:acquire=moa,non_seq_cst,acquire,no_membar:release=more,non_seq_cst,no_acquire,release:acq_rel=moar,non_seq_cst,acquire,release:seq_cst=mosc,seq_cst,acquire,seq_cst -// %FILECHECK% PREFIX_COMBINE non_block,seq_cst // %FILECHECK% PREFIX_COMBINE non_block,acquire +// %FILECHECK% PREFIX_COMBINE ptx,seq_cst +// %FILECHECK% PREFIX_COMBINE ptx,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE ptx,release +// %FILECHECK% PREFIX_COMBINE sm75,nvvm +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm80,nvvm +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm90,nvvm +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,release // clang-format on #include "atomic_codegen_helpers.h" @@ -30,11 +50,15 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SMXX-LABEL: {{[[:space:]]*}}Function : atomic_codegen_test ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; SMXX: {{.*}}LD.E.64{{(\.SYS)?}} [[ATOM_ADDR:R[0-9]+]], {{.*}} +; SM100_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM120_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} @@ -45,14 +69,28 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SUB: {{.*}}{{IADD3|IADD|IMAD\.IADD|PRMT|LOP3\.LUT}} {{R[0-9]+}}, {{.*}}[[SUB_VALUE]]{{.*}} ; MIN: {{.*}}IMNMX{{.*}} ; MAX: {{.*}}IMNMX{{.*}} -; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; PTX_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; PTX_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; PTX_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM80_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM90_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} ; NON_BLOCK: {{.*}}ATOM.E.CAS.STRONG.[[SASS_SCOPE]] PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]], [[EXPECTED]], {{.*}} +; PTX-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]]{{(\.reuse)?}}, [[EXPECTED]]{{(\.reuse)?}}, {{.*}} +; SM75_NVVM-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]]{{(\.reuse)?}}, [[OLD]]{{(\.reuse)?}}, {{.*}} +; SM80_NVVM-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]]{{(\.reuse)?}}, [[OLD]]{{(\.reuse)?}}, {{.*}} +; SM90_NVVM-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]]{{(\.reuse)?}}, [[OLD]]{{(\.reuse)?}}, {{.*}} ; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu index b963b13c438c..fe3909804bf8 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu @@ -33,8 +33,6 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; NON_BLOCK_ACQUIRE: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX: {{.*}}ISETP.NE{{.*}} +; SMXX: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.[SU]64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{AND|OR|XOR}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} +; SMXX: {{.*}}@[[RETRY_PRED]] BRA{{.*}} ; SMXX: {{.*}}RET.ABS.NODEC{{.*}} */ diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu index f649b373a4ec..e0ce1a11242a 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu @@ -11,10 +11,30 @@ // clang-format off // %PARAM% SCOPE,SASS_SCOPE,FILECHECK_PREFIX_SCOPE scope block=tsb,CTA,block:device=tsd,GPU,non_block:system=tss,SYS,non_block // %PARAM% TYPE type int8_t:uint8_t:int16_t:uint16_t -// %PARAM% OP,SASS_OP op and=fetch_and,AND:or=fetch_or,OR:xor=fetch_xor,XOR +// %PARAM% OP,SASS_OP,FILECHECK_PREFIX_BITWISE_OP op and=fetch_and,AND,and:or=fetch_or,OR,or_xor:xor=fetch_xor,XOR,or_xor // %PARAM% ORDER,FILECHECK_PREFIX_SEQ_CST,FILECHECK_PREFIX_ACQUIRE,FILECHECK_PREFIX_ORDER order relaxed=mor,non_seq_cst,no_acquire,no_membar:acquire=moa,non_seq_cst,acquire,no_membar:release=more,non_seq_cst,no_acquire,release:acq_rel=moar,non_seq_cst,acquire,release:seq_cst=mosc,seq_cst,acquire,seq_cst -// %FILECHECK% PREFIX_COMBINE non_block,seq_cst -// %FILECHECK% PREFIX_COMBINE non_block,acquire +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,block +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_block +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_seq_cst +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,seq_cst +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,release +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,no_membar +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_block,acquire +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,no_acquire +// %FILECHECK% PREFIX_COMBINE ptx,cas_subword_bitwise +// %FILECHECK% PREFIX_COMBINE nvvm,cas_subword_bitwise +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,block +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_block +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_seq_cst +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,seq_cst +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,release +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,no_membar +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_block,acquire +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,no_acquire +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,and +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,or_xor // clang-format on #include "atomic_codegen_helpers.h" @@ -24,38 +44,64 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at return atom.OP(value, ORDER); } -// The compiler may fuse the bitwise operation with subword packing, so check -// the CAS protocol without fixing a particular LOP3 truth table. +// The compiler may fuse the bitwise operation with subword packing, so avoid +// fixing a particular LOP3 truth table. +// TODO: Improve PTX codegen for SM100+ to use the native widened and masked +// subword bitwise operations too. // clang-format off /* ; SMXX-LABEL: {{[[:space:]]*}}Function : atomic_codegen_test -; SMXX-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} ; SMXX: {{.*}}LD.E.64{{(\.SYS)?}} [[ATOM_ADDR:R[0-9]+]], {{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; SMXX: {{.*}}LOP3.LUT {{.*}} -; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} -; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} -; SMXX-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} -; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; NON_BLOCK: {{.*}}ATOM.E.CAS.STRONG.[[SASS_SCOPE]] PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]], [[EXPECTED]], {{.*}} -; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} -; SMXX-NOT: {{.*}}ATOM.E.CAS{{.*}} -; SMXX: {{.*}}RET.ABS.NODEC{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE: {{.*}}LOP3.LUT [[CAS_ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; CAS_SUBWORD_BITWISE_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[CAS_ALIGNED_ADDR]]{{(\.64)?\].*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[CAS_ALIGNED_ADDR]]{{(\.64)?\].*}} +; CAS_SUBWORD_BITWISE: {{.*}}LOP3.LUT {{.*}} +; CAS_SUBWORD_BITWISE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; CAS_SUBWORD_BITWISE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[CAS_ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK: {{.*}}ATOM.E.CAS.STRONG.[[SASS_SCOPE]] PT, [[OLD:R[0-9]+]], {{\[}}[[CAS_ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} +; PTX_CAS_SUBWORD_BITWISE-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]], [[EXPECTED]], {{.*}} +; NVVM_CAS_SUBWORD_BITWISE-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]], [[OLD]], {{.*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; CAS_SUBWORD_BITWISE: {{.*}}RET.ABS.NODEC{{.*}} + +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; NATIVE_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_RELEASE-DAG: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; NATIVE_SUBWORD_BITWISE_SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} +; NATIVE_SUBWORD_BITWISE-DAG: {{.*}}LOP3.LUT [[NATIVE_ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; NATIVE_SUBWORD_BITWISE-DAG: {{.*}}LOP3.LUT [[SHIFT:R[0-9]+]], {{.*}}, 0x18, {{.*}} +; NATIVE_SUBWORD_BITWISE_AND-DAG: {{.*}}SHF.L.U32 [[SHIFTED_OPERAND:R[0-9]+]], {{.*}}, [[SHIFT]]{{(\.reuse)?}}, RZ{{.*}} +; NATIVE_SUBWORD_BITWISE_AND-DAG: {{.*}}LOP3.LUT [[NATIVE_OPERAND:R[0-9]+]], [[SHIFTED_OPERAND]], {{.*}} +; NATIVE_SUBWORD_BITWISE_OR_XOR-DAG: {{.*}}SHF.L.U32 [[NATIVE_OPERAND:R[0-9]+]], {{.*}}, [[SHIFT]]{{(\.reuse)?}}, RZ{{.*}} +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; NATIVE_SUBWORD_BITWISE_BLOCK: {{.*}}ATOM.E.[[SASS_OP]].STRONG.{{CTA|SM}} PT, [[NATIVE_OLD:R[0-9]+]], {{.*\[}}[[NATIVE_ALIGNED_ADDR]]{{(\.64)?\].*}}, [[NATIVE_OPERAND]]{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_BLOCK: {{.*}}ATOM.E.[[SASS_OP]].STRONG.[[SASS_SCOPE]] PT, [[NATIVE_OLD:R[0-9]+]], {{.*\[}}[[NATIVE_ALIGNED_ADDR]]{{(\.64)?\].*}}, [[NATIVE_OPERAND]]{{.*}} +; NATIVE_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE-DAG: {{.*}}SHF.R.U32.HI {{R[0-9]+}}, RZ, [[SHIFT]]{{(\.reuse)?}}, [[NATIVE_OLD]]{{.*}} +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; NATIVE_SUBWORD_BITWISE: {{.*}}RET.ABS.NODEC{{.*}} */ // clang-format on diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu index ef4d3c9db3f0..a605c319d7fb 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu @@ -35,8 +35,6 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref @@ -32,16 +72,55 @@ extern "C" __device__ bool atomic_codegen_test(cuda::atomic_ref& at ; SMXX-LABEL: {{[[:space:]]*}}Function : atomic_codegen_test ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} ; SMXX: {{.*}}LD.E.64{{(\.SYS)?}} [[ATOM_ADDR:R[0-9]+]], {{.*}} +; SM100_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM120_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_HALFWORD_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_HALFWORD_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_HALFWORD_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_HALFWORD_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} +; PTX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; NVVM_BYTE-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; SM75_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffd, {{.*}} +; SM80_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffd, {{.*}} +; SM90_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffd, {{.*}} +; SM100_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; SM120_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; SM75_NVVM_HALFWORD_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM80_NVVM_HALFWORD_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_HALFWORD_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_HALFWORD_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; PTX_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; PTX_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; NVVM_BYTE_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; NVVM_BYTE_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM75_NVVM_HALFWORD: {{.*}}LD.E.SYS {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{\].*}} +; SM80_NVVM_HALFWORD: {{.*}}LD.E {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{\].*}} +; SM90_NVVM_HALFWORD: {{.*}}LD.E {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{\].*}} +; SM100_NVVM_HALFWORD_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM100_NVVM_HALFWORD_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM120_NVVM_HALFWORD_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM120_NVVM_HALFWORD_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; PTX_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_BYTE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_BYTE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_BYTE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_HALFWORD_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; PTX_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_BYTE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_BYTE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_BYTE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; PTX_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_BYTE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM80_NVVM_BYTE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM90_NVVM_BYTE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}}{{.*\[}}[[ALIGNED_ADDR]]{{\].*}} diff --git a/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu index ca8f36a06998..3c26e9f2307c 100644 --- a/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu @@ -38,8 +38,6 @@ atomic_codegen_test(cuda::atomic_ref& atom, TYPE& expected ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-NOT: {{.*}}LD.E{{.*}}.STRONG{{.*}} @@ -47,6 +45,8 @@ atomic_codegen_test(cuda::atomic_ref& atom, TYPE& expected ; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; SMXX-NOT: {{.*}}LD.E{{.*}}.STRONG{{.*}} ; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}LD.E{{.*}}.STRONG{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu index 2d13f8cdb053..eb09aeeacc06 100644 --- a/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu @@ -34,13 +34,13 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[A:R[0-9]+]], [[ATOM_ADDR]], 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[E:R[0-9]+]], {{.*\[}}[[A]]{{(\.64)?\].*}} ; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[E:R[0-9]+]], {{.*\[}}[[A]]{{(\.64)?\].*}} ; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[C:R[0-9]+]], {{\[}}[[A]]{{\]}}, [[E]], [[C]]{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu index 8685bcfa9626..7cc36bf3375b 100644 --- a/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu @@ -35,8 +35,6 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref