From 6a883e83d2d075a648c2ce4c6d020b8830876d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Wed, 19 Aug 2026 11:00:08 -0700 Subject: [PATCH 01/24] Reorganize atomic backend implementation headers. --- libcudacxx/codegen/generators/header.h | 2 +- libcudacxx/include/cuda/std/__atomic/functions.h | 2 +- .../functions/{cuda_ptx_generated_helper.h => backend.h} | 6 +++--- .../cuda/std/__atomic/functions/cuda_ptx_generated.h | 2 +- .../__atomic/functions/{cuda_ptx_derived.h => generic.h} | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) rename libcudacxx/include/cuda/std/__atomic/functions/{cuda_ptx_generated_helper.h => backend.h} (97%) rename libcudacxx/include/cuda/std/__atomic/functions/{cuda_ptx_derived.h => generic.h} (99%) diff --git a/libcudacxx/codegen/generators/header.h b/libcudacxx/codegen/generators/header.h index 41a9b5cdf723..303bdc163be5 100644 --- a/libcudacxx/codegen/generators/header.h +++ b/libcudacxx/codegen/generators/header.h @@ -51,7 +51,7 @@ inline void FormatHeader(std::ostream& out) #include #include #include -#include +#include #include #include diff --git a/libcudacxx/include/cuda/std/__atomic/functions.h b/libcudacxx/include/cuda/std/__atomic/functions.h index b8de1c701762..945c63ad482c 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions.h +++ b/libcudacxx/include/cuda/std/__atomic/functions.h @@ -24,8 +24,8 @@ #include // Device atomics -#include #include +#include // Host atomics #include diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated_helper.h b/libcudacxx/include/cuda/std/__atomic/functions/backend.h similarity index 97% rename from libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated_helper.h rename to libcudacxx/include/cuda/std/__atomic/functions/backend.h index 4217629e38ba..371e57844abf 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated_helper.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/backend.h @@ -8,8 +8,8 @@ // //===----------------------------------------------------------------------===// -#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_HELPER_H -#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_HELPER_H +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_BACKEND_H #include @@ -181,4 +181,4 @@ _CCCL_END_NAMESPACE_CUDA_STD #include -#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 93d447271c2c..6ae61df96e75 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h b/libcudacxx/include/cuda/std/__atomic/functions/generic.h similarity index 99% rename from libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h rename to libcudacxx/include/cuda/std/__atomic/functions/generic.h index 6904430b75f3..d3d3d4f435b9 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_derived.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic.h @@ -8,8 +8,8 @@ // //===----------------------------------------------------------------------===// -#ifndef __CUDA_STD___ATOMIC_FUNCTIONS_DERIVED_H -#define __CUDA_STD___ATOMIC_FUNCTIONS_DERIVED_H +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_H #include @@ -451,4 +451,4 @@ _CCCL_END_NAMESPACE_CUDA_STD #include -#endif // __CUDA_STD___ATOMIC_FUNCTIONS_DERIVED_H +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_H From 5443eadca5eca0db1e7fc4d71a524dc9b0f696b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Thu, 27 Aug 2026 14:19:39 -0700 Subject: [PATCH 02/24] Refactor the PTX atomic backend. --- .../codegen/generators/compare_and_swap.h | 76 +- libcudacxx/codegen/generators/exchange.h | 76 +- libcudacxx/codegen/generators/fence.h | 4 +- libcudacxx/codegen/generators/fetch_ops.h | 82 +- libcudacxx/codegen/generators/header.h | 4 +- libcudacxx/codegen/generators/ld_st.h | 142 +- .../include/cuda/std/__atomic/functions.h | 8 +- .../cuda/std/__atomic/functions/backend.h | 271 ++- .../cuda/std/__atomic/functions/cuda_ptx.h | 62 + .../std/__atomic/functions/cuda_ptx_backend.h | 65 + .../__atomic/functions/cuda_ptx_generated.h | 2165 +++++++++-------- .../cuda/std/__atomic/functions/dispatch.h | 451 ++++ .../cuda/std/__atomic/functions/generic.h | 509 ++-- .../cuda/std/__atomic/functions/generic_rmw.h | 325 +++ 14 files changed, 2755 insertions(+), 1485 deletions(-) create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/dispatch.h create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index 1af672be52ca..67c5a674b99d 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -20,7 +20,10 @@ inline void FormatCompareAndSwap(std::ostream& out) { out << R"XXX( template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_order_dispatch(_Fn& __cuda_cas, int __success_memorder, int __failure_memorder, _Sco) { +[[nodiscard]] static _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_cas, memory_order __success, memory_order __failure, _Sco) { + const int __success_memorder = __atomic_order_to_int(__success); + const int __failure_memorder = __atomic_failure_order_to_int(__failure); bool __res = false; NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( @@ -61,7 +64,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_order_dispatch(_F constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, {4}, __cuda_atomic_operand_{0}{1}, {6}) {{ static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -82,7 +85,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, {4}, __cuda_atomic_operand_{0}{1}, {6}) {{ asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; constexpr Operand supported_types[] = { @@ -158,43 +161,76 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( out << "\n" << R"XXX( -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_compare_exchange { + _Backend __backend; _Type* __ptr; _Type* __exp; _Type* __des; + bool __weak; template - inline _CCCL_DEVICE bool operator()(_Atomic_Memorder) { - return __cuda_atomic_compare_exchange(__ptr, *__exp, *__exp, *__des, _Atomic_Memorder{}, _Tag{}, _Sco{}); + [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { + return __cuda_atomic_compare_exchange( + __backend, __ptr, *__exp, *__exp, *__des, __weak, __order, _Tag{}, _Sco{}); } }; -template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_dispatch(_Type* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( + _Backend __backend, + _Type* __ptr, + _Type* __exp, + _Type __des, + bool __weak, + memory_order __success, + memory_order __failure, + _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} + } + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + return __cuda_atomic_compare_exchange_order_dispatch( + __backend, __bound_compare_swap, __success, __failure, __scope); } -template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_dispatch(_Type volatile* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( + _Backend __backend, + _Type volatile* __ptr, + _Type* __exp, + _Type __des, + bool __weak, + memory_order __success, + memory_order __failure, + _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} + } + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + return __cuda_atomic_compare_exchange_order_dispatch( + __backend, __bound_compare_swap, __success, __failure, __scope); } + +#if _CCCL_CUDA_COMPILATION() )XXX"; } diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index 988b32894054..b59536fd6e4f 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -20,7 +20,9 @@ inline void FormatExchange(std::ostream& out) { out << R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch(_Fn& __cuda_exch, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_exch, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -59,7 +61,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch(_Fn& __cud constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) {{ static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -80,7 +82,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) {{ asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; constexpr Operand supported_types[] = { @@ -156,41 +158,75 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( out << "\n" << R"XXX( -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_exchange { + _Backend __backend; _Type* __ptr; _Type* __old; _Type* __new; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_exchange(__ptr, *__old, *__new, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_exchange(__backend, __ptr, *__old, *__new, __order, _Tag{}, _Sco{}); } }; -template -static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} + } + __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ + __backend, __ptr_proxy, __old_proxy, __new_proxy}; + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); } -template -static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type volatile* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) + +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type volatile* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} + } + __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ + __backend, __ptr_proxy, __old_proxy, __new_proxy}; + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); } + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) +{ + _Type __old; + __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); + return __old; +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) +{ + _Type __old; + __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); + return __old; +} + +#if _CCCL_CUDA_COMPILATION() )XXX"; } diff --git a/libcudacxx/codegen/generators/fence.h b/libcudacxx/codegen/generators/fence.h index cfc221196ce3..df9fe67a0ecd 100644 --- a/libcudacxx/codegen/generators/fence.h +++ b/libcudacxx/codegen/generators/fence.h @@ -78,7 +78,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_fence({0}, {2}) out << "\n" << R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_thread_fence(int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( + __cuda_atomic_ptx_backend, memory_order __order, _Sco) { + [[maybe_unused]] const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index 1975223fcaa9..5e33c302848f 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -53,7 +53,9 @@ inline void FormatFetchOps(std::ostream& out) // Memory order dispatcher out << R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch(_Fn& __cuda_fetch, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_fetch, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -93,56 +95,78 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch(_Fn& __cuda_f constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_fetch_{0}( - _Type* __ptr, _Type& __dst, _Type __op, {5}, __cuda_atomic_operand_{1}{2}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, {5}, __cuda_atomic_operand_{1}{2}, {7}) {{ asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; // 0 - Atomic Operation // 1 - Operand type constraint // 2 - Pointer op skip_v constexpr auto fetch_bind_invoke = R"XXX( -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_fetch_{0} {{ + _Backend __backend; _Type* __ptr; _Type* __dst; _Type* __op; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {{ - __cuda_atomic_fetch_{0}(__ptr, *__dst, *__op, _Atomic_Memorder{{}}, _Tag{{}}, _Sco{{}}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) {{ + __cuda_atomic_fetch_{0}(__backend, __ptr, *__dst, *__op, __order, _Tag{{}}, _Sco{{}}); }} }}; -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_{0}_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_{0}_dispatch( + _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) {{ {2} __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_{1}<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_{1}<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_{1}_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_{1}_tag_t<_Type>; _Type __dst{{}}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} - __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}}; - __cuda_atomic_fetch_order_dispatch(__bound_{0}, __memorder, _Sco{{}}); + if constexpr (_Backend::__requires_local_memory_workaround) + {{ + if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} + }} + __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_{0}{{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope); return __dst; }} -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_{0}_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_{0}_dispatch( + _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) {{ {2} __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_{1}<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_{1}<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_{1}_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_{1}_tag_t<_Type>; _Type __dst{{}}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} - __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}}; - __cuda_atomic_fetch_order_dispatch(__bound_{0}, __memorder, _Sco{{}}); + if constexpr (_Backend::__requires_local_memory_workaround) + {{ + if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} + }} + __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_{0}{{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope); return __dst; }} + )XXX"; constexpr size_t supported_sizes[] = { @@ -165,8 +189,15 @@ template -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_sub_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} -template -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_sub_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} -)XXX"; } #endif // FETCH_OPS_H diff --git a/libcudacxx/codegen/generators/header.h b/libcudacxx/codegen/generators/header.h index 303bdc163be5..76e4e66c8524 100644 --- a/libcudacxx/codegen/generators/header.h +++ b/libcudacxx/codegen/generators/header.h @@ -52,6 +52,8 @@ inline void FormatHeader(std::ostream& out) #include #include #include +#include +#include #include #include @@ -71,8 +73,6 @@ extern "C" _CCCL_DEVICE void __atomic_ldst_128b_unsupported_before_SM_70(); inline void FormatTail(std::ostream& out) { constexpr auto tail = R"XXX( -#endif // _CCCL_CUDA_COMPILATION() - _CCCL_END_NAMESPACE_CUDA_STD #include diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index db703a3e3a24..9c05efa810f4 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -40,7 +40,9 @@ inline void FormatLoad(std::ostream& out) { out << R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch(_Fn &__cuda_load, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_load, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -77,7 +79,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch(_Fn &__cuda_lo constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -95,12 +97,12 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ uint16_t __tmp; asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__tmp) : "l"(__ptr) : "memory"); @@ -212,38 +214,71 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } out << "\n" << R"XXX( -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_load { + _Backend __backend; const _Type* __ptr; _Type* __dst; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_load(__ptr, *__dst, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); } }; -template -static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type* __ptr, _Type& __dst, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, _Type& __dst, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; const __proxy_t* __ptr_proxy = reinterpret_cast(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __backend, __ptr_proxy, __dst_proxy}; + __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); } -template -static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type volatile* __ptr, _Type& __dst, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_load_dispatch( + _Backend __backend, const _Type volatile* __ptr, _Type& __dst, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; const __proxy_t* __ptr_proxy = reinterpret_cast(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __backend, __ptr_proxy, __dst_proxy}; + __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) +{ + _Type __dst; + __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + return __dst; +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_load_dispatch(_Backend __backend, const volatile _Type* __ptr, memory_order __order, _Sco __scope) +{ + _Type __dst; + __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + return __dst; } + +#if _CCCL_CUDA_COMPILATION() )XXX"; } @@ -251,7 +286,9 @@ inline void FormatStore(std::ostream& out) { out << R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch(_Fn &__cuda_store, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_store, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -285,7 +322,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch(_Fn &__cuda_s constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -303,12 +340,12 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ const uint16_t __tmp = static_cast(__val); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__tmp) : "memory"); @@ -416,38 +453,55 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } out << "\n" << R"XXX( -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_store { + _Backend __backend; _Type* __ptr; _Type* __val; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_store(__ptr, *__val, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_store(__backend, __ptr, *__val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; -template -static inline _CCCL_DEVICE void __atomic_store_cuda(_Type* __ptr, _Type& __val, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); + _Type __store = __val; + __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __backend, __ptr_proxy, __val_proxy}; + __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); } -template -static inline _CCCL_DEVICE void __atomic_store_cuda(volatile _Type* __ptr, _Type& __val, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_store_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); + _Type __store = __val; + __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __backend, __ptr_proxy, __val_proxy}; + __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); } + +#if _CCCL_CUDA_COMPILATION() )XXX"; } diff --git a/libcudacxx/include/cuda/std/__atomic/functions.h b/libcudacxx/include/cuda/std/__atomic/functions.h index 945c63ad482c..c69220d36dec 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions.h +++ b/libcudacxx/include/cuda/std/__atomic/functions.h @@ -21,13 +21,7 @@ # pragma system_header #endif // no system header +#include #include -// Device atomics -#include -#include - -// Host atomics -#include - #endif // __CUDA_STD___ATOMIC_FUNCTIONS_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/backend.h b/libcudacxx/include/cuda/std/__atomic/functions/backend.h index 371e57844abf..1ec37ed4d8d1 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/backend.h @@ -21,14 +21,13 @@ # pragma system_header #endif // no system header +#include #include -#include #include #include #include -#include +#include #include -#include #include #include @@ -56,6 +55,88 @@ using __cuda_atomic_order_acq_rel = __cuda_atomic_order_tag<__cuda_atomic_order using __cuda_atomic_order_seq_cst = __cuda_atomic_order_tag<__cuda_atomic_order::_seq_cst>; using __cuda_atomic_order_volatile = __cuda_atomic_order_tag<__cuda_atomic_order::_volatile>; +struct __cuda_atomic_runtime_cas_order +{ + memory_order __success; + memory_order __failure; +}; + +template = 0> +_CCCL_HOST_DEVICE_API void +__cuda_atomic_load_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) +{ + __fn(__order); +} + +template = 0> +_CCCL_HOST_DEVICE_API void +__cuda_atomic_store_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) +{ + __fn(__order); +} + +template = 0> +_CCCL_HOST_DEVICE_API void +__cuda_atomic_exchange_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) +{ + __fn(__order); +} + +template = 0> +_CCCL_HOST_DEVICE_API void +__cuda_atomic_fetch_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) +{ + __fn(__order); +} + +template = 0> +_CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( + _Backend __backend, _Fn& __fn, memory_order __success, memory_order __failure, _Sco __scope) +{ + return __fn(__cuda_atomic_runtime_cas_order{__success, __failure}); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_initial_load_order() +{ + if constexpr (is_same_v<_Order, __cuda_atomic_order_volatile>) + { + return __cuda_atomic_order_volatile{}; + } + else if constexpr (is_same_v<_Order, memory_order> || is_same_v<_Order, __cuda_atomic_runtime_cas_order>) + { + return memory_order_relaxed; + } + else + { + return __cuda_atomic_order_relaxed{}; + } +} + +// Compare-exchange may fail after the initial load without issuing a CAS, so that load must satisfy the failure order. +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __cuda_atomic_compare_exchange_initial_load_order(_Order __order) +{ + if constexpr (is_same_v<_Order, __cuda_atomic_runtime_cas_order>) + { + return __order.__failure; + } + else if constexpr (is_same_v<_Order, memory_order>) + { + return __order == memory_order_release + ? memory_order_relaxed + : (__order == memory_order_acq_rel ? memory_order_acquire : __order); + } + else if constexpr (is_same_v<_Order, __cuda_atomic_order_acquire> || is_same_v<_Order, __cuda_atomic_order_acq_rel>) + { + return __cuda_atomic_order_acquire{}; + } + else + { + return __cuda_atomic_initial_load_order<_Order>(); + } +} + template using __cuda_atomic_mmio_tag = integral_constant; @@ -112,70 +193,158 @@ struct _CCCL_ALIGNAS(16) __cuda_atomic_longlong2 }; template -using __cuda_atomic_deduce_bitwise = - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_case<8, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction<__cuda_atomic_longlong2, __cuda_atomic_operand_b128>>>; +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_bitwise_impl() noexcept +{ + if constexpr (sizeof(_Type) == 1) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 2) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 4) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 8) + { + return __cuda_atomic_operand_deduction{}; + } + else + { + return __cuda_atomic_operand_deduction<__cuda_atomic_longlong2, __cuda_atomic_operand_b128>{}; + } +} +// TODO: Once CUDA 12.0 is no longer supported, factor the repeated decltype below into a common deduction alias. +// CUDA 12.0 cudafe can substitute an unrelated alias for an intermediate alias template in large translation units. template -using __cuda_atomic_deduce_arithmetic = _If< - is_floating_point_v<_Type>, - _If, - __cuda_atomic_operand_deduction>, - _If, - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>, // There is no - // atom.add.s64 - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>>>; +using __cuda_atomic_deduce_bitwise_t = typename decltype(__cuda_atomic_deduce_bitwise_impl<_Type>())::__type; template -using __cuda_atomic_deduce_minmax = _If< - is_floating_point_v<_Type>, - _If, - __cuda_atomic_operand_deduction>, - _If, - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>, // atom.min|max.s64 - // supported - __type_switch>, - __type_case<2, __cuda_atomic_operand_deduction>, - __type_case<4, __cuda_atomic_operand_deduction>, - __type_default<__cuda_atomic_operand_deduction>>>>; +using __cuda_atomic_deduce_bitwise_tag_t = typename decltype(__cuda_atomic_deduce_bitwise_impl<_Type>())::__tag; template -using __atomic_enable_if_native_bitwise = enable_if_t<(sizeof(_Type) < 16), bool>; +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_arithmetic_impl() noexcept +{ + if constexpr (is_floating_point_v<_Type>) + { + if constexpr (sizeof(_Type) == 4) + { + return __cuda_atomic_operand_deduction{}; + } + else + { + return __cuda_atomic_operand_deduction{}; + } + } + else if constexpr (is_signed_v<_Type>) + { + if constexpr (sizeof(_Type) == 1) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 2) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 4) + { + return __cuda_atomic_operand_deduction{}; + } + else + { + // There is no atom.add.s64. + return __cuda_atomic_operand_deduction{}; + } + } + else + { + if constexpr (sizeof(_Type) == 1) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 2) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 4) + { + return __cuda_atomic_operand_deduction{}; + } + else + { + return __cuda_atomic_operand_deduction{}; + } + } +} template -using __atomic_enable_if_native_arithmetic = enable_if_t && (sizeof(_Type) < 16), bool>; +using __cuda_atomic_deduce_arithmetic_t = typename decltype(__cuda_atomic_deduce_arithmetic_impl<_Type>())::__type; template -using __atomic_enable_if_native_minmax = enable_if_t && (sizeof(_Type) < 16), bool>; +using __cuda_atomic_deduce_arithmetic_tag_t = typename decltype(__cuda_atomic_deduce_arithmetic_impl<_Type>())::__tag; template -using __atomic_enable_if_not_native_bitwise = enable_if_t<(sizeof(_Type) == 16), bool>; +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_minmax_impl() noexcept +{ + if constexpr (is_floating_point_v<_Type>) + { + if constexpr (sizeof(_Type) == 4) + { + return __cuda_atomic_operand_deduction{}; + } + else + { + return __cuda_atomic_operand_deduction{}; + } + } + else if constexpr (is_signed_v<_Type>) + { + if constexpr (sizeof(_Type) == 1) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 2) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 4) + { + return __cuda_atomic_operand_deduction{}; + } + else + { + return __cuda_atomic_operand_deduction{}; + } + } + else + { + if constexpr (sizeof(_Type) == 1) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 2) + { + return __cuda_atomic_operand_deduction{}; + } + else if constexpr (sizeof(_Type) == 4) + { + return __cuda_atomic_operand_deduction{}; + } + else + { + return __cuda_atomic_operand_deduction{}; + } + } +} template -using __atomic_enable_if_not_native_arithmetic = enable_if_t && (sizeof(_Type) == 16), bool>; +using __cuda_atomic_deduce_minmax_t = typename decltype(__cuda_atomic_deduce_minmax_impl<_Type>())::__type; template -using __atomic_enable_if_not_native_minmax = - enable_if_t || (is_scalar_v<_Type> && sizeof(_Type) == 16), bool>; +using __cuda_atomic_deduce_minmax_tag_t = typename decltype(__cuda_atomic_deduce_minmax_impl<_Type>())::__tag; _CCCL_END_NAMESPACE_CUDA_STD diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h new file mode 100644 index 000000000000..62d6f768d382 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template , bool> = false> +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_ptx_negate(_Type __value) +{ + using __unsigned_type = make_unsigned_t<_Type>; + const auto __bits = ::cuda::std::bit_cast<__unsigned_type>(__value); + const auto __negated = static_cast<__unsigned_type>(__unsigned_type{} - __bits); + return ::cuda::std::bit_cast<_Type>(__negated); +} + +template , bool> = false> +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_ptx_negate(_Type __value) +{ + return -__value; +} + +template +_CCCL_DEVICE_API void __cuda_atomic_fetch_sub( + __cuda_atomic_ptx_backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) +{ + __cuda_atomic_fetch_add(__backend, __ptr, __dst, __cuda_atomic_ptx_negate(__op), __order, _Operand{}, __scope); +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h new file mode 100644 index 000000000000..62d1d5b000b0 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h @@ -0,0 +1,65 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __cuda_atomic_ptx_backend +{ + template + using __enable_if_direct_bitwise = enable_if_t<(sizeof(_Type) < 16), bool>; + + template + using __enable_if_direct_arithmetic = enable_if_t && (sizeof(_Type) < 16), bool>; + + template + using __enable_if_direct_minmax = enable_if_t && (sizeof(_Type) < 16), bool>; + + template + using __enable_if_fallback_bitwise = enable_if_t<(sizeof(_Type) == 16), bool>; + + template + using __enable_if_fallback_arithmetic = enable_if_t && (sizeof(_Type) == 16), bool>; + + template + using __enable_if_fallback_minmax = + enable_if_t || (is_scalar_v<_Type> && sizeof(_Type) == 16), bool>; + + static constexpr bool __needs_constant_order = true; + static constexpr bool __requires_local_memory_workaround = true; + static constexpr size_t __smallest_cas = 32; + static constexpr size_t __widest_cas = 128; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 6ae61df96e75..0e063698d47f 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -35,6 +35,8 @@ #include #include #include +#include +#include #include #include @@ -71,7 +73,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_system_tag, _ { asm volatile("fence.sc.sys;" ::: "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_thread_fence(int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( + __cuda_atomic_ptx_backend, memory_order __order, _Sco) { + [[maybe_unused]] const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -99,7 +103,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_thread_fence(int __memorder, _Sco) } template -static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch(_Fn &__cuda_load, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_load, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -124,7 +130,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch(_Fn &__cuda_lo template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -132,7 +138,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -140,7 +146,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -148,7 +154,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -156,7 +162,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -164,7 +170,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -172,7 +178,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -180,7 +186,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -188,7 +194,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -196,7 +202,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -204,7 +210,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -212,7 +218,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -220,7 +226,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -228,7 +234,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -236,7 +242,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -244,7 +250,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -252,7 +258,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -260,7 +266,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -268,7 +274,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -276,7 +282,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -284,7 +290,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -292,7 +298,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -300,7 +306,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -308,7 +314,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -316,7 +322,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -324,7 +330,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -332,7 +338,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -340,7 +346,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -348,7 +354,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -356,7 +362,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.acquire.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -364,7 +370,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -372,7 +378,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -380,7 +386,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -388,7 +394,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -396,7 +402,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -404,7 +410,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -412,7 +418,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -420,7 +426,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -428,7 +434,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); @@ -436,579 +442,579 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1025,7 +1031,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1042,7 +1048,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1059,7 +1065,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1076,7 +1082,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1093,7 +1099,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1110,7 +1116,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1127,7 +1133,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1144,7 +1150,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1161,7 +1167,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1178,7 +1184,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1195,7 +1201,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1212,7 +1218,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1228,41 +1234,76 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_load { + _Backend __backend; const _Type* __ptr; _Type* __dst; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_load(__ptr, *__dst, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); } }; -template -static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type* __ptr, _Type& __dst, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, _Type& __dst, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; const __proxy_t* __ptr_proxy = reinterpret_cast(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __backend, __ptr_proxy, __dst_proxy}; + __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); } -template -static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type volatile* __ptr, _Type& __dst, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_load_dispatch( + _Backend __backend, const _Type volatile* __ptr, _Type& __dst, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; const __proxy_t* __ptr_proxy = reinterpret_cast(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__bound_load, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __backend, __ptr_proxy, __dst_proxy}; + __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) +{ + _Type __dst; + __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + return __dst; +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_load_dispatch(_Backend __backend, const volatile _Type* __ptr, memory_order __order, _Sco __scope) +{ + _Type __dst; + __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + return __dst; } +#if _CCCL_CUDA_COMPILATION() + template -static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch(_Fn &__cuda_store, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_store, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -1285,254 +1326,254 @@ static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch(_Fn &__cuda_s template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.release.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.release.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.mmio.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1549,7 +1590,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1566,7 +1607,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1583,7 +1624,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1600,7 +1641,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1617,7 +1658,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1634,7 +1675,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1651,7 +1692,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1668,7 +1709,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1685,7 +1726,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1702,7 +1743,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1719,7 +1760,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1736,7 +1777,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1752,41 +1793,61 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_store { + _Backend __backend; _Type* __ptr; _Type* __val; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_store(__ptr, *__val, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_store(__backend, __ptr, *__val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; -template -static inline _CCCL_DEVICE void __atomic_store_cuda(_Type* __ptr, _Type& __val, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); + _Type __store = __val; + __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __backend, __ptr_proxy, __val_proxy}; + __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); } -template -static inline _CCCL_DEVICE void __atomic_store_cuda(volatile _Type* __ptr, _Type& __val, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_store_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val); - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}} - __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{__ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__bound_store, __memorder, _Sco{}); + _Type __store = __val; + __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} + } + __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __backend, __ptr_proxy, __val_proxy}; + __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); } +#if _CCCL_CUDA_COMPILATION() + template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_order_dispatch(_Fn& __cuda_cas, int __success_memorder, int __failure_memorder, _Sco) { +[[nodiscard]] static _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_cas, memory_order __success, memory_order __failure, _Sco) { + const int __success_memorder = __atomic_order_to_int(__success); + const int __failure_memorder = __atomic_failure_order_to_int(__failure); bool __res = false; NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( @@ -1817,167 +1878,167 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_order_dispatch(_F template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -1996,7 +2057,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2015,7 +2076,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2034,7 +2095,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2053,7 +2114,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2072,7 +2133,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2091,7 +2152,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2110,7 +2171,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2129,7 +2190,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2148,7 +2209,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2167,7 +2228,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2186,7 +2247,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2205,7 +2266,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2224,7 +2285,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2243,7 +2304,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2262,7 +2323,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2281,7 +2342,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2300,7 +2361,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2319,7 +2380,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2338,7 +2399,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2356,46 +2417,81 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_compare_exchange { + _Backend __backend; _Type* __ptr; _Type* __exp; _Type* __des; + bool __weak; template - inline _CCCL_DEVICE bool operator()(_Atomic_Memorder) { - return __cuda_atomic_compare_exchange(__ptr, *__exp, *__exp, *__des, _Atomic_Memorder{}, _Tag{}, _Sco{}); + [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { + return __cuda_atomic_compare_exchange( + __backend, __ptr, *__exp, *__exp, *__des, __weak, __order, _Tag{}, _Sco{}); } }; -template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_dispatch(_Type* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( + _Backend __backend, + _Type* __ptr, + _Type* __exp, + _Type __des, + bool __weak, + memory_order __success, + memory_order __failure, + _Sco __scope) +{ + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} + } + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + return __cuda_atomic_compare_exchange_order_dispatch( + __backend, __bound_compare_swap, __success, __failure, __scope); } -template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange_dispatch(_Type volatile* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco) -{ - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( + _Backend __backend, + _Type volatile* __ptr, + _Type* __exp, + _Type __des, + bool __weak, + memory_order __success, + memory_order __failure, + _Sco __scope) +{ + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); bool __res = false; - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} + } + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + return __cuda_atomic_compare_exchange_order_dispatch( + __backend, __bound_compare_swap, __success, __failure, __scope); } +#if _CCCL_CUDA_COMPILATION() + template -static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch(_Fn& __cuda_exch, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_exch, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -2424,167 +2520,167 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch(_Fn& __cud template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2603,7 +2699,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2622,7 +2718,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2641,7 +2737,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2660,7 +2756,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2679,7 +2775,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2698,7 +2794,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2717,7 +2813,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2736,7 +2832,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2755,7 +2851,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2774,7 +2870,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2793,7 +2889,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2812,7 +2908,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2831,7 +2927,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2850,7 +2946,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2869,7 +2965,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2888,7 +2984,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2907,7 +3003,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2926,7 +3022,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2945,7 +3041,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2963,44 +3059,80 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_exchange { + _Backend __backend; _Type* __ptr; _Type* __old; _Type* __new; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_exchange(__ptr, *__old, *__new, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_exchange(__backend, __ptr, *__old, *__new, __order, _Tag{}, _Sco{}); } }; -template -static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} + } + __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ + __backend, __ptr_proxy, __old_proxy, __new_proxy}; + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); } -template -static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type volatile* __ptr, _Type& __old, _Type __new, int __memorder, _Sco) + +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type volatile* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) { - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}} - __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} + } + __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ + __backend, __ptr_proxy, __old_proxy, __new_proxy}; + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); } +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) +{ + _Type __old; + __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); + return __old; +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) +{ + _Type __old; + __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); + return __old; +} + +#if _CCCL_CUDA_COMPILATION() + template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch(_Fn& __cuda_fetch, int __memorder, _Sco) { +static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch( + __cuda_atomic_ptx_backend, _Fn& __cuda_fetch, memory_order __order, _Sco) { + const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { @@ -3029,1806 +3161,1935 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch(_Fn& __cuda_f template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag) { asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag) { asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_block_tag) { asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_block_tag) { asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag) { asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag) { asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag) { asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_device_tag) { asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_device_tag) { asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag) { asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag) { asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag) { asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_system_tag) { asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_system_tag) { asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag) { asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag) { asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag) { asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_block_tag) { asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_block_tag) { asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag) { asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag) { asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag) { asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_device_tag) { asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_device_tag) { asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag) { asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag) { asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag) { asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_system_tag) { asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_system_tag) { asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag) { asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_fetch_add { + _Backend __backend; _Type* __ptr; _Type* __dst; _Type* __op; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_add(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_fetch_add(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); } }; -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_add_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch( + _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_arithmetic<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_arithmetic<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_arithmetic_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_add<__proxy_t, __proxy_tag, _Sco> __bound_add{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_add, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_add<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_add{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope); return __dst; } -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_add_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch( + _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_arithmetic<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_arithmetic<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_arithmetic_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_add<__proxy_t, __proxy_tag, _Sco> __bound_add{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_add, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_add<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_add{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope); return __dst; } + +#if _CCCL_CUDA_COMPILATION() + template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_fetch_and { + _Backend __backend; _Type* __ptr; _Type* __dst; _Type* __op; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_and(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_fetch_and(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); } }; -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_and_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch( + _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_and<__proxy_t, __proxy_tag, _Sco> __bound_and{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_and, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_and<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_and{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope); return __dst; } -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_and_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch( + _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_and<__proxy_t, __proxy_tag, _Sco> __bound_and{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_and, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_and<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_and{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope); return __dst; } + +#if _CCCL_CUDA_COMPILATION() + template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_fetch_max { + _Backend __backend; _Type* __ptr; _Type* __dst; _Type* __op; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_max(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_fetch_max(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); } }; -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_max_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch( + _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_max<__proxy_t, __proxy_tag, _Sco> __bound_max{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_max, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_max<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_max{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope); return __dst; } -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_max_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch( + _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_max<__proxy_t, __proxy_tag, _Sco> __bound_max{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_max, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_max<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_max{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope); return __dst; } + +#if _CCCL_CUDA_COMPILATION() + template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) { asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) { asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) { asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) { asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) { asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) { asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) { asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) { asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) { asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) { asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) { asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) { asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_fetch_min { + _Backend __backend; _Type* __ptr; _Type* __dst; _Type* __op; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_min(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_fetch_min(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); } }; -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_min_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch( + _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_min<__proxy_t, __proxy_tag, _Sco> __bound_min{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_min, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_min<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_min{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope); return __dst; } -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_min_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch( + _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_minmax<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_minmax<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_min<__proxy_t, __proxy_tag, _Sco> __bound_min{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_min, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_min<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_min{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope); return __dst; } + +#if _CCCL_CUDA_COMPILATION() + template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_fetch_or { + _Backend __backend; _Type* __ptr; _Type* __dst; _Type* __op; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_or(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_fetch_or(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); } }; -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_or_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch( + _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_or<__proxy_t, __proxy_tag, _Sco> __bound_or{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_or, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_or<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_or{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope); return __dst; } -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_or_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch( + _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_or<__proxy_t, __proxy_tag, _Sco> __bound_or{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_or, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_or<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_or{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope); return __dst; } + +#if _CCCL_CUDA_COMPILATION() + template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -template +#endif // _CCCL_CUDA_COMPILATION() + +template struct __cuda_atomic_bind_fetch_xor { + _Backend __backend; _Type* __ptr; _Type* __dst; _Type* __op; template - inline _CCCL_DEVICE void operator()(_Atomic_Memorder) { - __cuda_atomic_fetch_xor(__ptr, *__dst, *__op, _Atomic_Memorder{}, _Tag{}, _Sco{}); + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + __cuda_atomic_fetch_xor(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); } }; -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_xor_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch( + _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_xor<__proxy_t, __proxy_tag, _Sco> __bound_xor{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_xor, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_xor<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_xor{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope); return __dst; } -template = 0> -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_xor_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) +template = 0> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch( + _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) { constexpr auto __skip_v = 1; __op = __op * __skip_v; - using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type; - using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; _Type __dst{}; __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - __cuda_atomic_bind_fetch_xor<__proxy_t, __proxy_tag, _Sco> __bound_xor{__ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__bound_xor, __memorder, _Sco{}); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + } + __cuda_atomic_bind_fetch_xor<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_xor{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope); return __dst; } -template -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_sub_dispatch(_Type* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} -template -[[nodiscard]] static inline _CCCL_DEVICE _Type __cuda_atomic_fetch_sub_dispatch(_Type volatile* __ptr, _Up __op, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_add_dispatch(__ptr, -__op, __memorder, _Sco{}); -} - -#endif // _CCCL_CUDA_COMPILATION() _CCCL_END_NAMESPACE_CUDA_STD diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h new file mode 100644 index 000000000000..089f0d99d02a --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -0,0 +1,451 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_DISPATCH_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_DISPATCH_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +struct __cuda_atomic_bind_fetch_fallback +{ + _Backend __backend; + _Type* __ptr; + _Type* __dst; + _Type* __op; + _Fn __fn; + + template + _CCCL_HOST_DEVICE_API void operator()(_Order __order) + { + __fn(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); + } +}; + +template +struct __cuda_atomic_bind_fetch_sub +{ + _Backend __backend; + _Type* __ptr; + _Type* __dst; + _Type* __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Order __order) + { + __cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); + } +}; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_fallback_dispatch( + _Backend __backend, _Type* __ptr, _Type __op, memory_order __order, _Sco __scope, _Fn __fn) +{ + using __operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, sizeof(_Type) * 8>; + _Type __dst{}; + __cuda_atomic_bind_fetch_fallback<_Backend, _Type, _Fn, __operand, _Sco> __bound_fetch_fallback{ + __backend, __ptr, &__dst, &__op, __fn}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_fallback, __order, __scope); + return __dst; +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + constexpr auto __skip = __atomic_ptr_skip_t<_Type>::__skip; + __op = __op * __skip; + using __proxy_type = __cuda_atomic_deduce_arithmetic_t<_Type>; + using __proxy_operand = __cuda_atomic_deduce_arithmetic_tag_t<_Type>; + _Type __dst{}; + auto* __ptr_proxy = reinterpret_cast<__proxy_type*>(__ptr); + auto* __dst_proxy = reinterpret_cast<__proxy_type*>(&__dst); + auto* __op_proxy = reinterpret_cast<__proxy_type*>(&__op); + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } + __cuda_atomic_bind_fetch_sub<_Backend, __proxy_type, __proxy_operand, _Sco> __bound_fetch_sub{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_sub, __order, __scope); + return __dst; +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_sub_dispatch(__backend, const_cast<_Type*>(__ptr), __op, __order, __scope); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_add_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + constexpr auto __skip = __atomic_ptr_skip_t<_Type>::__skip; + const _Type __op = static_cast<_Type>(__val * __skip); + if constexpr (_Backend::__requires_local_memory_workaround) + { + _Type __dst{}; + if (__cuda_atomic_fetch_add_weak_if_local(__ptr, __op, &__dst)) + { + return __dst; + } + } + return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_add_op{}); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_add_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + constexpr auto __skip = __atomic_ptr_skip_t<_Type>::__skip; + const _Type __op = static_cast<_Type>(__val * __skip); + if constexpr (_Backend::__requires_local_memory_workaround) + { + _Type __dst{}; + if (__cuda_atomic_fetch_sub_weak_if_local(__ptr, __op, &__dst)) + { + return __dst; + } + } + return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_sub_op{}); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_sub_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_and_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + const _Type __op = static_cast<_Type>(__val); + if constexpr (_Backend::__requires_local_memory_workaround) + { + _Type __dst{}; + if (__cuda_atomic_fetch_and_weak_if_local(__ptr, __op, &__dst)) + { + return __dst; + } + } + return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_and_op{}); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_and_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_or_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + const _Type __op = static_cast<_Type>(__val); + if constexpr (_Backend::__requires_local_memory_workaround) + { + _Type __dst{}; + if (__cuda_atomic_fetch_or_weak_if_local(__ptr, __op, &__dst)) + { + return __dst; + } + } + return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_or_op{}); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_or_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_xor_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + const _Type __op = static_cast<_Type>(__val); + if constexpr (_Backend::__requires_local_memory_workaround) + { + _Type __dst{}; + if (__cuda_atomic_fetch_xor_weak_if_local(__ptr, __op, &__dst)) + { + return __dst; + } + } + return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_xor_op{}); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_xor_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_min_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + const _Type __op = static_cast<_Type>(__val); + if constexpr (_Backend::__requires_local_memory_workaround) + { + _Type __dst{}; + if (__cuda_atomic_fetch_min_weak_if_local(__ptr, __op, &__dst)) + { + return __dst; + } + } + return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_min_op{}); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_min_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_max_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + const _Type __op = static_cast<_Type>(__val); + if constexpr (_Backend::__requires_local_memory_workaround) + { + _Type __dst{}; + if (__cuda_atomic_fetch_max_weak_if_local(__ptr, __op, &__dst)) + { + return __dst; + } + } + return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_max_op{}); +} + +template = false> +[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch( + _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + return __cuda_atomic_fetch_max_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +} + +#if _CCCL_CUDA_COMPILATION() +template +_CCCL_DEVICE_API void __atomic_thread_fence_cuda(int __order, _Sco __scope) +{ + __cuda_atomic_thread_fence(__cuda_atomic_ptx_backend{}, static_cast(__order), __scope); +} + +template +[[nodiscard]] _CCCL_DEVICE_API _Type __atomic_load_n_cuda(const _Type* __ptr, int __order, _Sco __scope) +{ + return __cuda_atomic_load_dispatch(__cuda_atomic_ptx_backend{}, __ptr, static_cast(__order), __scope); +} + +template +[[nodiscard]] _CCCL_DEVICE_API _Type __atomic_load_n_cuda(const volatile _Type* __ptr, int __order, _Sco __scope) +{ + return __cuda_atomic_load_dispatch(__cuda_atomic_ptx_backend{}, __ptr, static_cast(__order), __scope); +} + +template +_CCCL_DEVICE_API void __atomic_store_n_cuda(_Type* __ptr, _Type __val, int __order, _Sco __scope) +{ + __cuda_atomic_store_dispatch(__cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); +} + +template +_CCCL_DEVICE_API void __atomic_store_n_cuda(volatile _Type* __ptr, _Type __val, int __order, _Sco __scope) +{ + __cuda_atomic_store_dispatch(__cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); +} + +template +[[nodiscard]] _CCCL_DEVICE_API _Type __atomic_exchange_n_cuda(_Type* __ptr, _Type __val, int __order, _Sco __scope) +{ + return __cuda_atomic_exchange_dispatch( + __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); +} + +template +[[nodiscard]] _CCCL_DEVICE_API _Type +__atomic_exchange_n_cuda(volatile _Type* __ptr, _Type __val, int __order, _Sco __scope) +{ + return __cuda_atomic_exchange_dispatch( + __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); +} + +template +_CCCL_DEVICE_API bool __atomic_compare_exchange_cuda( + _Type* __ptr, _Type* __expected, _Type __desired, bool __weak, int __success, int __failure, _Sco __scope) +{ + return __cuda_atomic_compare_exchange_dispatch( + __cuda_atomic_ptx_backend{}, + __ptr, + __expected, + __desired, + __weak, + static_cast(__success), + static_cast(__failure), + __scope); +} + +template +_CCCL_DEVICE_API bool __atomic_compare_exchange_cuda( + volatile _Type* __ptr, _Type* __expected, _Type __desired, bool __weak, int __success, int __failure, _Sco __scope) +{ + return __cuda_atomic_compare_exchange_dispatch( + __cuda_atomic_ptx_backend{}, + __ptr, + __expected, + __desired, + __weak, + static_cast(__success), + static_cast(__failure), + __scope); +} + +# define _CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(_Op) \ + template \ + [[nodiscard]] _CCCL_DEVICE_API _Type __atomic_fetch_##_Op##_cuda( \ + _Type* __ptr, _Up __val, int __order, _Sco __scope) \ + { \ + return __cuda_atomic_fetch_##_Op##_dispatch( \ + __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); \ + } \ + \ + template \ + [[nodiscard]] _CCCL_DEVICE_API _Type __atomic_fetch_##_Op##_cuda( \ + volatile _Type* __ptr, _Up __val, int __order, _Sco __scope) \ + { \ + return __cuda_atomic_fetch_##_Op##_dispatch( \ + __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); \ + } + +_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(add) +_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(sub) +_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(and) +_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(or) +_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(xor) +_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(min) +_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(max) + +# undef _CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER + +_CCCL_DEVICE static inline void __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend, memory_order) +{ + asm volatile("" ::: "memory"); +} + +_CCCL_DEVICE static inline void __atomic_signal_fence_cuda(int __order) +{ + __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend{}, static_cast(__order)); +} +#endif // _CCCL_CUDA_COMPILATION() + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_DISPATCH_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic.h b/libcudacxx/include/cuda/std/__atomic/functions/generic.h index d3d3d4f435b9..0c3997608d8f 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic.h @@ -4,7 +4,7 @@ // under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. // //===----------------------------------------------------------------------===// @@ -21,432 +21,229 @@ # pragma system_header #endif // no system header -#include +#include #include -#include #include -#include #include #include _CCCL_BEGIN_NAMESPACE_CUDA_STD -#if _CCCL_CUDA_COMPILATION() - -template -using __cuda_atomic_enable_non_native_arithmetic = - enable_if_t<_Operand::__size <= 16 || _Operand::__op == __cuda_atomic_operand::_f, bool>; - -template -using __cuda_atomic_enable_non_native_bitwise = enable_if_t<_Operand::__size <= 16, bool>; - -template -using __cuda_atomic_enable_native_bitwise = enable_if_t<_Operand::__size >= 32, bool>; - -template = 0> -_CCCL_DEVICE static bool -__cuda_atomic_compare_exchange(_Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, _Order, _Operand, _Sco) +template = false> +_CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, _Order __order, _Operand, _Sco __scope) { - constexpr uint64_t __alignmask = (sizeof(uint32_t) - 1); - constexpr uint32_t __sizemask = (1 << (sizeof(_Type) * 8)) - 1; - uint32_t* __aligned = (uint32_t*) ((intptr_t) __ptr & (~__alignmask)); // NOLINT(performance-no-int-to-ptr) - const uint8_t __offset = uint32_t((intptr_t) __ptr & __alignmask) * 8; - const uint32_t __valueMask = __sizemask << __offset; - const uint32_t __windowMask = ~__valueMask; - const uint32_t __cmpOffset = __cmp << __offset; - const uint32_t __opOffset = __op << __offset; - - // Algorithm for 8b CAS with 32b intrinsics - // __old = __window[0:32] where [__cmp] resides within some offset. - uint32_t __old; - // Start by loading __old with the current value, this optimizes for early return when __cmp is wrong - NV_IF_TARGET( - NV_PROVIDES_SM_70, - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_relaxed{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});), - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_volatile{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});)) - // Reemit CAS instructions until we succeed or the old value is a mismatch - while (__cmpOffset == (__old & __valueMask)) - { - // Combine the desired value and most recently fetched expected masked portion of the window - const uint32_t __attempt = (__old & __windowMask) | __opOffset; + constexpr size_t __rmw_size = _Backend::__smallest_cas; + static_assert(__rmw_size <= _Backend::__widest_cas, "atomic CAS cannot be widened beyond the backend's widest CAS"); - if (__cuda_atomic_compare_exchange( - __aligned, __old, __old, __attempt, _Order{}, __cuda_atomic_operand_b32{}, _Sco{})) - { - // CAS was successful - return true; - } - } - __dst = static_cast<_Type>(__old >> __offset); + using __rmw_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, __rmw_size>; + const auto __result = __cuda_atomic_rmw( + __backend, + __ptr, + __cuda_atomic_compare_exchange_op<_Type>{__cmp, __op}, + __order, + __cuda_atomic_compare_exchange_initial_load_order(__order), + _Operand{}, + __rmw_operand{}, + __scope); + __dst = __result.__old; + return __result.__applied; +} + +template _Backend::__widest_cas), + bool> = false> +_CCCL_HOST_DEVICE static bool +__cuda_atomic_compare_exchange(_Backend, _Type*, _Type&, _Type, _Type, bool, _Order, _Operand, _Sco) +{ + static_assert(_Operand::__size < _Backend::__widest_cas, "the backend must provide its widest CAS operation"); return false; } -// Optimized fetch_update CAS loop with op determined after first load reducing waste. -template = 0> -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_update(_Type* __ptr, const _Fn& __op, _Order, _Operand, _Sco) + __cuda_atomic_enable_generic_rmw<_Backend, _Operand> = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_add( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) { - constexpr uint64_t __alignmask = (sizeof(uint32_t) - 1); - constexpr uint32_t __sizemask = (1 << (sizeof(_Type) * 8)) - 1; - uint32_t* __aligned = (uint32_t*) ((intptr_t) __ptr & (~__alignmask)); // NOLINT(performance-no-int-to-ptr) - const uint8_t __offset = uint8_t((intptr_t) __ptr & __alignmask) * 8; - const uint32_t __valueMask = __sizemask << __offset; - const uint32_t __windowMask = ~__valueMask; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_add>{__op}, __order, _Operand{}, __scope); +} - // 8/16b fetch update is similar to CAS implementation, but compresses the logic for recalculating the operand - // __old = __window[0:32] where [__cmp] resides within some offset. - uint32_t __old; - NV_IF_TARGET( - NV_PROVIDES_SM_70, - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_relaxed{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});), - (__cuda_atomic_load( - __aligned, - __old, - __cuda_atomic_order_volatile{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});)) +template = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_sub( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) +{ + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_sub>{__op}, __order, _Operand{}, __scope); +} - // Reemit CAS instructions until we succeed - while (true) +struct __cuda_atomic_fetch_and_op +{ + template + _CCCL_HOST_DEVICE_API void + operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const { - // Calculate new desired value from last fetched __old - // Use of the value mask is required due to the possibility of overflow when ops are widened. Possible compiler bug? - const uint32_t __attempt = - ((static_cast(__op(static_cast<_Type>(__old >> __offset))) << __offset) & __valueMask) - | (__old & __windowMask); - - if (__cuda_atomic_compare_exchange( - __aligned, __old, __old, __attempt, _Order{}, __cuda_atomic_operand_b32{}, _Sco{})) - { - // CAS was successful - return static_cast<_Type>(__old >> __offset); - } + __cuda_atomic_fetch_and(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); } -} +}; -template class _Op> -struct __cuda_atomic_op_bind +struct __cuda_atomic_fetch_add_op { - _Type __val; - - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __old) const + template + _CCCL_HOST_DEVICE_API void + operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const { - return _Op<_Type>{}(__val, __old); + __cuda_atomic_fetch_add(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); } }; -template -struct __cuda_atomic_op_store +struct __cuda_atomic_fetch_sub_op { - // Just return first value - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __val, _Type) const + template + _CCCL_HOST_DEVICE_API void + operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const { - return __val; + __cuda_atomic_fetch_sub(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); } }; -template -struct __cuda_atomic_op_fetch_min +struct __cuda_atomic_fetch_or_op { - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __op, _Type __old) const + template + _CCCL_HOST_DEVICE_API void + operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const { - return __op < __old ? __op : __old; + __cuda_atomic_fetch_or(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); } }; -template -struct __cuda_atomic_op_fetch_max +struct __cuda_atomic_fetch_xor_op { - [[nodiscard]] _CCCL_DEVICE_API _Type operator()(_Type __op, _Type __old) const + template + _CCCL_HOST_DEVICE_API void + operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const { - return __old < __op ? __op : __old; + __cuda_atomic_fetch_xor(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); } }; -// Optimized fetch_update CAS loop with op determined after first load reducing waste. -template = 0> -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_update(_Type* __ptr, const _Fn& __op, _Order, _Operand, _Sco) +struct __cuda_atomic_fetch_min_op { - _Type __expected = 0; - NV_IF_TARGET( - NV_PROVIDES_SM_70, - (__cuda_atomic_load( - __ptr, - __expected, - __cuda_atomic_order_relaxed{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});), - (__cuda_atomic_load( - __ptr, - __expected, - __cuda_atomic_order_volatile{}, - __cuda_atomic_operand_b32{}, - _Sco{}, - __cuda_atomic_mmio_disable{});)) - - _Type __desired = __op(__expected); - while (!__cuda_atomic_compare_exchange(__ptr, __expected, __expected, __desired, _Order{}, _Operand{}, _Sco{})) + template + _CCCL_HOST_DEVICE_API void + operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const { - __desired = __op(__expected); + __cuda_atomic_fetch_min(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); } - return __expected; -} +}; -template = 0> -_CCCL_DEVICE static void __cuda_atomic_fetch_add(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) +struct __cuda_atomic_fetch_max_op { - __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::plus>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} + template + _CCCL_HOST_DEVICE_API void + operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const + { + __cuda_atomic_fetch_max(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); + } +}; -template = 0> -_CCCL_DEVICE static void __cuda_atomic_fetch_and(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) +template = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_and( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) { __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::bit_and>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); + __backend, __ptr, __cuda_atomic_op_bind<_Type, ::cuda::std::bit_and>{__op}, __order, _Operand{}, __scope); } -template = 0> -_CCCL_DEVICE static void __cuda_atomic_fetch_xor(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) +template = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_or( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) { __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::bit_xor>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); + __backend, __ptr, __cuda_atomic_op_bind<_Type, ::cuda::std::bit_or>{__op}, __order, _Operand{}, __scope); } -template = 0> -_CCCL_DEVICE static void __cuda_atomic_fetch_or(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) +template = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_xor( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) { __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::bit_or>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); + __backend, __ptr, __cuda_atomic_op_bind<_Type, ::cuda::std::bit_xor>{__op}, __order, _Operand{}, __scope); } -template = 0> -_CCCL_DEVICE static void __cuda_atomic_fetch_min(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) +template = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_min( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) { __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_min>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); + __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_min>{__op}, __order, _Operand{}, __scope); } -template = 0> -_CCCL_DEVICE static void __cuda_atomic_fetch_max(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) +template = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_max( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) { __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_max>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); + __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_max>{__op}, __order, _Operand{}, __scope); } -template = 0> -_CCCL_DEVICE static void __cuda_atomic_exchange(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco) +template = false> +_CCCL_HOST_DEVICE static void __cuda_atomic_exchange( + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) { __dst = __cuda_atomic_fetch_update( - __ptr, - __cuda_atomic_op_bind<_Type, ::cuda::std::__cuda_atomic_op_store>{__op}, - _Order{}, - __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>{}, - _Sco{}); -} - -template -[[nodiscard]] _CCCL_DEVICE static _Tp __cuda_atomic_fetch_update(_Tp* __ptr, const _Fn& __op, int __memorder, _Sco) -{ - _Tp __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, _Sco{}); - _Tp __desired = __op(__expected); - while (!__cuda_atomic_compare_exchange_dispatch(__ptr, &__expected, __desired, true, __memorder, __memorder, _Sco{})) - { - __desired = __op(__expected); - } - return __expected; -} -template -[[nodiscard]] _CCCL_DEVICE static _Tp -__cuda_atomic_fetch_update(_Tp volatile* __ptr, const _Fn& __op, int __memorder, _Sco) -{ - _Tp __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, _Sco{}); - _Tp __desired = __op(__expected); - while (!__cuda_atomic_compare_exchange_dispatch(__ptr, &__expected, __desired, true, __memorder, __memorder, _Sco{})) - { - __desired = __op(__expected); - } - return __expected; -} - -template -[[nodiscard]] _CCCL_DEVICE static _Tp __atomic_load_n_cuda(const _Tp* __ptr, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_load_cuda(__ptr, __ret, __memorder, _Sco{}); - return __ret; -} -template -[[nodiscard]] _CCCL_DEVICE static _Tp __atomic_load_n_cuda(const _Tp volatile* __ptr, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_load_cuda(__ptr, __ret, __memorder, _Sco{}); - return __ret; -} - -template -_CCCL_DEVICE_API void __atomic_store_n_cuda(_Tp* __ptr, _Tp __val, int __memorder, _Sco) -{ - __atomic_store_cuda(__ptr, __val, __memorder, _Sco{}); -} -template -_CCCL_DEVICE_API void __atomic_store_n_cuda(_Tp volatile* __ptr, _Tp __val, int __memorder, _Sco) -{ - __atomic_store_cuda(__ptr, __val, __memorder, _Sco{}); -} - -template -[[nodiscard]] _CCCL_DEVICE static _Tp __atomic_exchange_n_cuda(_Tp* __ptr, _Tp __val, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_exchange_cuda(__ptr, __ret, __val, __memorder, _Sco{}); - return __ret; -} -template -[[nodiscard]] _CCCL_DEVICE static _Tp __atomic_exchange_n_cuda(_Tp volatile* __ptr, _Tp __val, int __memorder, _Sco) -{ - _Tp __ret; - __atomic_exchange_cuda(__ptr, __ret, __val, __memorder, _Sco{}); - return __ret; -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp __cuda_atomic_fetch_add_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::plus>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp -__cuda_atomic_fetch_add_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::plus>{__val}, __memorder, _Sco{}); + __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_store>{__op}, __order, _Operand{}, __scope); } - -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp __cuda_atomic_fetch_and_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_and>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp -__cuda_atomic_fetch_and_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_and>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp __cuda_atomic_fetch_or_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_or>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp -__cuda_atomic_fetch_or_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_or>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp __cuda_atomic_fetch_xor_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_xor>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp -__cuda_atomic_fetch_xor_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_xor>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp __cuda_atomic_fetch_min_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_min>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp -__cuda_atomic_fetch_min_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_min>{__val}, __memorder, _Sco{}); -} - -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp __cuda_atomic_fetch_max_dispatch(_Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_max>{__val}, __memorder, _Sco{}); -} -template = 0> -[[nodiscard]] _CCCL_DEVICE static _Tp -__cuda_atomic_fetch_max_dispatch(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco) -{ - return __cuda_atomic_fetch_update( - __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_max>{__val}, __memorder, _Sco{}); -} - -_CCCL_DEVICE static inline void __cuda_atomic_signal_fence(int) -{ - asm volatile("" ::: "memory"); -} - -#endif // _CCCL_CUDA_COMPILATION() - _CCCL_END_NAMESPACE_CUDA_STD #include diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h new file mode 100644 index 000000000000..0172430e4cb0 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h @@ -0,0 +1,325 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_RMW_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_RMW_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +template +using __cuda_atomic_enable_generic_rmw = enable_if_t<(_Operand::__size <= _Backend::__widest_cas), bool>; + +template +struct __cuda_atomic_rmw_type; + +template <> +struct __cuda_atomic_rmw_type<8> +{ + using type = uint8_t; +}; + +template <> +struct __cuda_atomic_rmw_type<16> +{ + using type = uint16_t; +}; + +template <> +struct __cuda_atomic_rmw_type<32> +{ + using type = uint32_t; +}; + +template <> +struct __cuda_atomic_rmw_type<64> +{ + using type = uint64_t; +}; + +template <> +struct __cuda_atomic_rmw_type<128> +{ + using type = __cuda_atomic_longlong2; +}; + +template +struct __cuda_atomic_rmw_window +{ + using __logical_type = typename __cuda_atomic_rmw_type::type; + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _RmwType __replace(_RmwType __old, _Type __op, uint8_t __offset) + { + constexpr auto __sizemask = (_RmwType{1} << (sizeof(_Type) * 8)) - 1; + const auto __value_mask = __sizemask << __offset; + const auto __op_bits = static_cast<_RmwType>(::cuda::std::bit_cast<__logical_type>(__op)); + return (__old & ~__value_mask) | ((__op_bits << __offset) & __value_mask); + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _Type __extract(_RmwType __old, uint8_t __offset) + { + constexpr auto __sizemask = (_RmwType{1} << (sizeof(_Type) * 8)) - 1; + const auto __old_bits = static_cast<__logical_type>((__old >> __offset) & __sizemask); + return ::cuda::std::bit_cast<_Type>(__old_bits); + } +}; + +template +struct __cuda_atomic_rmw_window<_Type, _RmwType, true> +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API static _RmwType __replace(_RmwType, _Type __op, uint8_t) + { + return ::cuda::std::bit_cast<_RmwType>(__op); + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _Type __extract(_RmwType __old, uint8_t) + { + return ::cuda::std::bit_cast<_Type>(__old); + } +}; + +template +struct __cuda_atomic_rmw_window<_Type, __cuda_atomic_longlong2, false> +{ + static_assert(sizeof(_Type) == sizeof(uint64_t), "only 64-bit atomics can be widened to 128 bits"); + using __logical_type = typename __cuda_atomic_rmw_type::type; + + [[nodiscard]] _CCCL_HOST_DEVICE_API static __cuda_atomic_longlong2 + __replace(__cuda_atomic_longlong2 __old, _Type __op, uint8_t __offset) + { + (__offset == 0 ? __old.__x : __old.__y) = ::cuda::std::bit_cast<__logical_type>(__op); + return __old; + } + + [[nodiscard]] _CCCL_HOST_DEVICE_API static _Type __extract(__cuda_atomic_longlong2 __old, uint8_t __offset) + { + return ::cuda::std::bit_cast<_Type>(static_cast<__logical_type>(__offset == 0 ? __old.__x : __old.__y)); + } +}; + +template +struct __cuda_atomic_rmw_update +{ + _Type __value; + bool __apply; +}; + +template +struct __cuda_atomic_rmw_result +{ + _Type __old; + bool __applied; +}; + +template +_CCCL_HOST_DEVICE static __cuda_atomic_rmw_result<_Type> __cuda_atomic_rmw( + _Backend __backend, + _Type* __ptr, + const _Fn& __op, + _Order __order, + _InitialOrder __initial_order, + _Operand, + _RmwOperand, + _Sco __scope) +{ + static_assert(_Operand::__op == __cuda_atomic_operand::_b, "generic RMW requires a bitwise operand"); + static_assert(_RmwOperand::__op == __cuda_atomic_operand::_b, "generic RMW requires a bitwise CAS operand"); + static_assert(_Operand::__size <= _RmwOperand::__size, "generic RMW cannot use a narrower CAS operand"); + + using __rmw_type = typename __cuda_atomic_rmw_type<_RmwOperand::__size>::type; + using __window = __cuda_atomic_rmw_window<_Type, __rmw_type>; + + __rmw_type* __aligned; + uint8_t __offset; + if constexpr (sizeof(_Type) == sizeof(__rmw_type)) + { + __aligned = reinterpret_cast<__rmw_type*>(__ptr); + __offset = 0; + } + else + { + constexpr uintptr_t __alignmask = sizeof(__rmw_type) - 1; + __aligned = reinterpret_cast<__rmw_type*>(reinterpret_cast(__ptr) & ~__alignmask); // NOLINT + __offset = static_cast((reinterpret_cast(__ptr) & __alignmask) * 8); + } + + __rmw_type __old; + __cuda_atomic_load(__backend, __aligned, __old, __initial_order, _RmwOperand{}, __scope, __cuda_atomic_mmio_disable{}); + + while (true) + { + const _Type __logical_old = __window::__extract(__old, __offset); + const __cuda_atomic_rmw_update<_Type> __update = __op(__logical_old); + if (!__update.__apply) + { + return {__logical_old, false}; + } + + const __rmw_type __attempt = __window::__replace(__old, __update.__value, __offset); + if (__cuda_atomic_compare_exchange( + __backend, __aligned, __old, __old, __attempt, true, __order, _RmwOperand{}, __scope)) + { + return {__logical_old, true}; + } + } +} + +template +struct __cuda_atomic_rmw_op +{ + _Fn __op; + + [[nodiscard]] _CCCL_HOST_DEVICE_API __cuda_atomic_rmw_update<_Type> operator()(_Type __old) const + { + return {__op(__old), true}; + } +}; + +template +struct __cuda_atomic_compare_exchange_op +{ + _Type __cmp; + _Type __op; + + [[nodiscard]] _CCCL_HOST_DEVICE_API __cuda_atomic_rmw_update<_Type> operator()(_Type __old) const + { + return {__op, __old == __cmp}; + } +}; + +template class _Op> +struct __cuda_atomic_op_bind +{ + _Type __val; + + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __old) const + { + return _Op<_Type>{}(__val, __old); + } +}; + +template +struct __cuda_atomic_op_store +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __val, _Type) const + { + return __val; + } +}; + +template +struct __cuda_atomic_op_fetch_add +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + if constexpr (is_integral_v<_Type>) + { + using __unsigned_type = make_unsigned_t<_Type>; + const auto __op_bits = ::cuda::std::bit_cast<__unsigned_type>(__op); + const auto __old_bits = ::cuda::std::bit_cast<__unsigned_type>(__old); + return ::cuda::std::bit_cast<_Type>(static_cast<__unsigned_type>(__old_bits + __op_bits)); + } + else + { + return __old + __op; + } + } +}; + +template +struct __cuda_atomic_op_fetch_sub +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + if constexpr (is_integral_v<_Type>) + { + using __unsigned_type = make_unsigned_t<_Type>; + const auto __op_bits = ::cuda::std::bit_cast<__unsigned_type>(__op); + const auto __old_bits = ::cuda::std::bit_cast<__unsigned_type>(__old); + return ::cuda::std::bit_cast<_Type>(static_cast<__unsigned_type>(__old_bits - __op_bits)); + } + else + { + return __old - __op; + } + } +}; + +template +struct __cuda_atomic_op_fetch_min +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + return __op < __old ? __op : __old; + } +}; + +template +struct __cuda_atomic_op_fetch_max +{ + [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const + { + return __old < __op ? __op : __old; + } +}; + +template +_CCCL_HOST_DEVICE_API _Type +__cuda_atomic_fetch_update(_Backend __backend, _Type* __ptr, const _Fn& __op, _Order __order, _Operand, _Sco __scope) +{ + static_assert(sizeof(_Type) * 8 == _Operand::__size, "generic RMW requires matching type and operand sizes"); + constexpr size_t __rmw_size = + _Operand::__size < _Backend::__smallest_cas ? _Backend::__smallest_cas : _Operand::__size; + static_assert(__rmw_size <= _Backend::__widest_cas, "generic RMW requires a supported CAS width"); + + using __bitwise_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>; + using __rmw_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, __rmw_size>; + return __cuda_atomic_rmw( + __backend, + __ptr, + __cuda_atomic_rmw_op<_Type, _Fn>{__op}, + __order, + __cuda_atomic_initial_load_order<_Order>(), + __bitwise_operand{}, + __rmw_operand{}, + __scope) + .__old; +} + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_GENERIC_RMW_H From 3c1bedf837063b37c11aaffce564c66413abbc89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Wed, 19 Aug 2026 11:11:17 -0700 Subject: [PATCH 03/24] Unify atomic dispatch across backends. --- libcudacxx/include/cuda/__atomic/atomic.h | 9 +- .../cuda/std/__atomic/functions/dispatch.h | 107 ------- .../cuda/std/__atomic/functions/host.h | 273 +++++++++--------- .../std/__atomic/functions/host_backend.h | 64 ++++ libcudacxx/include/cuda/std/__atomic/scopes.h | 12 +- .../include/cuda/std/__atomic/types/base.h | 121 ++------ 6 files changed, 239 insertions(+), 347 deletions(-) create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/host_backend.h diff --git a/libcudacxx/include/cuda/__atomic/atomic.h b/libcudacxx/include/cuda/__atomic/atomic.h index 388a85329cb6..9188122ea2e1 100644 --- a/libcudacxx/include/cuda/__atomic/atomic.h +++ b/libcudacxx/include/cuda/__atomic/atomic.h @@ -144,13 +144,16 @@ atomic_thread_fence(memory_order __m, [[maybe_unused]] thread_scope _Scope = thr NV_IS_DEVICE, (switch (_Scope) { case thread_scope::thread_scope_system: - ::cuda::std::__cuda_atomic_thread_fence((int) __m, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_thread_fence( + ::cuda::std::__cuda_atomic_ptx_backend{}, __m, __thread_scope_system_tag{}); break; case thread_scope::thread_scope_device: - ::cuda::std::__cuda_atomic_thread_fence((int) __m, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_thread_fence( + ::cuda::std::__cuda_atomic_ptx_backend{}, __m, __thread_scope_device_tag{}); break; case thread_scope::thread_scope_block: - ::cuda::std::__cuda_atomic_thread_fence((int) __m, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_thread_fence( + ::cuda::std::__cuda_atomic_ptx_backend{}, __m, __thread_scope_block_tag{}); break; // Atomics scoped to themselves do not require fencing case thread_scope::thread_scope_thread: diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index 089f0d99d02a..3d2211d148ca 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -331,117 +331,10 @@ template -_CCCL_DEVICE_API void __atomic_thread_fence_cuda(int __order, _Sco __scope) -{ - __cuda_atomic_thread_fence(__cuda_atomic_ptx_backend{}, static_cast(__order), __scope); -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Type __atomic_load_n_cuda(const _Type* __ptr, int __order, _Sco __scope) -{ - return __cuda_atomic_load_dispatch(__cuda_atomic_ptx_backend{}, __ptr, static_cast(__order), __scope); -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Type __atomic_load_n_cuda(const volatile _Type* __ptr, int __order, _Sco __scope) -{ - return __cuda_atomic_load_dispatch(__cuda_atomic_ptx_backend{}, __ptr, static_cast(__order), __scope); -} - -template -_CCCL_DEVICE_API void __atomic_store_n_cuda(_Type* __ptr, _Type __val, int __order, _Sco __scope) -{ - __cuda_atomic_store_dispatch(__cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); -} - -template -_CCCL_DEVICE_API void __atomic_store_n_cuda(volatile _Type* __ptr, _Type __val, int __order, _Sco __scope) -{ - __cuda_atomic_store_dispatch(__cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Type __atomic_exchange_n_cuda(_Type* __ptr, _Type __val, int __order, _Sco __scope) -{ - return __cuda_atomic_exchange_dispatch( - __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); -} - -template -[[nodiscard]] _CCCL_DEVICE_API _Type -__atomic_exchange_n_cuda(volatile _Type* __ptr, _Type __val, int __order, _Sco __scope) -{ - return __cuda_atomic_exchange_dispatch( - __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); -} - -template -_CCCL_DEVICE_API bool __atomic_compare_exchange_cuda( - _Type* __ptr, _Type* __expected, _Type __desired, bool __weak, int __success, int __failure, _Sco __scope) -{ - return __cuda_atomic_compare_exchange_dispatch( - __cuda_atomic_ptx_backend{}, - __ptr, - __expected, - __desired, - __weak, - static_cast(__success), - static_cast(__failure), - __scope); -} - -template -_CCCL_DEVICE_API bool __atomic_compare_exchange_cuda( - volatile _Type* __ptr, _Type* __expected, _Type __desired, bool __weak, int __success, int __failure, _Sco __scope) -{ - return __cuda_atomic_compare_exchange_dispatch( - __cuda_atomic_ptx_backend{}, - __ptr, - __expected, - __desired, - __weak, - static_cast(__success), - static_cast(__failure), - __scope); -} - -# define _CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(_Op) \ - template \ - [[nodiscard]] _CCCL_DEVICE_API _Type __atomic_fetch_##_Op##_cuda( \ - _Type* __ptr, _Up __val, int __order, _Sco __scope) \ - { \ - return __cuda_atomic_fetch_##_Op##_dispatch( \ - __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); \ - } \ - \ - template \ - [[nodiscard]] _CCCL_DEVICE_API _Type __atomic_fetch_##_Op##_cuda( \ - volatile _Type* __ptr, _Up __val, int __order, _Sco __scope) \ - { \ - return __cuda_atomic_fetch_##_Op##_dispatch( \ - __cuda_atomic_ptx_backend{}, __ptr, __val, static_cast(__order), __scope); \ - } - -_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(add) -_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(sub) -_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(and) -_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(or) -_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(xor) -_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(min) -_CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER(max) - -# undef _CCCL_CUDA_ATOMIC_LEGACY_FETCH_WRAPPER - _CCCL_DEVICE static inline void __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend, memory_order) { asm volatile("" ::: "memory"); } - -_CCCL_DEVICE static inline void __atomic_signal_fence_cuda(int __order) -{ - __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend{}, static_cast(__order)); -} #endif // _CCCL_CUDA_COMPILATION() _CCCL_END_NAMESPACE_CUDA_STD diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host.h b/libcudacxx/include/cuda/std/__atomic/functions/host.h index 0a20a333ff81..8a929dd9f0be 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/host.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/host.h @@ -21,12 +21,14 @@ # pragma system_header #endif // no system header +#include #include +#include #include #include #include #include -#include +#include #include @@ -69,166 +71,153 @@ __atomic_alignment_wrapper<_Tp>* __atomic_force_align_host(_Tp* __a) } // Guard ifdef for lock free query in case it is assigned elsewhere (MSVC/CUDA) -inline void __atomic_thread_fence_host(memory_order __order) +inline void __cuda_atomic_thread_fence(__cuda_atomic_host_backend, memory_order __order, __thread_scope_tag) { __atomic_thread_fence(__atomic_order_to_int(__order)); } -inline void __atomic_signal_fence_host(memory_order __order) +inline void __cuda_atomic_signal_fence(__cuda_atomic_host_backend, memory_order __order) { __atomic_signal_fence(__atomic_order_to_int(__order)); } -template -inline void __atomic_store_host(_Tp* __a, _Up __val, memory_order __order) +[[nodiscard]] _CCCL_HOST_API constexpr memory_order __cuda_atomic_failure_order(memory_order __order) { - _LIBCUDACXX_INT128_WARN(_Tp) - __atomic_store(&__atomic_force_align_host(__a)->__atom, &__val, __atomic_order_to_int(__order)); + return __order == memory_order_release + ? memory_order_relaxed + : (__order == memory_order_acq_rel ? memory_order_acquire : __order); } -template -inline auto __atomic_load_host(_Tp* __a, memory_order __order) -> remove_cv_t<_Tp> +template +_CCCL_HOST_API void __cuda_atomic_load( + __cuda_atomic_host_backend, + const _Type* __ptr, + _Type& __dst, + memory_order __order, + _Operand, + __thread_scope_tag, + _Mmio) { - _LIBCUDACXX_INT128_WARN(_Tp) - remove_cv_t<_Tp> __ret; - __atomic_load(&__atomic_force_align_host(__a)->__atom, &__ret, __atomic_order_to_int(__order)); - return __ret; + _LIBCUDACXX_INT128_WARN(_Type) + __atomic_load(&__atomic_force_align_host(__ptr)->__atom, &__dst, __atomic_order_to_int(__order)); } -template -inline auto __atomic_exchange_host(_Tp* __a, _Up __val, memory_order __order) -> remove_cv_t<_Tp> +template +_CCCL_HOST_API void __cuda_atomic_store( + __cuda_atomic_host_backend, _Type* __ptr, _Type& __val, memory_order __order, _Operand, __thread_scope_tag, _Mmio) { - _LIBCUDACXX_INT128_WARN(_Tp) - remove_cv_t<_Tp> __ret; - __atomic_exchange(&__atomic_force_align_host(__a)->__atom, &__val, &__ret, __atomic_order_to_int(__order)); - return __ret; -} - -template -inline bool __atomic_compare_exchange_strong_host( - _Tp* __a, _Up* __expected, _Up __desired, memory_order __success, memory_order __failure) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_compare_exchange( - &__atomic_force_align_host(__a)->__atom, - // This is only alignment wrapped in order to prevent GCC-6 from triggering unused warning - &__atomic_force_align_host(__expected)->__atom, - &__desired, - false, - __atomic_order_to_int(__success), - __atomic_failure_order_to_int(__failure)); + _LIBCUDACXX_INT128_WARN(_Type) + __atomic_store(&__atomic_force_align_host(__ptr)->__atom, &__val, __atomic_order_to_int(__order)); } -template -inline bool __atomic_compare_exchange_weak_host( - _Tp* __a, _Up* __expected, _Up __desired, memory_order __success, memory_order __failure) +template +_CCCL_HOST_API bool __cuda_atomic_compare_exchange( + __cuda_atomic_host_backend, + _Type* __ptr, + _Type& __dst, + _Type __cmp, + _Type __op, + bool __weak, + __cuda_atomic_runtime_cas_order __order, + _Operand, + __thread_scope_tag) { - _LIBCUDACXX_INT128_WARN(_Tp) + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __cmp; return __atomic_compare_exchange( - &__atomic_force_align_host(__a)->__atom, - // This is only alignment wrapped in order to prevent GCC-6 from triggering unused warning - &__atomic_force_align_host(__expected)->__atom, - &__desired, - true, - __atomic_order_to_int(__success), - __atomic_failure_order_to_int(__failure)); -} - -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_add_host(_Tp* __a, _Td __delta, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - constexpr auto __skip_v = __atomic_ptr_skip_t<_Tp>::__skip; - return __atomic_fetch_add(__a, __delta * __skip_v, __atomic_order_to_int(__order)); -} - -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_add_host(_Tp* __a, _Td __delta, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected + __delta; - - while (!__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected + __delta; - } - - return __expected; -} - -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_sub_host(_Tp* __a, _Td __delta, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - constexpr auto __skip_v = __atomic_ptr_skip_t<_Tp>::__skip; - return __atomic_fetch_sub(__a, __delta * __skip_v, __atomic_order_to_int(__order)); -} - -template , int> = 0> -inline remove_cv_t<_Tp> __atomic_fetch_sub_host(_Tp* __a, _Td __delta, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected - __delta; - - while (!__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected - __delta; - } - - return __expected; -} - -template -inline remove_cv_t<_Tp> __atomic_fetch_and_host(_Tp* __a, _Td __pattern, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_fetch_and(__a, __pattern, __atomic_order_to_int(__order)); -} - -template -inline remove_cv_t<_Tp> __atomic_fetch_or_host(_Tp* __a, _Td __pattern, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_fetch_or(__a, __pattern, __atomic_order_to_int(__order)); -} - -template -inline remove_cv_t<_Tp> __atomic_fetch_xor_host(_Tp* __a, _Td __pattern, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - return __atomic_fetch_xor(__a, __pattern, __atomic_order_to_int(__order)); -} - -template -inline remove_cv_t<_Tp> __atomic_fetch_max_host(_Tp* __a, _Td __val, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected > __val ? __expected : __val; - - while (__desired == __val && !__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected > __val ? __expected : __val; - } - - return __expected; -} - -template -inline remove_cv_t<_Tp> __atomic_fetch_min_host(_Tp* __a, _Td __val, memory_order __order) -{ - _LIBCUDACXX_INT128_WARN(_Tp) - auto __expected = __atomic_load_host(__a, memory_order_relaxed); - auto __desired = __expected < __val ? __expected : __val; - - while (__desired == __val && !__atomic_compare_exchange_strong_host(__a, &__expected, __desired, __order, __order)) - { - __desired = __expected < __val ? __expected : __val; - } - - return __expected; + &__atomic_force_align_host(__ptr)->__atom, + // This is only alignment wrapped in order to prevent GCC-6 from triggering an unused warning. + &__atomic_force_align_host(&__dst)->__atom, + &__op, + __weak, + __atomic_order_to_int(__order.__success), + __atomic_failure_order_to_int(__order.__failure)); +} + +template +_CCCL_HOST_API bool __cuda_atomic_compare_exchange( + __cuda_atomic_host_backend __backend, + _Type* __ptr, + _Type& __dst, + _Type __cmp, + _Type __op, + bool __weak, + memory_order __order, + _Operand __operand, + __thread_scope_tag __scope) +{ + return __cuda_atomic_compare_exchange( + __backend, + __ptr, + __dst, + __cmp, + __op, + __weak, + __cuda_atomic_runtime_cas_order{__order, __cuda_atomic_failure_order(__order)}, + __operand, + __scope); +} + +template +_CCCL_HOST_API void __cuda_atomic_exchange( + __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) +{ + _LIBCUDACXX_INT128_WARN(_Type) + __atomic_exchange(&__atomic_force_align_host(__ptr)->__atom, &__op, &__dst, __atomic_order_to_int(__order)); +} + +template < + class _Type, + class _Operand, + enable_if_t && (_Operand::__op != __cuda_atomic_operand::_f) && (_Operand::__size <= 128), + bool> = false> +_CCCL_HOST_API void __cuda_atomic_fetch_add( + __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) +{ + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_add(__ptr, __op, __atomic_order_to_int(__order)); +} + +template < + class _Type, + class _Operand, + enable_if_t && (_Operand::__op != __cuda_atomic_operand::_f) && (_Operand::__size <= 128), + bool> = false> +_CCCL_HOST_API void __cuda_atomic_fetch_sub( + __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) +{ + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_sub(__ptr, __op, __atomic_order_to_int(__order)); +} + +template = false> +_CCCL_HOST_API void __cuda_atomic_fetch_and( + __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) +{ + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_and(__ptr, __op, __atomic_order_to_int(__order)); +} + +template = false> +_CCCL_HOST_API void __cuda_atomic_fetch_or( + __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) +{ + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_or(__ptr, __op, __atomic_order_to_int(__order)); +} + +template = false> +_CCCL_HOST_API void __cuda_atomic_fetch_xor( + __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) +{ + _LIBCUDACXX_INT128_WARN(_Type) + __dst = __atomic_fetch_xor(__ptr, __op, __atomic_order_to_int(__order)); } #endif // !_CCCL_COMPILER(NVRTC) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h new file mode 100644 index 000000000000..aab661378b2b --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_HOST_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_HOST_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __cuda_atomic_host_backend +{ + template + using __enable_if_direct_bitwise = enable_if_t<(sizeof(_Type) < 16), bool>; + + template + using __enable_if_direct_arithmetic = enable_if_t && (sizeof(_Type) < 16), bool>; + + template + using __enable_if_direct_minmax = enable_if_t && (sizeof(_Type) < 16), bool>; + + template + using __enable_if_fallback_bitwise = enable_if_t<(sizeof(_Type) == 16), bool>; + + template + using __enable_if_fallback_arithmetic = enable_if_t && (sizeof(_Type) == 16), bool>; + + template + using __enable_if_fallback_minmax = + enable_if_t || (is_scalar_v<_Type> && sizeof(_Type) == 16), bool>; + + static constexpr bool __needs_constant_order = false; + static constexpr bool __requires_local_memory_workaround = false; + static constexpr size_t __smallest_cas = 8; + static constexpr size_t __widest_cas = 128; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_HOST_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/scopes.h b/libcudacxx/include/cuda/std/__atomic/scopes.h index cd65782b6c00..16fe52679af1 100644 --- a/libcudacxx/include/cuda/std/__atomic/scopes.h +++ b/libcudacxx/include/cuda/std/__atomic/scopes.h @@ -42,15 +42,17 @@ enum thread_scope thread_scope_thread = __ATOMIC_THREAD }; -struct __thread_scope_thread_tag +struct __thread_scope_tag {}; -struct __thread_scope_block_tag +struct __thread_scope_thread_tag : __thread_scope_tag {}; -struct __thread_scope_cluster_tag +struct __thread_scope_block_tag : __thread_scope_tag {}; -struct __thread_scope_device_tag +struct __thread_scope_cluster_tag : __thread_scope_tag {}; -struct __thread_scope_system_tag +struct __thread_scope_device_tag : __thread_scope_tag +{}; +struct __thread_scope_system_tag : __thread_scope_tag {}; template diff --git a/libcudacxx/include/cuda/std/__atomic/types/base.h b/libcudacxx/include/cuda/std/__atomic/types/base.h index 538bacb5abc9..2d200ca77447 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/base.h +++ b/libcudacxx/include/cuda/std/__atomic/types/base.h @@ -69,21 +69,26 @@ struct __atomic_storage } }; +#define _CCCL_DISPATCH_ATOMIC_BACKEND(_Fn, ...) \ + NV_DISPATCH_TARGET(NV_IS_DEVICE, \ + (return _Fn(__cuda_atomic_ptx_backend{}, __VA_ARGS__);), \ + NV_IS_HOST, \ + (return _Fn(__cuda_atomic_host_backend{}, __VA_ARGS__);)) + +#define _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(_Fn, _Scope, ...) \ + NV_DISPATCH_TARGET(NV_IS_DEVICE, \ + (return _Fn(__cuda_atomic_ptx_backend{}, __VA_ARGS__, _Scope);), \ + NV_IS_HOST, \ + (return _Fn(__cuda_atomic_host_backend{}, __VA_ARGS__, __thread_scope_tag{});)) + _CCCL_HOST_DEVICE_API inline void __atomic_thread_fence_dispatch(memory_order __order) { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__cuda_atomic_thread_fence(static_cast<__memory_order_underlying_t>(__order), __thread_scope_system_tag());), - NV_IS_HOST, - (__atomic_thread_fence_host(__order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_thread_fence, __thread_scope_system_tag{}, __order); } _CCCL_HOST_DEVICE_API inline void __atomic_signal_fence_dispatch(memory_order __order) { - NV_DISPATCH_TARGET(NV_IS_DEVICE, - (__cuda_atomic_signal_fence(static_cast<__memory_order_underlying_t>(__order));), - NV_IS_HOST, - (__atomic_signal_fence_host(__order);)) + _CCCL_DISPATCH_ATOMIC_BACKEND(__cuda_atomic_signal_fence, __order); } template = 0> @@ -95,155 +100,91 @@ _CCCL_HOST_DEVICE_API void __atomic_init_dispatch(_Sto* __a, _Up __val) template = 0> _CCCL_HOST_DEVICE_API void __atomic_store_dispatch(_Sto* __a, _Up __val, memory_order __order, _Sco = {}) { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__atomic_store_n_cuda(__a->get(), __val, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (__atomic_store_host(__a->get(), __val, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_store_dispatch, _Sco{}, __a->get(), __val, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_load_dispatch(const _Sto* __a, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __atomic_load_n_cuda(__a->get(), static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_load_host(__a->get(), __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_load_dispatch, _Sco{}, __a->get(), __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_exchange_dispatch(_Sto* __a, _Up __value, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __atomic_exchange_n_cuda(__a->get(), __value, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_exchange_host(__a->get(), __value, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_exchange_dispatch, _Sco{}, __a->get(), __value, __order); } template = 0> _CCCL_HOST_DEVICE_API bool __atomic_compare_exchange_strong_dispatch( _Sto* __a, _Up* __expected, _Up __val, memory_order __success, memory_order __failure, _Sco = {}) { - bool __result = false; - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__result = __cuda_atomic_compare_exchange_dispatch( - __a->get(), - __expected, - __val, - false, - static_cast<__memory_order_underlying_t>(__success), - static_cast<__memory_order_underlying_t>(__failure), - _Sco{});), - NV_IS_HOST, - (__result = __atomic_compare_exchange_strong_host(__a->get(), __expected, __val, __success, __failure);)) - return __result; + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND( + __cuda_atomic_compare_exchange_dispatch, _Sco{}, __a->get(), __expected, __val, false, __success, __failure); } template = 0> _CCCL_HOST_DEVICE_API bool __atomic_compare_exchange_weak_dispatch( _Sto* __a, _Up* __expected, _Up __val, memory_order __success, memory_order __failure, _Sco = {}) { - bool __result = false; - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (__result = __cuda_atomic_compare_exchange_dispatch( - __a->get(), - __expected, - __val, - true, - static_cast<__memory_order_underlying_t>(__success), - static_cast<__memory_order_underlying_t>(__failure), - _Sco{});), - NV_IS_HOST, - (__result = __atomic_compare_exchange_weak_host(__a->get(), __expected, __val, __success, __failure);)) - return __result; + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND( + __cuda_atomic_compare_exchange_dispatch, _Sco{}, __a->get(), __expected, __val, true, __success, __failure); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_add_dispatch(_Sto* __a, _Up __delta, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_add_dispatch( - __a->get(), __delta, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_add_host(__a->get(), __delta, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_add_dispatch, _Sco{}, __a->get(), __delta, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_sub_dispatch(_Sto* __a, _Up __delta, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_sub_dispatch( - __a->get(), __delta, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_sub_host(__a->get(), __delta, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_sub_dispatch, _Sco{}, __a->get(), __delta, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_and_dispatch(_Sto* __a, _Up __pattern, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_and_dispatch( - __a->get(), __pattern, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_and_host(__a->get(), __pattern, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_and_dispatch, _Sco{}, __a->get(), __pattern, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_or_dispatch(_Sto* __a, _Up __pattern, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_or_dispatch( - __a->get(), __pattern, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_or_host(__a->get(), __pattern, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_or_dispatch, _Sco{}, __a->get(), __pattern, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_xor_dispatch(_Sto* __a, _Up __pattern, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_DISPATCH_TARGET( - NV_IS_DEVICE, - (return __cuda_atomic_fetch_xor_dispatch( - __a->get(), __pattern, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - NV_IS_HOST, - (return __atomic_fetch_xor_host(__a->get(), __pattern, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_xor_dispatch, _Sco{}, __a->get(), __pattern, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_max_dispatch(_Sto* __a, _Up __val, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_IF_TARGET(NV_IS_DEVICE, - (return __cuda_atomic_fetch_max_dispatch( - __a->get(), __val, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - (return __atomic_fetch_max_host(__a->get(), __val, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_max_dispatch, _Sco{}, __a->get(), __val, __order); } template = 0> _CCCL_HOST_DEVICE_API auto __atomic_fetch_min_dispatch(_Sto* __a, _Up __val, memory_order __order, _Sco = {}) -> __atomic_underlying_remove_cv_t<_Sto> { - NV_IF_TARGET(NV_IS_DEVICE, - (return __cuda_atomic_fetch_min_dispatch( - __a->get(), __val, static_cast<__memory_order_underlying_t>(__order), _Sco{});), - (return __atomic_fetch_min_host(__a->get(), __val, __order);)) + _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(__cuda_atomic_fetch_min_dispatch, _Sco{}, __a->get(), __val, __order); } +#undef _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND +#undef _CCCL_DISPATCH_ATOMIC_BACKEND + _CCCL_END_NAMESPACE_CUDA_STD #include From 1800050f06fa4e6ce1d16949c011c9880a836b3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Wed, 19 Aug 2026 15:31:38 -0700 Subject: [PATCH 04/24] Preserve compare-exchange strength through backend dispatch. --- .../codegen/generators/compare_and_swap.h | 25 ++-- .../cuda/std/__atomic/functions/backend.h | 16 ++ .../__atomic/functions/cuda_ptx_generated.h | 141 +++++++++--------- .../cuda/std/__atomic/functions/generic.h | 6 +- .../cuda/std/__atomic/functions/generic_rmw.h | 2 +- .../cuda/std/__atomic/functions/host.h | 12 +- .../include/cuda/std/__atomic/types/base.h | 18 ++- 7 files changed, 125 insertions(+), 95 deletions(-) diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index 67c5a674b99d..656d4922bfd1 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -64,7 +64,7 @@ template constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, {4}, __cuda_atomic_operand_{0}{1}, {6}) {{ static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -85,7 +85,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, {4}, __cuda_atomic_operand_{0}{1}, {6}) {{ asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; constexpr Operand supported_types[] = { @@ -163,27 +163,26 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( << R"XXX( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_compare_exchange { _Backend __backend; _Type* __ptr; _Type* __exp; _Type* __des; - bool __weak; template [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { return __cuda_atomic_compare_exchange( - __backend, __ptr, *__exp, *__exp, *__des, __weak, __order, _Tag{}, _Sco{}); + __backend, __ptr, *__exp, *__exp, *__des, _Cas{}, __order, _Tag{}, _Sco{}); } }; -template +template [[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( _Backend __backend, _Type* __ptr, _Type* __exp, _Type __des, - bool __weak, + _Cas, memory_order __success, memory_order __failure, _Sco __scope) @@ -198,18 +197,18 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( __backend, __bound_compare_swap, __success, __failure, __scope); } -template +template [[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( _Backend __backend, _Type volatile* __ptr, _Type* __exp, _Type __des, - bool __weak, + _Cas, memory_order __success, memory_order __failure, _Sco __scope) @@ -224,8 +223,8 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( __backend, __bound_compare_swap, __success, __failure, __scope); } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/backend.h b/libcudacxx/include/cuda/std/__atomic/functions/backend.h index 1ec37ed4d8d1..c364bb7d61f0 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/backend.h @@ -61,6 +61,22 @@ struct __cuda_atomic_runtime_cas_order memory_order __failure; }; +struct __cuda_atomic_cas_strong +{}; + +struct __cuda_atomic_cas_weak : __cuda_atomic_cas_strong +{}; + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __cuda_atomic_cas_is_weak(__cuda_atomic_cas_strong) +{ + return false; +} + +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr bool __cuda_atomic_cas_is_weak(__cuda_atomic_cas_weak) +{ + return true; +} + template = 0> _CCCL_HOST_DEVICE_API void __cuda_atomic_load_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 0e063698d47f..a381247eebbf 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -1878,167 +1878,167 @@ template template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) { asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) { asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) { asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) { asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) { asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) { asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2057,7 +2057,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2076,7 +2076,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2095,7 +2095,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2114,7 +2114,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2133,7 +2133,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2152,7 +2152,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2171,7 +2171,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2190,7 +2190,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2209,7 +2209,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2228,7 +2228,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2247,7 +2247,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2266,7 +2266,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2285,7 +2285,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2304,7 +2304,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2323,7 +2323,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2342,7 +2342,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2361,7 +2361,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2380,7 +2380,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2399,7 +2399,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) { static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( @@ -2419,27 +2419,26 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_compare_exchange { _Backend __backend; _Type* __ptr; _Type* __exp; _Type* __des; - bool __weak; template [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { return __cuda_atomic_compare_exchange( - __backend, __ptr, *__exp, *__exp, *__des, __weak, __order, _Tag{}, _Sco{}); + __backend, __ptr, *__exp, *__exp, *__des, _Cas{}, __order, _Tag{}, _Sco{}); } }; -template +template [[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( _Backend __backend, _Type* __ptr, _Type* __exp, _Type __des, - bool __weak, + _Cas, memory_order __success, memory_order __failure, _Sco __scope) @@ -2454,18 +2453,18 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( __backend, __bound_compare_swap, __success, __failure, __scope); } -template +template [[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( _Backend __backend, _Type volatile* __ptr, _Type* __exp, _Type __des, - bool __weak, + _Cas, memory_order __success, memory_order __failure, _Sco __scope) @@ -2480,8 +2479,8 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy, __weak}; + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, __des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( __backend, __bound_compare_swap, __success, __failure, __scope); } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic.h b/libcudacxx/include/cuda/std/__atomic/functions/generic.h index 0c3997608d8f..c7dbde3b7926 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic.h @@ -32,13 +32,14 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD template = false> _CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, bool, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, _Cas, _Order __order, _Operand, _Sco __scope) { constexpr size_t __rmw_size = _Backend::__smallest_cas; static_assert(__rmw_size <= _Backend::__widest_cas, "atomic CAS cannot be widened beyond the backend's widest CAS"); @@ -59,13 +60,14 @@ _CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( template _Backend::__widest_cas), bool> = false> _CCCL_HOST_DEVICE static bool -__cuda_atomic_compare_exchange(_Backend, _Type*, _Type&, _Type, _Type, bool, _Order, _Operand, _Sco) +__cuda_atomic_compare_exchange(_Backend, _Type*, _Type&, _Type, _Type, _Cas, _Order, _Operand, _Sco) { static_assert(_Operand::__size < _Backend::__widest_cas, "the backend must provide its widest CAS operation"); return false; diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h index 0172430e4cb0..d3f3cd276180 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h @@ -189,7 +189,7 @@ _CCCL_HOST_DEVICE static __cuda_atomic_rmw_result<_Type> __cuda_atomic_rmw( const __rmw_type __attempt = __window::__replace(__old, __update.__value, __offset); if (__cuda_atomic_compare_exchange( - __backend, __aligned, __old, __old, __attempt, true, __order, _RmwOperand{}, __scope)) + __backend, __aligned, __old, __old, __attempt, __cuda_atomic_cas_weak{}, __order, _RmwOperand{}, __scope)) { return {__logical_old, true}; } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host.h b/libcudacxx/include/cuda/std/__atomic/functions/host.h index 8a929dd9f0be..95b9215d651f 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/host.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/host.h @@ -110,14 +110,14 @@ _CCCL_HOST_API void __cuda_atomic_store( __atomic_store(&__atomic_force_align_host(__ptr)->__atom, &__val, __atomic_order_to_int(__order)); } -template +template _CCCL_HOST_API bool __cuda_atomic_compare_exchange( __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, - bool __weak, + _Cas __cas, __cuda_atomic_runtime_cas_order __order, _Operand, __thread_scope_tag) @@ -129,19 +129,19 @@ _CCCL_HOST_API bool __cuda_atomic_compare_exchange( // This is only alignment wrapped in order to prevent GCC-6 from triggering an unused warning. &__atomic_force_align_host(&__dst)->__atom, &__op, - __weak, + __cuda_atomic_cas_is_weak(__cas), __atomic_order_to_int(__order.__success), __atomic_failure_order_to_int(__order.__failure)); } -template +template _CCCL_HOST_API bool __cuda_atomic_compare_exchange( __cuda_atomic_host_backend __backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, - bool __weak, + _Cas __cas, memory_order __order, _Operand __operand, __thread_scope_tag __scope) @@ -152,7 +152,7 @@ _CCCL_HOST_API bool __cuda_atomic_compare_exchange( __dst, __cmp, __op, - __weak, + __cas, __cuda_atomic_runtime_cas_order{__order, __cuda_atomic_failure_order(__order)}, __operand, __scope); diff --git a/libcudacxx/include/cuda/std/__atomic/types/base.h b/libcudacxx/include/cuda/std/__atomic/types/base.h index 2d200ca77447..17259c9014a6 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/base.h +++ b/libcudacxx/include/cuda/std/__atomic/types/base.h @@ -122,7 +122,14 @@ _CCCL_HOST_DEVICE_API bool __atomic_compare_exchange_strong_dispatch( _Sto* __a, _Up* __expected, _Up __val, memory_order __success, memory_order __failure, _Sco = {}) { _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND( - __cuda_atomic_compare_exchange_dispatch, _Sco{}, __a->get(), __expected, __val, false, __success, __failure); + __cuda_atomic_compare_exchange_dispatch, + _Sco{}, + __a->get(), + __expected, + __val, + __cuda_atomic_cas_strong{}, + __success, + __failure); } template = 0> @@ -130,7 +137,14 @@ _CCCL_HOST_DEVICE_API bool __atomic_compare_exchange_weak_dispatch( _Sto* __a, _Up* __expected, _Up __val, memory_order __success, memory_order __failure, _Sco = {}) { _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND( - __cuda_atomic_compare_exchange_dispatch, _Sco{}, __a->get(), __expected, __val, true, __success, __failure); + __cuda_atomic_compare_exchange_dispatch, + _Sco{}, + __a->get(), + __expected, + __val, + __cuda_atomic_cas_weak{}, + __success, + __failure); } template = 0> From 1dde28ff22320318f4d2697ca09d6878d0774549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Thu, 27 Aug 2026 14:20:53 -0700 Subject: [PATCH 05/24] Complete atomic backend dispatch. --- .../codegen/generators/compare_and_swap.h | 98 +- libcudacxx/codegen/generators/definitions.h | 12 + libcudacxx/codegen/generators/exchange.h | 105 +- libcudacxx/codegen/generators/fence.h | 14 + libcudacxx/codegen/generators/fetch_ops.h | 104 +- libcudacxx/codegen/generators/header.h | 1 + libcudacxx/codegen/generators/ld_st.h | 170 +- .../cuda/std/__atomic/functions/backend.h | 237 +- .../cuda/std/__atomic/functions/common.h | 4 + .../cuda/std/__atomic/functions/cuda_local.h | 106 +- .../cuda/std/__atomic/functions/cuda_ptx.h | 81 +- .../std/__atomic/functions/cuda_ptx_backend.h | 78 +- .../__atomic/functions/cuda_ptx_generated.h | 3662 ++++++++--------- .../cuda/std/__atomic/functions/dispatch.h | 421 +- .../cuda/std/__atomic/functions/generic.h | 148 +- .../cuda/std/__atomic/functions/generic_rmw.h | 34 +- .../cuda/std/__atomic/functions/host.h | 96 +- .../std/__atomic/functions/host_backend.h | 20 - .../include/cuda/std/__atomic/types/common.h | 33 + .../include/cuda/std/__atomic/types/small.h | 29 +- 20 files changed, 2499 insertions(+), 2954 deletions(-) diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index 656d4922bfd1..f9b30d82d1b2 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -18,41 +18,6 @@ inline void FormatCompareAndSwap(std::ostream& out) { - out << R"XXX( -template -[[nodiscard]] static _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_cas, memory_order __success, memory_order __failure, _Sco) { - const int __success_memorder = __atomic_order_to_int(__success); - const int __failure_memorder = __atomic_failure_order_to_int(__failure); - bool __res = false; - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __res = __cuda_cas(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __res = __cuda_cas(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) - return __res; -} -)XXX"; - // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -64,8 +29,9 @@ template constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ + __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -81,12 +47,11 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( mov.b128 {{%0, %1}}, _d; }} )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; }})XXX"; - - constexpr auto asm_intrinsic_format = R"XXX( + constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, {4}, __cuda_atomic_operand_{0}{1}, {6}) -{{ asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) +{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; constexpr Operand supported_types[] = { Operand::Bit, @@ -138,7 +103,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -150,7 +115,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -167,55 +132,32 @@ template * __exp; + __unv<_Type> __cmp; + __unv<_Type> __des; template [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { return __cuda_atomic_compare_exchange( - __backend, __ptr, *__exp, *__exp, *__des, _Cas{}, __order, _Tag{}, _Sco{}); + __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); } }; template [[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( _Backend __backend, _Type* __ptr, - _Type* __exp, - _Type __des, - _Cas, - memory_order __success, - memory_order __failure, - _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); - bool __res = false; - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch( - __backend, __bound_compare_swap, __success, __failure, __scope); -} -template -[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( - _Backend __backend, - _Type volatile* __ptr, - _Type* __exp, - _Type __des, + __unv<_Type>* __exp, + __unv<_Type> __des, _Cas, memory_order __success, memory_order __failure, _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); bool __res = false; @@ -223,8 +165,8 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy}; + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( __backend, __bound_compare_swap, __success, __failure, __scope); } diff --git a/libcudacxx/codegen/generators/definitions.h b/libcudacxx/codegen/generators/definitions.h index fb9c034df046..d1886103e8c6 100644 --- a/libcudacxx/codegen/generators/definitions.h +++ b/libcudacxx/codegen/generators/definitions.h @@ -153,6 +153,18 @@ inline std::string semantic_tag(Semantic sem) return sem_map[sem]; } +inline std::string ptx_semantic_tag(Semantic sem) +{ + static std::map sem_map = { + std::pair{Semantic::Relaxed, "__cuda_atomic_ptx_order_relaxed"}, + std::pair{Semantic::Release, "__cuda_atomic_ptx_order_release"}, + std::pair{Semantic::Acquire, "__cuda_atomic_ptx_order_acquire"}, + std::pair{Semantic::Acq_Rel, "__cuda_atomic_ptx_order_acq_rel"}, + std::pair{Semantic::Volatile, "__cuda_atomic_order_volatile"}, + }; + return sem_map[sem]; +} + enum class Scope { Thread, diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index b59536fd6e4f..ac5f92435dd8 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -18,38 +18,6 @@ inline void FormatExchange(std::ostream& out) { - out << R"XXX( -template -static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_exch, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_exch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_exch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_exch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; - // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -61,8 +29,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch( constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ + __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -78,12 +47,11 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( }} )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); }})XXX"; - - constexpr auto asm_intrinsic_format = R"XXX( + constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6}) -{{ asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) +{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; constexpr Operand supported_types[] = { Operand::Bit, @@ -135,7 +103,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -147,7 +115,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( size, constraints(type, size), semantic(sem), - semantic_tag(sem), + ptx_semantic_tag(sem), scope(sco), scope_tag(sco)); } @@ -164,65 +132,46 @@ template struct __cuda_atomic_bind_exchange { _Backend __backend; _Type* __ptr; - _Type* __old; - _Type* __new; + __unv<_Type>* __old; + __unv<_Type> __new; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_exchange(__backend, __ptr, *__old, *__new, __order, _Tag{}, _Sco{}); + __cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); } }; template _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( - _Backend __backend, _Type* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) + _Backend __backend, + _Type* __ptr, + __unv<_Type>& __old, + __unv<_Type> __new, + memory_order __order, + _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); if constexpr (_Backend::__requires_local_memory_workaround) { if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } - __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ - __backend, __ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); -} - -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( - _Backend __backend, _Type volatile* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} - } - __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ - __backend, __ptr_proxy, __old_proxy, __new_proxy}; + __cuda_atomic_bind_exchange<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_swap{ + __backend, __ptr_proxy, __old_proxy, *__new_proxy}; __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); } template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_exchange_dispatch( _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) { - _Type __old; - __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); - return __old; -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) -{ - _Type __old; - __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); + using __value_type = __unv<_Type>; + __value_type __old; + __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); return __old; } diff --git a/libcudacxx/codegen/generators/fence.h b/libcudacxx/codegen/generators/fence.h index df9fe67a0ecd..117b810c77f9 100644 --- a/libcudacxx/codegen/generators/fence.h +++ b/libcudacxx/codegen/generators/fence.h @@ -77,6 +77,20 @@ static inline _CCCL_DEVICE void __cuda_atomic_fence({0}, {2}) } out << "\n" << R"XXX( +template +static inline _CCCL_DEVICE void +__cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco __scope) +{ + if (__order.__was_seq_cst) + { + __cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); + } +} + +template +static inline _CCCL_DEVICE void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) +{} + template static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( __cuda_atomic_ptx_backend, memory_order __order, _Sco) { diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index 5e33c302848f..412094cd9cc6 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -17,13 +17,13 @@ #include "definitions.h" -inline std::string fetch_op_skip_v(std::string fetch_op) +inline std::string fetch_op_transform(std::string fetch_op) { if (fetch_op == "add") { - return "constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip;"; + return "\n __op = __op * __atomic_ptr_skip_t<_Type>::__skip;"; } - return "constexpr auto __skip_v = 1;"; + return {}; } inline void FormatFetchOps(std::ostream& out) @@ -50,39 +50,6 @@ inline void FormatFetchOps(std::ostream& out) std::pair{std::string{"and"}, std::pair{bitwise_types, std::string{"bitwise"}}}, }; - // Memory order dispatcher - out << R"XXX( -template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_fetch, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_fetch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_fetch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_fetch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; - // Argument ID Reference // 0 - Atomic Operation // 1 - Operand Type @@ -95,12 +62,11 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_fetch_{0}( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, {5}, __cuda_atomic_operand_{1}{2}, {7}) -{{ asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; - + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, {5} __order, __cuda_atomic_operand_{1}{2}, {7}) +{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {7}{{}}); asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; // 0 - Atomic Operation // 1 - Operand type constraint - // 2 - Pointer op skip_v + // 2 - Operand transform constexpr auto fetch_bind_invoke = R"XXX( #endif // _CCCL_CUDA_COMPILATION() @@ -108,61 +74,35 @@ template struct __cuda_atomic_bind_fetch_{0} {{ _Backend __backend; _Type* __ptr; - _Type* __dst; - _Type* __op; + __unv<_Type>* __dst; + __unv<_Type> __op; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) {{ - __cuda_atomic_fetch_{0}(__backend, __ptr, *__dst, *__op, __order, _Tag{{}}, _Sco{{}}); + __cuda_atomic_fetch_{0}(__backend, __ptr, *__dst, __op, __order, _Tag{{}}, _Sco{{}}); }} }}; template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_{0}_dispatch( + class _Sco> +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_{0}_dispatch( _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{{ - {2} - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_{1}_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_{1}_tag_t<_Type>; - _Type __dst{{}}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - {{ - if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} - }} - __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_{0}{{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope); - return __dst; -}} -template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_{0}_dispatch( - _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) -{{ - {2} - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_{1}_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_{1}_tag_t<_Type>; - _Type __dst{{}}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); +{{{2} + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_{1}_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_{1}_tag_t<__value_type>; + __value_type __dst{{}}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) {{ if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} }} - __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_{0}{{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}}; + __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_{0}{{ + __backend, __ptr_proxy, __dst_proxy, *__op_proxy}}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope); return __dst; }} @@ -223,14 +163,14 @@ template #include +#include #include #include #include diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index 9c05efa810f4..60704eacfba2 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -38,34 +38,6 @@ inline std::string scope_ld_st(Semantic sem, Scope sco) inline void FormatLoad(std::ostream& out) { - out << R"XXX( -template -static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_load, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; - // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -79,8 +51,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch( constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -97,16 +70,17 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) -{{ asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) +{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); uint16_t __tmp; asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); }})XXX"; constexpr size_t supported_sizes[] = { @@ -172,7 +146,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -186,7 +160,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -200,7 +174,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -220,7 +194,7 @@ template * __dst; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { @@ -229,51 +203,33 @@ struct __cuda_atomic_bind_load { }; template _CCCL_HOST_DEVICE_API void -__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, _Type& __dst, memory_order __order, _Sco __scope) +__cuda_atomic_load_dispatch( + _Backend __backend, + const _Type* __ptr, + __unv<_Type>& __dst, + memory_order __order, + _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - const __proxy_t* __ptr_proxy = reinterpret_cast(__ptr); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ - __backend, __ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); -} -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_load_dispatch( - _Backend __backend, const _Type volatile* __ptr, _Type& __dst, memory_order __order, _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - const __proxy_t* __ptr_proxy = reinterpret_cast(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} - } - __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __cuda_atomic_bind_load<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); } template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) { - _Type __dst; - __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); - return __dst; -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_load_dispatch(_Backend __backend, const volatile _Type* __ptr, memory_order __order, _Sco __scope) -{ - _Type __dst; + __unv<_Type> __dst; __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); return __dst; } @@ -284,31 +240,6 @@ __cuda_atomic_load_dispatch(_Backend __backend, const volatile _Type* __ptr, mem inline void FormatStore(std::ostream& out) { - out << R"XXX( -template -static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_store, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_RELEASE: __cuda_store(__cuda_atomic_order_release{}); break; - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_RELEASE: [[fallthrough]]; - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} -)XXX"; // Argument ID Reference // 0 - Operand Type // 1 - Operand Size @@ -322,8 +253,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch( constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -340,13 +272,14 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) -{{ asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) +{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, {3}, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ + __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); const uint16_t __tmp = static_cast(__val); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__tmp) : "memory"); }})XXX"; @@ -411,7 +344,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -425,7 +358,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -439,7 +372,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( /* 0 */ operand(type), /* 1 */ size, /* 2 */ constraints(type, size), - /* 3 */ semantic_tag(sem), + /* 3 */ ptx_semantic_tag(sem), /* 4 */ semantic_ld_st(sem), /* 5 */ scope_tag(sco), /* 6 */ scope_ld_st(sem, sco), @@ -459,45 +392,30 @@ template __val; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_store(__backend, __ptr, *__val, __order, _Tag{}, _Sco{}, _Mmio{}); + __cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; template _CCCL_HOST_DEVICE_API void __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - _Type __store = __val; - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} - } - __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ - __backend, __ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); -} -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_store_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - _Type __store = __val; + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __value_type __store = __val; __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ - __backend, __ptr_proxy, __val_proxy}; + __cuda_atomic_bind_store<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __backend, __ptr_proxy, *__val_proxy}; __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/backend.h b/libcudacxx/include/cuda/std/__atomic/functions/backend.h index c364bb7d61f0..398a617db39a 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/backend.h @@ -23,11 +23,12 @@ #include #include -#include #include +#include #include #include #include +#include #include #include @@ -55,12 +56,43 @@ using __cuda_atomic_order_acq_rel = __cuda_atomic_order_tag<__cuda_atomic_order using __cuda_atomic_order_seq_cst = __cuda_atomic_order_tag<__cuda_atomic_order::_seq_cst>; using __cuda_atomic_order_volatile = __cuda_atomic_order_tag<__cuda_atomic_order::_volatile>; +template +struct __cuda_atomic_ptx_order : _Order +{ + bool __was_seq_cst; + + _CCCL_HOST_DEVICE_API constexpr explicit __cuda_atomic_ptx_order(bool __was_seq_cst_) noexcept + : __was_seq_cst(__was_seq_cst_) + {} +}; + +using __cuda_atomic_ptx_order_relaxed = __cuda_atomic_ptx_order<__cuda_atomic_order_relaxed>; +using __cuda_atomic_ptx_order_release = __cuda_atomic_ptx_order<__cuda_atomic_order_release>; +using __cuda_atomic_ptx_order_acquire = __cuda_atomic_ptx_order<__cuda_atomic_order_acquire>; +using __cuda_atomic_ptx_order_acq_rel = __cuda_atomic_ptx_order<__cuda_atomic_order_acq_rel>; + +struct __cuda_atomic_operation_load +{}; + +struct __cuda_atomic_operation_store +{}; + +struct __cuda_atomic_operation_rmw +{}; + struct __cuda_atomic_runtime_cas_order { memory_order __success; memory_order __failure; }; +template +struct __cuda_atomic_cas_order +{ + using __success = _Success; + using __failure = _Failure; +}; + struct __cuda_atomic_cas_strong {}; @@ -77,43 +109,8 @@ struct __cuda_atomic_cas_weak : __cuda_atomic_cas_strong return true; } -template = 0> -_CCCL_HOST_DEVICE_API void -__cuda_atomic_load_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) -{ - __fn(__order); -} - -template = 0> -_CCCL_HOST_DEVICE_API void -__cuda_atomic_store_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) -{ - __fn(__order); -} - -template = 0> -_CCCL_HOST_DEVICE_API void -__cuda_atomic_exchange_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) -{ - __fn(__order); -} - -template = 0> -_CCCL_HOST_DEVICE_API void -__cuda_atomic_fetch_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) -{ - __fn(__order); -} - -template = 0> -_CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( - _Backend __backend, _Fn& __fn, memory_order __success, memory_order __failure, _Sco __scope) -{ - return __fn(__cuda_atomic_runtime_cas_order{__success, __failure}); -} - template -[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_initial_load_order() +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __cuda_atomic_initial_load_order(_Order) { if constexpr (is_same_v<_Order, __cuda_atomic_order_volatile>) { @@ -129,6 +126,19 @@ template } } +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __cuda_atomic_initial_load_order(__cuda_atomic_ptx_order<_Order>) +{ + return __cuda_atomic_ptx_order_relaxed{false}; +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +__cuda_atomic_compare_exchange_initial_load_order(__cuda_atomic_cas_order<_Success, _Failure>) +{ + return _Failure{}; +} + // Compare-exchange may fail after the initial load without issuing a CAS, so that load must satisfy the failure order. template [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto __cuda_atomic_compare_exchange_initial_load_order(_Order __order) @@ -149,7 +159,21 @@ template } else { - return __cuda_atomic_initial_load_order<_Order>(); + return __cuda_atomic_initial_load_order(__order); + } +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr auto +__cuda_atomic_compare_exchange_initial_load_order(__cuda_atomic_ptx_order<_Order> __order) +{ + if constexpr (is_same_v<_Order, __cuda_atomic_order_acquire> || is_same_v<_Order, __cuda_atomic_order_acq_rel>) + { + return __cuda_atomic_ptx_order_acquire{__order.__was_seq_cst}; + } + else + { + return __cuda_atomic_ptx_order_relaxed{false}; } } @@ -206,30 +230,37 @@ struct _CCCL_ALIGNAS(16) __cuda_atomic_longlong2 { uint64_t __x; uint64_t __y; -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_bitwise_impl() noexcept -{ - if constexpr (sizeof(_Type) == 1) + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __cuda_atomic_longlong2 + operator&(__cuda_atomic_longlong2 __lhs, __cuda_atomic_longlong2 __rhs) noexcept { - return __cuda_atomic_operand_deduction{}; + return {__lhs.__x & __rhs.__x, __lhs.__y & __rhs.__y}; } - else if constexpr (sizeof(_Type) == 2) + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __cuda_atomic_longlong2 + operator|(__cuda_atomic_longlong2 __lhs, __cuda_atomic_longlong2 __rhs) noexcept { - return __cuda_atomic_operand_deduction{}; + return {__lhs.__x | __rhs.__x, __lhs.__y | __rhs.__y}; } - else if constexpr (sizeof(_Type) == 4) + + [[nodiscard]] _CCCL_HOST_DEVICE_API friend constexpr __cuda_atomic_longlong2 + operator^(__cuda_atomic_longlong2 __lhs, __cuda_atomic_longlong2 __rhs) noexcept { - return __cuda_atomic_operand_deduction{}; + return {__lhs.__x ^ __rhs.__x, __lhs.__y ^ __rhs.__y}; } - else if constexpr (sizeof(_Type) == 8) +}; + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_bitwise_impl() noexcept +{ + using __tag = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, sizeof(_Type) * 8>; + if constexpr (sizeof(_Type) == 16) { - return __cuda_atomic_operand_deduction{}; + return __cuda_atomic_operand_deduction<__cuda_atomic_longlong2, __tag>{}; } else { - return __cuda_atomic_operand_deduction<__cuda_atomic_longlong2, __cuda_atomic_operand_b128>{}; + return __cuda_atomic_operand_deduction<__make_nbit_uint_t, __tag>{}; } } @@ -244,55 +275,18 @@ using __cuda_atomic_deduce_bitwise_tag_t = typename decltype(__cuda_atomic_deduc template [[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_arithmetic_impl() noexcept { - if constexpr (is_floating_point_v<_Type>) + constexpr bool __is_floating = is_floating_point_v<_Type> || __is_extended_floating_point_v<_Type>; + constexpr auto __op = + __is_floating ? __cuda_atomic_operand::_f + : (is_signed_v<_Type> && sizeof(_Type) != 8 ? __cuda_atomic_operand::_s : __cuda_atomic_operand::_u); + using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * 8>; + if constexpr (__is_floating || sizeof(_Type) == 16) { - if constexpr (sizeof(_Type) == 4) - { - return __cuda_atomic_operand_deduction{}; - } - else - { - return __cuda_atomic_operand_deduction{}; - } - } - else if constexpr (is_signed_v<_Type>) - { - if constexpr (sizeof(_Type) == 1) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 2) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 4) - { - return __cuda_atomic_operand_deduction{}; - } - else - { - // There is no atom.add.s64. - return __cuda_atomic_operand_deduction{}; - } + return __cuda_atomic_operand_deduction<_Type, __tag>{}; } else { - if constexpr (sizeof(_Type) == 1) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 2) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 4) - { - return __cuda_atomic_operand_deduction{}; - } - else - { - return __cuda_atomic_operand_deduction{}; - } + return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; } } @@ -305,54 +299,17 @@ using __cuda_atomic_deduce_arithmetic_tag_t = typename decltype(__cuda_atomic_de template [[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_minmax_impl() noexcept { - if constexpr (is_floating_point_v<_Type>) - { - if constexpr (sizeof(_Type) == 4) - { - return __cuda_atomic_operand_deduction{}; - } - else - { - return __cuda_atomic_operand_deduction{}; - } - } - else if constexpr (is_signed_v<_Type>) + constexpr bool __is_floating = is_floating_point_v<_Type> || __is_extended_floating_point_v<_Type>; + constexpr auto __op = __is_floating ? __cuda_atomic_operand::_f + : (is_signed_v<_Type> ? __cuda_atomic_operand::_s : __cuda_atomic_operand::_u); + using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * 8>; + if constexpr (__is_floating || sizeof(_Type) == 16) { - if constexpr (sizeof(_Type) == 1) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 2) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 4) - { - return __cuda_atomic_operand_deduction{}; - } - else - { - return __cuda_atomic_operand_deduction{}; - } + return __cuda_atomic_operand_deduction<_Type, __tag>{}; } else { - if constexpr (sizeof(_Type) == 1) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 2) - { - return __cuda_atomic_operand_deduction{}; - } - else if constexpr (sizeof(_Type) == 4) - { - return __cuda_atomic_operand_deduction{}; - } - else - { - return __cuda_atomic_operand_deduction{}; - } + return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; } } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/common.h b/libcudacxx/include/cuda/std/__atomic/functions/common.h index 07ece7c4abeb..ca9b967d0e94 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/common.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/common.h @@ -22,11 +22,15 @@ #endif // no system header #include +#include #include _CCCL_BEGIN_NAMESPACE_CUDA_STD +template +using __unv = remove_volatile_t<_Tp>; + template struct __atomic_ptr_skip { diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h index 49e770d646c1..bcbb8843d176 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h @@ -21,6 +21,7 @@ #endif // no system header #include +#include #include #include #include @@ -52,151 +53,170 @@ _CCCL_DEVICE_API inline bool __cuda_atomic_is_local(const volatile void* __ptr) } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_and(volatile _Type& __atom, _Type const& __v) +_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_and(_Type __atom, _Type const& __v) { - __atom = __atom & __v; + return __atom & __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_or(volatile _Type& __atom, _Type const& __v) +_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_or(_Type __atom, _Type const& __v) { - __atom = __atom | __v; + return __atom | __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_xor(volatile _Type& __atom, _Type const& __v) +_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_xor(_Type __atom, _Type const& __v) { - __atom = __atom ^ __v; + return __atom ^ __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_add(volatile _Type& __atom, _Type const& __v) +_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_add(_Type __atom, _Type const& __v) { - __atom = __atom + __v; + return __atom + __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_sub(volatile _Type& __atom, _Type const& __v) +_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_sub(_Type __atom, _Type const& __v) { - __atom = __atom - __v; + return __atom - __v; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_max(volatile _Type& __atom, _Type const& __v) +_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_max(_Type __atom, _Type const& __v) { - __atom = __atom < __v ? __v : __atom; + return __cuda_atomic_less(__atom, __v) ? __v : __atom; } template -_CCCL_DEVICE_API void __cuda_atomic_fetch_local_bop_min(volatile _Type& __atom, _Type const& __v) +_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_min(_Type __atom, _Type const& __v) { - __atom = __v < __atom ? __v : __atom; + return __cuda_atomic_less(__v, __atom) ? __v : __atom; } -_CCCL_DEVICE_API inline bool __cuda_atomic_load_weak_if_local(const volatile void* __ptr, void* __ret, size_t __size) +template +_CCCL_DEVICE_API bool +__cuda_atomic_load_weak_if_local(const _Type* __ptr, __unv<_Type>* __ret, [[maybe_unused]] size_t __size) { if (!__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(__ret, const_cast(__ptr), __size); + __atomic_assign_volatile(__ret, *__ptr); // Required to workaround a compiler bug, see nvbug/4064730 NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } -_CCCL_DEVICE_API inline bool __cuda_atomic_store_weak_if_local(volatile void* __ptr, const void* __val, size_t __size) +template +_CCCL_DEVICE_API bool +__cuda_atomic_store_weak_if_local(_Type* __ptr, const __unv<_Type>* __val, [[maybe_unused]] size_t __size) { if (!__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(const_cast(__ptr), __val, __size); + __atomic_assign_volatile(__ptr, *__val); return true; } template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_weak_if_local( - volatile _Type* __ptr, _Type* __expected, const _Type* __desired, bool* __success) + _Type* __ptr, __unv<_Type>* __expected, const __unv<_Type>* __desired, bool* __success) { if (!__cuda_atomic_is_local(__ptr)) { return false; } - if (__atomic_memcmp(const_cast(__ptr), const_cast(__expected), sizeof(_Type)) == 0) + using _ValueType = __unv<_Type>; + _ValueType __old{}; + __atomic_assign_volatile(&__old, *__ptr); + if (__atomic_memcmp(&__old, __expected, sizeof(_ValueType)) == 0) { - ::cuda::std::memcpy(const_cast<_Type*>(__ptr), const_cast<_Type const*>(__desired), sizeof(_Type)); + __atomic_assign_volatile(__ptr, *__desired); *__success = true; } else { - ::cuda::std::memcpy(const_cast<_Type*>(__expected), const_cast<_Type const*>(__ptr), sizeof(_Type)); - *__success = false; + *__expected = __old; + *__success = false; } NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } template -_CCCL_DEVICE_API bool __cuda_atomic_exchange_weak_if_local(volatile _Type* __ptr, _Type* __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_exchange_weak_if_local(_Type* __ptr, __unv<_Type>* __val, __unv<_Type>* __ret) { if (!__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(const_cast<_Type*>(__ret), const_cast(__ptr), sizeof(_Type)); - ::cuda::std::memcpy(const_cast<_Type*>(__ptr), const_cast(__val), sizeof(_Type)); + __atomic_assign_volatile(__ret, *__ptr); + __atomic_assign_volatile(__ptr, *__val); NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret, _BOp&& __bop) +_CCCL_DEVICE_API bool +__cuda_atomic_fetch_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret, _BOp&& __bop) { if (!__cuda_atomic_is_local(__ptr)) { return false; } - ::cuda::std::memcpy(const_cast<_Type*>(__ret), const_cast(__ptr), sizeof(_Type)); - __bop(*__ptr, __val); + using _ValueType = __unv<_Type>; + _ValueType __old{}; + __atomic_assign_volatile(&__old, *__ptr); + *__ret = __old; + const _ValueType __desired = __bop(__old, __val); + __atomic_assign_volatile(__ptr, __desired); NV_IF_TARGET(NV_PROVIDES_SM_70, (__nanosleep(0);)) return true; } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_and_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_and_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_and<_Type>); + using _ValueType = __unv<_Type>; + return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_and<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_or_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_or_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_or<_Type>); + using _ValueType = __unv<_Type>; + return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_or<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_xor_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_xor_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_xor<_Type>); + using _ValueType = __unv<_Type>; + return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_xor<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_add_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_add_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_add<_Type>); + using _ValueType = __unv<_Type>; + return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_add<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_sub_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_sub_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_sub<_Type>); + using _ValueType = __unv<_Type>; + return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_sub<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_max_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_max_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_max<_Type>); + using _ValueType = __unv<_Type>; + return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_max<_ValueType>); } template -_CCCL_DEVICE_API bool __cuda_atomic_fetch_min_weak_if_local(volatile _Type* __ptr, _Type __val, _Type* __ret) +_CCCL_DEVICE_API bool __cuda_atomic_fetch_min_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_min<_Type>); + using _ValueType = __unv<_Type>; + return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_min<_ValueType>); } #endif // _CCCL_CUDA_COMPILATION() diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h index 62d6f768d382..6a2b7bf882a4 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -21,18 +21,85 @@ # pragma system_header #endif // no system header +#include #include #include #include #include #include #include +#include #include #include _CCCL_BEGIN_NAMESPACE_CUDA_STD +template +_CCCL_DEVICE_API auto +__cuda_atomic_ptx_backend::__with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope) + -> decltype(__fn(__order)) +{ + constexpr bool __is_load = is_same_v<_Operation, __cuda_atomic_operation_load>; + constexpr bool __is_store = is_same_v<_Operation, __cuda_atomic_operation_store>; + constexpr bool __is_rmw = is_same_v<_Operation, __cuda_atomic_operation_rmw>; + static_assert(__is_load || __is_store || __is_rmw, "invalid atomic operation class"); + + constexpr bool __is_seq_cst = is_same_v<_Order, __cuda_atomic_order_seq_cst>; + constexpr bool __is_release = is_same_v<_Order, __cuda_atomic_order_release>; + constexpr bool __is_acq_rel = is_same_v<_Order, __cuda_atomic_order_acq_rel>; + constexpr bool __is_acquire = is_same_v<_Order, __cuda_atomic_order_acquire>; + constexpr bool __membar_before = + __is_seq_cst || (__is_store && __is_release) || (__is_rmw && (__is_release || __is_acq_rel)); + constexpr bool __membar_after = (__is_load || __is_rmw) && (__is_acquire || __is_acq_rel || __is_seq_cst); + + NV_DISPATCH_TARGET( + NV_PROVIDES_SM_70, + ({ + if constexpr (__is_seq_cst) + { + if constexpr (__is_store) + { + return __fn(__cuda_atomic_ptx_order_relaxed{true}); + } + else + { + return __fn(__cuda_atomic_ptx_order_acquire{true}); + } + } + else + { + return __fn(__transform_order(__order)); + } + }), + NV_IS_DEVICE, + ({ + if constexpr (__membar_before) + { + __cuda_atomic_membar(__scope); + } + if constexpr (__membar_after) + { + if constexpr (is_void_v) + { + __fn(__cuda_atomic_order_volatile{}); + __cuda_atomic_membar(__scope); + return; + } + else + { + auto __result = __fn(__cuda_atomic_order_volatile{}); + __cuda_atomic_membar(__scope); + return __result; + } + } + else + { + return __fn(__cuda_atomic_order_volatile{}); + } + })) +} + template , bool> = false> [[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_ptx_negate(_Type __value) { @@ -48,9 +115,19 @@ template , bool> = false> return -__value; } -template +template = 32) && (_Operand::__size <= 64), bool> = false> _CCCL_DEVICE_API void __cuda_atomic_fetch_sub( - __cuda_atomic_ptx_backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + __cuda_atomic_ptx_backend __backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + _Order __order, + _Operand, + _Sco __scope) { __cuda_atomic_fetch_add(__backend, __ptr, __dst, __cuda_atomic_ptx_negate(__op), __order, _Operand{}, __scope); } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h index 62d1d5b000b0..24de4ae2c91f 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h @@ -22,7 +22,6 @@ #endif // no system header #include -#include #include #include #include @@ -33,24 +32,65 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD struct __cuda_atomic_ptx_backend { - template - using __enable_if_direct_bitwise = enable_if_t<(sizeof(_Type) < 16), bool>; - - template - using __enable_if_direct_arithmetic = enable_if_t && (sizeof(_Type) < 16), bool>; - - template - using __enable_if_direct_minmax = enable_if_t && (sizeof(_Type) < 16), bool>; - - template - using __enable_if_fallback_bitwise = enable_if_t<(sizeof(_Type) == 16), bool>; - - template - using __enable_if_fallback_arithmetic = enable_if_t && (sizeof(_Type) == 16), bool>; - - template - using __enable_if_fallback_minmax = - enable_if_t || (is_scalar_v<_Type> && sizeof(_Type) == 16), bool>; + template + _CCCL_HOST_DEVICE_API static constexpr auto __transform_order(_Order) + { + if constexpr (is_same_v<_Order, __cuda_atomic_order_relaxed>) + { + return __cuda_atomic_ptx_order_relaxed{false}; + } + else if constexpr (is_same_v<_Order, __cuda_atomic_order_release>) + { + return __cuda_atomic_ptx_order_release{false}; + } + else if constexpr (is_same_v<_Order, __cuda_atomic_order_acquire>) + { + return __cuda_atomic_ptx_order_acquire{false}; + } + else + { + static_assert(is_same_v<_Order, __cuda_atomic_order_acq_rel>, "invalid PTX atomic order"); + return __cuda_atomic_ptx_order_acq_rel{false}; + } + } + + template + _CCCL_HOST_DEVICE_API static constexpr _Order __collapse_cas_order(_Order __order) + { + return __order; + } + + template + _CCCL_HOST_DEVICE_API static constexpr auto __collapse_cas_order(__cuda_atomic_cas_order<_Success, _Failure>) + { + if constexpr (is_same_v<_Success, __cuda_atomic_order_seq_cst> || is_same_v<_Failure, __cuda_atomic_order_seq_cst>) + { + return __cuda_atomic_order_seq_cst{}; + } + else if constexpr (is_same_v<_Success, __cuda_atomic_order_acq_rel> + || (is_same_v<_Success, __cuda_atomic_order_release> + && is_same_v<_Failure, __cuda_atomic_order_acquire>) ) + { + return __cuda_atomic_order_acq_rel{}; + } + else if constexpr (is_same_v<_Success, __cuda_atomic_order_release>) + { + return __cuda_atomic_order_release{}; + } + else if constexpr (is_same_v<_Success, __cuda_atomic_order_acquire> + || is_same_v<_Failure, __cuda_atomic_order_acquire>) + { + return __cuda_atomic_order_acquire{}; + } + else + { + return __cuda_atomic_order_relaxed{}; + } + } + + template + _CCCL_DEVICE_API static auto __with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope) + -> decltype(__fn(__order)); static constexpr bool __needs_constant_order = true; static constexpr bool __requires_local_memory_workaround = true; diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index a381247eebbf..d4bc435a58f9 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -72,6 +73,20 @@ static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_system_tag, _ static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_system_tag, __cuda_atomic_order_seq_cst) { asm volatile("fence.sc.sys;" ::: "memory"); } +template +static inline _CCCL_DEVICE void +__cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco __scope) +{ + if (__order.__was_seq_cst) + { + __cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); + } +} + +template +static inline _CCCL_DEVICE void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) +{} + template static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( __cuda_atomic_ptx_backend, memory_order __order, _Sco) { @@ -102,920 +117,934 @@ static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( ) } -template -static inline _CCCL_DEVICE void __cuda_atomic_load_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_load, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_load(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELAXED: __cuda_load(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); - __dst = static_cast<_Type>(__tmp); + __dst = static_cast<__unv<_Type>>(__tmp); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1031,8 +1060,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1048,8 +1078,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1065,8 +1096,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1082,8 +1114,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1099,8 +1132,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1116,8 +1150,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1133,8 +1168,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1150,8 +1186,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1167,8 +1204,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1184,8 +1222,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1201,8 +1240,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1218,8 +1258,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( } template static inline _CCCL_DEVICE void __cuda_atomic_load( - __cuda_atomic_ptx_backend, const _Type* __ptr, _Type& __dst, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1240,7 +1281,7 @@ template * __dst; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { @@ -1249,332 +1290,304 @@ struct __cuda_atomic_bind_load { }; template _CCCL_HOST_DEVICE_API void -__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, _Type& __dst, memory_order __order, _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - const __proxy_t* __ptr_proxy = reinterpret_cast(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} - } - __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ - __backend, __ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); -} -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_load_dispatch( - _Backend __backend, const _Type volatile* __ptr, _Type& __dst, memory_order __order, _Sco __scope) +__cuda_atomic_load_dispatch( + _Backend __backend, + const _Type* __ptr, + __unv<_Type>& __dst, + memory_order __order, + _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - const __proxy_t* __ptr_proxy = reinterpret_cast(const_cast<_Type*>(__ptr)); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_load<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __cuda_atomic_bind_load<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); } template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) { - _Type __dst; - __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); - return __dst; -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_load_dispatch(_Backend __backend, const volatile _Type* __ptr, memory_order __order, _Sco __scope) -{ - _Type __dst; + __unv<_Type> __dst; __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); return __dst; } #if _CCCL_CUDA_COMPILATION() -template -static inline _CCCL_DEVICE void __cuda_atomic_store_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_store, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_RELEASE: __cuda_store(__cuda_atomic_order_release{}); break; - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_RELEASE: [[fallthrough]]; - case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_RELAXED: __cuda_store(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.mmio.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1590,8 +1603,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1607,8 +1621,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1624,8 +1639,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1641,8 +1657,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1658,8 +1675,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1675,8 +1693,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1692,8 +1711,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1709,8 +1729,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1726,8 +1747,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1743,8 +1765,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1760,8 +1783,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1777,8 +1801,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __val, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1799,247 +1824,200 @@ template __val; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_store(__backend, __ptr, *__val, __order, _Tag{}, _Sco{}, _Mmio{}); + __cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; template _CCCL_HOST_DEVICE_API void __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - _Type __store = __val; - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} - } - __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ - __backend, __ptr_proxy, __val_proxy}; - __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); -} -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_store_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - _Type __store = __val; + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __value_type __store = __val; __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_store<_Backend, __proxy_t, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ - __backend, __ptr_proxy, __val_proxy}; + __cuda_atomic_bind_store<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __backend, __ptr_proxy, *__val_proxy}; __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); } #if _CCCL_CUDA_COMPILATION() -template -[[nodiscard]] static _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_cas, memory_order __success, memory_order __failure, _Sco) { - const int __success_memorder = __atomic_order_to_int(__success); - const int __failure_memorder = __atomic_failure_order_to_int(__failure); - bool __res = false; - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __res = __cuda_cas(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __res = __cuda_cas(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) - return __res; -} - template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2057,8 +2035,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2076,8 +2055,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2095,8 +2075,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2114,8 +2095,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2133,8 +2115,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2152,8 +2135,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2171,8 +2155,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2190,8 +2175,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2209,8 +2195,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2228,8 +2215,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2247,8 +2235,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2266,8 +2255,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2285,8 +2275,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2304,8 +2295,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2323,8 +2315,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2342,8 +2335,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2361,8 +2355,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2380,8 +2375,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2399,8 +2395,9 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2423,55 +2420,32 @@ template * __exp; + __unv<_Type> __cmp; + __unv<_Type> __des; template [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { return __cuda_atomic_compare_exchange( - __backend, __ptr, *__exp, *__exp, *__des, _Cas{}, __order, _Tag{}, _Sco{}); + __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); } }; template [[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( _Backend __backend, _Type* __ptr, - _Type* __exp, - _Type __des, - _Cas, - memory_order __success, - memory_order __failure, - _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); - bool __res = false; - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch( - __backend, __bound_compare_swap, __success, __failure, __scope); -} -template -[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( - _Backend __backend, - _Type volatile* __ptr, - _Type* __exp, - _Type __des, + __unv<_Type>* __exp, + __unv<_Type> __des, _Cas, memory_order __success, memory_order __failure, _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); bool __res = false; @@ -2479,208 +2453,179 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_t, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, __des_proxy}; + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( __backend, __bound_compare_swap, __success, __failure, __scope); } #if _CCCL_CUDA_COMPILATION() -template -static inline _CCCL_DEVICE void __cuda_atomic_exchange_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_exch, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_exch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_exch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_exch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2698,8 +2643,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2717,8 +2663,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2736,8 +2683,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acquire, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2755,8 +2703,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2774,8 +2723,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2793,8 +2743,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2812,8 +2763,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2831,8 +2783,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2850,8 +2803,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2869,8 +2823,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2888,8 +2843,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_release, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2907,8 +2863,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2926,8 +2883,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2945,8 +2903,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2964,8 +2923,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2983,8 +2943,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_block_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -3002,8 +2963,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -3021,8 +2983,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_device_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -3040,8 +3003,9 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __old, _Type __new, __cuda_atomic_order_volatile, __cuda_atomic_operand_b128, __thread_scope_system_tag) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { + __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -3064,500 +3028,451 @@ template struct __cuda_atomic_bind_exchange { _Backend __backend; _Type* __ptr; - _Type* __old; - _Type* __new; + __unv<_Type>* __old; + __unv<_Type> __new; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_exchange(__backend, __ptr, *__old, *__new, __order, _Tag{}, _Sco{}); + __cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); } }; template _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( - _Backend __backend, _Type* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) -{ - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} - } - __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ - __backend, __ptr_proxy, __old_proxy, __new_proxy}; - __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); -} - -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( - _Backend __backend, _Type volatile* __ptr, _Type& __old, _Type __new, memory_order __order, _Sco __scope) + _Backend __backend, + _Type* __ptr, + __unv<_Type>& __old, + __unv<_Type> __new, + memory_order __order, + _Sco __scope) { - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); if constexpr (_Backend::__requires_local_memory_workaround) { if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } - __cuda_atomic_bind_exchange<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_swap{ - __backend, __ptr_proxy, __old_proxy, __new_proxy}; + __cuda_atomic_bind_exchange<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_swap{ + __backend, __ptr_proxy, __old_proxy, *__new_proxy}; __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); } template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_exchange_dispatch( _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) { - _Type __old; - __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); - return __old; -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_exchange_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) -{ - _Type __old; - __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<_Type>(__new), __order, __scope); + using __value_type = __unv<_Type>; + __value_type __old; + __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); return __old; } #if _CCCL_CUDA_COMPILATION() -template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_order_dispatch( - __cuda_atomic_ptx_backend, _Fn& __cuda_fetch, memory_order __order, _Sco) { - const int __memorder = __atomic_order_to_int(__order); - NV_DISPATCH_TARGET( - NV_PROVIDES_SM_70, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_acquire{}); break; - case __ATOMIC_ACQ_REL: __cuda_fetch(__cuda_atomic_order_acq_rel{}); break; - case __ATOMIC_RELEASE: __cuda_fetch(__cuda_atomic_order_release{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_relaxed{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ), - NV_IS_DEVICE, ( - switch (__memorder) { - case __ATOMIC_SEQ_CST: [[fallthrough]]; - case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]]; - case __ATOMIC_CONSUME: [[fallthrough]]; - case __ATOMIC_ACQUIRE: __cuda_fetch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_fetch(__cuda_atomic_order_volatile{}); break; - case __ATOMIC_RELAXED: __cuda_fetch(__cuda_atomic_order_volatile{}); break; - default: _CCCL_ASSERT(false, "invalid memory order"); - } - ) - ) -} - template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -3565,61 +3480,36 @@ template struct __cuda_atomic_bind_fetch_add { _Backend __backend; _Type* __ptr; - _Type* __dst; - _Type* __op; + __unv<_Type>* __dst; + __unv<_Type> __op; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_fetch_add(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); + __cuda_atomic_fetch_add(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch( + class _Sco> +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_add_dispatch( _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { - constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_arithmetic_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } - __cuda_atomic_bind_fetch_add<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_add{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch( - _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_arithmetic_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); + __op = __op * __atomic_ptr_skip_t<_Type>::__skip; + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_arithmetic_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } - __cuda_atomic_bind_fetch_add<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_add{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_bind_fetch_add<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_add{ + __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope); return __dst; } @@ -3629,164 +3519,164 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -3794,61 +3684,35 @@ template struct __cuda_atomic_bind_fetch_and { _Backend __backend; _Type* __ptr; - _Type* __dst; - _Type* __op; + __unv<_Type>* __dst; + __unv<_Type> __op; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_fetch_and(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); + __cuda_atomic_fetch_and(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch( + class _Sco> +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_and_dispatch( _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } - __cuda_atomic_bind_fetch_and<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_and{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch( - _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } - __cuda_atomic_bind_fetch_and<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_and{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_bind_fetch_and<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_and{ + __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope); return __dst; } @@ -3858,324 +3722,324 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -4183,61 +4047,35 @@ template struct __cuda_atomic_bind_fetch_max { _Backend __backend; _Type* __ptr; - _Type* __dst; - _Type* __op; + __unv<_Type>* __dst; + __unv<_Type> __op; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_fetch_max(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); + __cuda_atomic_fetch_max(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch( + class _Sco> +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_max_dispatch( _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } - __cuda_atomic_bind_fetch_max<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_max{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch( - _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } - __cuda_atomic_bind_fetch_max<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_max{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_bind_fetch_max<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_max{ + __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope); return __dst; } @@ -4247,324 +4085,324 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -4572,61 +4410,35 @@ template struct __cuda_atomic_bind_fetch_min { _Backend __backend; _Type* __ptr; - _Type* __dst; - _Type* __op; + __unv<_Type>* __dst; + __unv<_Type> __op; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_fetch_min(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); + __cuda_atomic_fetch_min(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch( + class _Sco> +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_min_dispatch( _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } - __cuda_atomic_bind_fetch_min<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_min{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch( - _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_minmax_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } - __cuda_atomic_bind_fetch_min<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_min{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_bind_fetch_min<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_min{ + __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope); return __dst; } @@ -4636,164 +4448,164 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -4801,61 +4613,35 @@ template struct __cuda_atomic_bind_fetch_or { _Backend __backend; _Type* __ptr; - _Type* __dst; - _Type* __op; + __unv<_Type>* __dst; + __unv<_Type> __op; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_fetch_or(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); + __cuda_atomic_fetch_or(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch( + class _Sco> +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_or_dispatch( _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } - __cuda_atomic_bind_fetch_or<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_or{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch( - _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } - __cuda_atomic_bind_fetch_or<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_or{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_bind_fetch_or<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_or{ + __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope); return __dst; } @@ -4865,164 +4651,164 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acquire, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_relaxed, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_release, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_acq_rel, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( - __cuda_atomic_ptx_backend, _Type* __ptr, _Type& __dst, _Type __op, __cuda_atomic_order_volatile, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) +{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -5030,61 +4816,35 @@ template struct __cuda_atomic_bind_fetch_xor { _Backend __backend; _Type* __ptr; - _Type* __dst; - _Type* __op; + __unv<_Type>* __dst; + __unv<_Type> __op; template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { - __cuda_atomic_fetch_xor(__backend, __ptr, *__dst, *__op, __order, _Tag{}, _Sco{}); + __cuda_atomic_fetch_xor(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch( + class _Sco> +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_xor_dispatch( _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } - __cuda_atomic_bind_fetch_xor<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_xor{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope); - return __dst; -} -template = 0> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch( - _Backend __backend, _Type volatile* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - constexpr auto __skip_v = 1; - __op = __op * __skip_v; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<_Type>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<_Type>; - _Type __dst{}; - __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr)); + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) { if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } - __cuda_atomic_bind_fetch_xor<_Backend, __proxy_t, __proxy_tag, _Sco> __bound_xor{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + __cuda_atomic_bind_fetch_xor<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_xor{ + __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope); return __dst; } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index 3d2211d148ca..afc13633ca11 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -21,313 +21,240 @@ # pragma system_header #endif // no system header +#include +#include #include +#include #include +#include +#include +#include #include _CCCL_BEGIN_NAMESPACE_CUDA_STD -template -struct __cuda_atomic_bind_fetch_fallback +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_load_order_dispatch( + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope) { - _Backend __backend; - _Type* __ptr; - _Type* __dst; - _Type* __op; - _Fn __fn; - - template - _CCCL_HOST_DEVICE_API void operator()(_Order __order) + if constexpr (!_Backend::__needs_constant_order) { - __fn(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); + __fn(__order); } -}; - -template -struct __cuda_atomic_bind_fetch_sub -{ - _Backend __backend; - _Type* __ptr; - _Type* __dst; - _Type* __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Order __order) + else { - __cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); - } -}; - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_fallback_dispatch( - _Backend __backend, _Type* __ptr, _Type __op, memory_order __order, _Sco __scope, _Fn __fn) -{ - using __operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, sizeof(_Type) * 8>; - _Type __dst{}; - __cuda_atomic_bind_fetch_fallback<_Backend, _Type, _Fn, __operand, _Sco> __bound_fetch_fallback{ - __backend, __ptr, &__dst, &__op, __fn}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_fallback, __order, __scope); - return __dst; -} - -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - constexpr auto __skip = __atomic_ptr_skip_t<_Type>::__skip; - __op = __op * __skip; - using __proxy_type = __cuda_atomic_deduce_arithmetic_t<_Type>; - using __proxy_operand = __cuda_atomic_deduce_arithmetic_tag_t<_Type>; - _Type __dst{}; - auto* __ptr_proxy = reinterpret_cast<__proxy_type*>(__ptr); - auto* __dst_proxy = reinterpret_cast<__proxy_type*>(&__dst); - auto* __op_proxy = reinterpret_cast<__proxy_type*>(&__op); - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + switch (__atomic_order_to_int(__order)) { - return __dst; + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_relaxed{}, __scope); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_acquire{}, __scope); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_seq_cst{}, __scope); + default: + _CCCL_ASSERT(false, "invalid load memory order"); } } - __cuda_atomic_bind_fetch_sub<_Backend, __proxy_type, __proxy_operand, _Sco> __bound_fetch_sub{ - __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_sub, __order, __scope); - return __dst; } -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_store_order_dispatch( + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope) { - return __cuda_atomic_fetch_sub_dispatch(__backend, const_cast<_Type*>(__ptr), __op, __order, __scope); -} - -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_add_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - constexpr auto __skip = __atomic_ptr_skip_t<_Type>::__skip; - const _Type __op = static_cast<_Type>(__val * __skip); - if constexpr (_Backend::__requires_local_memory_workaround) + if constexpr (!_Backend::__needs_constant_order) { - _Type __dst{}; - if (__cuda_atomic_fetch_add_weak_if_local(__ptr, __op, &__dst)) - { - return __dst; - } + __fn(__order); } - return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_add_op{}); -} - -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_add_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - return __cuda_atomic_fetch_add_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); -} - -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - constexpr auto __skip = __atomic_ptr_skip_t<_Type>::__skip; - const _Type __op = static_cast<_Type>(__val * __skip); - if constexpr (_Backend::__requires_local_memory_workaround) + else { - _Type __dst{}; - if (__cuda_atomic_fetch_sub_weak_if_local(__ptr, __op, &__dst)) + switch (__atomic_order_to_int(__order)) { - return __dst; + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_relaxed{}, __scope); + case __ATOMIC_RELEASE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_release{}, __scope); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_seq_cst{}, __scope); + default: + _CCCL_ASSERT(false, "invalid store memory order"); } } - return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_sub_op{}); -} - -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_sub_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - return __cuda_atomic_fetch_sub_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); } -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_and_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_rmw_order_dispatch( + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope) { - const _Type __op = static_cast<_Type>(__val); - if constexpr (_Backend::__requires_local_memory_workaround) + if constexpr (!_Backend::__needs_constant_order) { - _Type __dst{}; - if (__cuda_atomic_fetch_and_weak_if_local(__ptr, __op, &__dst)) + __fn(__order); + } + else + { + switch (__atomic_order_to_int(__order)) { - return __dst; + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_relaxed{}, __scope); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acquire{}, __scope); + case __ATOMIC_RELEASE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_release{}, __scope); + case __ATOMIC_ACQ_REL: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acq_rel{}, __scope); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_seq_cst{}, __scope); + default: + _CCCL_ASSERT(false, "invalid read-modify-write memory order"); } } - return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_and_op{}); } -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_and_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_exchange_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) { - return __cuda_atomic_fetch_and_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); + __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope); } -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_or_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_fetch_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) { - const _Type __op = static_cast<_Type>(__val); - if constexpr (_Backend::__requires_local_memory_workaround) - { - _Type __dst{}; - if (__cuda_atomic_fetch_or_weak_if_local(__ptr, __op, &__dst)) - { - return __dst; - } - } - return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_or_op{}); + __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope); } -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_or_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool +__cuda_atomic_compare_exchange_failure_order_dispatch(_Backend __backend, _Fn& __fn, int __failure, _Sco __scope) { - return __cuda_atomic_fetch_or_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); + switch (__failure) + { + case __ATOMIC_RELAXED: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, + __fn, + __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_relaxed>{}), + __scope); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, + __fn, + __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_acquire>{}), + __scope); + case __ATOMIC_SEQ_CST: + return __backend.__with_transformed_order( + __cuda_atomic_operation_rmw{}, + __fn, + __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_seq_cst>{}), + __scope); + default: + _CCCL_ASSERT(false, "invalid compare-exchange failure memory order"); + _CCCL_UNREACHABLE(); + } } -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_xor_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( + [[maybe_unused]] _Backend __backend, + _Fn& __fn, + memory_order __success, + memory_order __failure, + [[maybe_unused]] _Sco __scope) { - const _Type __op = static_cast<_Type>(__val); - if constexpr (_Backend::__requires_local_memory_workaround) + if constexpr (!_Backend::__needs_constant_order) + { + return __fn(__cuda_atomic_runtime_cas_order{__success, __failure}); + } + else { - _Type __dst{}; - if (__cuda_atomic_fetch_xor_weak_if_local(__ptr, __op, &__dst)) + const int __failure_order = __atomic_failure_order_to_int(__failure); + switch (__atomic_order_to_int(__success)) { - return __dst; + case __ATOMIC_RELAXED: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_relaxed>( + __backend, __fn, __failure_order, __scope); + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_acquire>( + __backend, __fn, __failure_order, __scope); + case __ATOMIC_RELEASE: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_release>( + __backend, __fn, __failure_order, __scope); + case __ATOMIC_ACQ_REL: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_acq_rel>( + __backend, __fn, __failure_order, __scope); + case __ATOMIC_SEQ_CST: + return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_seq_cst>( + __backend, __fn, __failure_order, __scope); + default: + _CCCL_ASSERT(false, "invalid compare-exchange success memory order"); + _CCCL_UNREACHABLE(); } } - return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_xor_op{}); } -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_xor_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +template +struct __cuda_atomic_bind_fetch_sub { - return __cuda_atomic_fetch_xor_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); -} + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type>* __op; -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_min_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - const _Type __op = static_cast<_Type>(__val); - if constexpr (_Backend::__requires_local_memory_workaround) + template + _CCCL_HOST_DEVICE_API void operator()(_Order __order, _Operand, _Sco) { - _Type __dst{}; - if (__cuda_atomic_fetch_min_weak_if_local(__ptr, __op, &__dst)) - { - return __dst; - } + ::cuda::std::__cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); } - return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_min_op{}); -} - -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_min_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - return __cuda_atomic_fetch_min_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); -} +}; -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_max_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) { - const _Type __op = static_cast<_Type>(__val); + using __value_type = __unv<_Type>; + constexpr auto __skip = __atomic_ptr_skip_t<__value_type>::__skip; + __op = __op * __skip; + using __proxy_type = __cuda_atomic_deduce_arithmetic_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_type>; + using __proxy_operand = __cuda_atomic_deduce_arithmetic_tag_t<__value_type>; + __value_type __dst{}; + auto* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + auto* __dst_proxy = reinterpret_cast<__proxy_type*>(&__dst); + auto* __op_proxy = reinterpret_cast<__proxy_type*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { - _Type __dst{}; - if (__cuda_atomic_fetch_max_weak_if_local(__ptr, __op, &__dst)) + if (::cuda::std::__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) { return __dst; } } - return __cuda_atomic_fetch_fallback_dispatch(__backend, __ptr, __op, __order, __scope, __cuda_atomic_fetch_max_op{}); -} - -template = false> -[[nodiscard]] _CCCL_HOST_DEVICE_API _Type __cuda_atomic_fetch_max_dispatch( - _Backend __backend, volatile _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - return __cuda_atomic_fetch_max_dispatch(__backend, const_cast<_Type*>(__ptr), __val, __order, __scope); +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_sub<_Backend, __proxy_pointee> __bound_fetch_sub{ + __backend, __ptr_proxy, __dst_proxy, __op_proxy}; + ::cuda::std::__cuda_atomic_fetch_order_dispatch( + __backend, __bound_fetch_sub, __order, __scope, __proxy_operand{}); + return __dst; } #if _CCCL_CUDA_COMPILATION() diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic.h b/libcudacxx/include/cuda/std/__atomic/functions/generic.h index c7dbde3b7926..922a26567fcc 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic.h @@ -21,6 +21,7 @@ # pragma system_header #endif // no system header +#include #include #include #include @@ -31,7 +32,7 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD template = false> _CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, _Cas, _Order __order, _Operand, _Sco __scope) -{ + _Backend __backend, + _Pointee* __ptr, + __unv<_Pointee>& __dst, + __unv<_Pointee> __cmp, + __unv<_Pointee> __op, + _Cas, + _Order __order, + _Operand, + _Sco __scope) +{ + using _Type = __unv<_Pointee>; constexpr size_t __rmw_size = _Backend::__smallest_cas; static_assert(__rmw_size <= _Backend::__widest_cas, "atomic CAS cannot be widened beyond the backend's widest CAS"); @@ -59,15 +69,15 @@ _CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( } template _Backend::__widest_cas), bool> = false> -_CCCL_HOST_DEVICE static bool -__cuda_atomic_compare_exchange(_Backend, _Type*, _Type&, _Type, _Type, _Cas, _Order, _Operand, _Sco) +_CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( + _Backend, _Pointee*, __unv<_Pointee>&, __unv<_Pointee>, __unv<_Pointee>, _Cas, _Order, _Operand, _Sco) { static_assert(_Operand::__size < _Backend::__widest_cas, "the backend must provide its widest CAS operation"); return false; @@ -80,10 +90,11 @@ template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_add( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_add>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_add>{__op}, __order, _Operand{}, __scope); } template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_sub( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_sub>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_sub>{__op}, __order, _Operand{}, __scope); } -struct __cuda_atomic_fetch_and_op -{ - template - _CCCL_HOST_DEVICE_API void - operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const - { - __cuda_atomic_fetch_and(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); - } -}; - -struct __cuda_atomic_fetch_add_op -{ - template - _CCCL_HOST_DEVICE_API void - operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const - { - __cuda_atomic_fetch_add(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); - } -}; - -struct __cuda_atomic_fetch_sub_op -{ - template - _CCCL_HOST_DEVICE_API void - operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const - { - __cuda_atomic_fetch_sub(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); - } -}; - -struct __cuda_atomic_fetch_or_op -{ - template - _CCCL_HOST_DEVICE_API void - operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const - { - __cuda_atomic_fetch_or(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); - } -}; - -struct __cuda_atomic_fetch_xor_op -{ - template - _CCCL_HOST_DEVICE_API void - operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const - { - __cuda_atomic_fetch_xor(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); - } -}; - -struct __cuda_atomic_fetch_min_op -{ - template - _CCCL_HOST_DEVICE_API void - operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const - { - __cuda_atomic_fetch_min(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); - } -}; - -struct __cuda_atomic_fetch_max_op -{ - template - _CCCL_HOST_DEVICE_API void - operator()(_Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) const - { - __cuda_atomic_fetch_max(__backend, __ptr, __dst, __op, __order, _Operand{}, __scope); - } -}; - template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_and( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, ::cuda::std::bit_and>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_and>{__op}, __order, _Operand{}, __scope); } template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_or( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, ::cuda::std::bit_or>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_or>{__op}, __order, _Operand{}, __scope); } template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_xor( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, ::cuda::std::bit_xor>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_xor>{__op}, __order, _Operand{}, __scope); } template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_min( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_min>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_min>{__op}, __order, _Operand{}, __scope); } template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_max( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_max>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_max>{__op}, __order, _Operand{}, __scope); } template = false> _CCCL_HOST_DEVICE static void __cuda_atomic_exchange( - _Backend __backend, _Type* __ptr, _Type& __dst, _Type __op, _Order __order, _Operand, _Sco __scope) + _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { - __dst = __cuda_atomic_fetch_update( - __backend, __ptr, __cuda_atomic_op_bind<_Type, __cuda_atomic_op_store>{__op}, __order, _Operand{}, __scope); + using _ValueType = __unv<_Type>; + __dst = __cuda_atomic_fetch_update( + __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_store>{__op}, __order, _Operand{}, __scope); } _CCCL_END_NAMESPACE_CUDA_STD diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h index d3f3cd276180..f9c31d6a12de 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h @@ -22,7 +22,10 @@ #endif // no system header #include +#include +#include #include +#include #include #include #include @@ -137,16 +140,16 @@ struct __cuda_atomic_rmw_result }; template -_CCCL_HOST_DEVICE static __cuda_atomic_rmw_result<_Type> __cuda_atomic_rmw( +_CCCL_HOST_DEVICE static __cuda_atomic_rmw_result<__unv<_Pointee>> __cuda_atomic_rmw( _Backend __backend, - _Type* __ptr, + _Pointee* __ptr, const _Fn& __op, _Order __order, _InitialOrder __initial_order, @@ -154,24 +157,26 @@ _CCCL_HOST_DEVICE static __cuda_atomic_rmw_result<_Type> __cuda_atomic_rmw( _RmwOperand, _Sco __scope) { + using _Type = __unv<_Pointee>; static_assert(_Operand::__op == __cuda_atomic_operand::_b, "generic RMW requires a bitwise operand"); static_assert(_RmwOperand::__op == __cuda_atomic_operand::_b, "generic RMW requires a bitwise CAS operand"); static_assert(_Operand::__size <= _RmwOperand::__size, "generic RMW cannot use a narrower CAS operand"); - using __rmw_type = typename __cuda_atomic_rmw_type<_RmwOperand::__size>::type; - using __window = __cuda_atomic_rmw_window<_Type, __rmw_type>; + using __rmw_type = typename __cuda_atomic_rmw_type<_RmwOperand::__size>::type; + using __rmw_pointee = __copy_cv_t<_Pointee, __rmw_type>; + using __window = __cuda_atomic_rmw_window<_Type, __rmw_type>; - __rmw_type* __aligned; + __rmw_pointee* __aligned; uint8_t __offset; if constexpr (sizeof(_Type) == sizeof(__rmw_type)) { - __aligned = reinterpret_cast<__rmw_type*>(__ptr); + __aligned = reinterpret_cast<__rmw_pointee*>(__ptr); __offset = 0; } else { constexpr uintptr_t __alignmask = sizeof(__rmw_type) - 1; - __aligned = reinterpret_cast<__rmw_type*>(reinterpret_cast(__ptr) & ~__alignmask); // NOLINT + __aligned = reinterpret_cast<__rmw_pointee*>(reinterpret_cast(__ptr) & ~__alignmask); // NOLINT __offset = static_cast((reinterpret_cast(__ptr) & __alignmask) * 8); } @@ -282,7 +287,7 @@ struct __cuda_atomic_op_fetch_min { [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const { - return __op < __old ? __op : __old; + return __cuda_atomic_less(__op, __old) ? __op : __old; } }; @@ -291,14 +296,15 @@ struct __cuda_atomic_op_fetch_max { [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const { - return __old < __op ? __op : __old; + return __cuda_atomic_less(__old, __op) ? __op : __old; } }; -template -_CCCL_HOST_DEVICE_API _Type -__cuda_atomic_fetch_update(_Backend __backend, _Type* __ptr, const _Fn& __op, _Order __order, _Operand, _Sco __scope) +template +_CCCL_HOST_DEVICE_API __unv<_Pointee> +__cuda_atomic_fetch_update(_Backend __backend, _Pointee* __ptr, const _Fn& __op, _Order __order, _Operand, _Sco __scope) { + using _Type = __unv<_Pointee>; static_assert(sizeof(_Type) * 8 == _Operand::__size, "generic RMW requires matching type and operand sizes"); constexpr size_t __rmw_size = _Operand::__size < _Backend::__smallest_cas ? _Backend::__smallest_cas : _Operand::__size; @@ -311,7 +317,7 @@ __cuda_atomic_fetch_update(_Backend __backend, _Type* __ptr, const _Fn& __op, _O __ptr, __cuda_atomic_rmw_op<_Type, _Fn>{__op}, __order, - __cuda_atomic_initial_load_order<_Order>(), + __cuda_atomic_initial_load_order(__order), __bitwise_operand{}, __rmw_operand{}, __scope) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host.h b/libcudacxx/include/cuda/std/__atomic/functions/host.h index 95b9215d651f..9ccfa03c991e 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/host.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/host.h @@ -92,7 +92,7 @@ template _CCCL_HOST_API void __cuda_atomic_load( __cuda_atomic_host_backend, const _Type* __ptr, - _Type& __dst, + __unv<_Type>& __dst, memory_order __order, _Operand, __thread_scope_tag, @@ -104,7 +104,13 @@ _CCCL_HOST_API void __cuda_atomic_load( template _CCCL_HOST_API void __cuda_atomic_store( - __cuda_atomic_host_backend, _Type* __ptr, _Type& __val, memory_order __order, _Operand, __thread_scope_tag, _Mmio) + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __val, + memory_order __order, + _Operand, + __thread_scope_tag, + _Mmio) { _LIBCUDACXX_INT128_WARN(_Type) __atomic_store(&__atomic_force_align_host(__ptr)->__atom, &__val, __atomic_order_to_int(__order)); @@ -114,9 +120,9 @@ template _CCCL_HOST_API bool __cuda_atomic_compare_exchange( __cuda_atomic_host_backend, _Type* __ptr, - _Type& __dst, - _Type __cmp, - _Type __op, + __unv<_Type>& __dst, + __unv<_Type> __cmp, + __unv<_Type> __op, _Cas __cas, __cuda_atomic_runtime_cas_order __order, _Operand, @@ -138,9 +144,9 @@ template _CCCL_HOST_API bool __cuda_atomic_compare_exchange( __cuda_atomic_host_backend __backend, _Type* __ptr, - _Type& __dst, - _Type __cmp, - _Type __op, + __unv<_Type>& __dst, + __unv<_Type> __cmp, + __unv<_Type> __op, _Cas __cas, memory_order __order, _Operand __operand, @@ -160,31 +166,49 @@ _CCCL_HOST_API bool __cuda_atomic_compare_exchange( template _CCCL_HOST_API void __cuda_atomic_exchange( - __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { _LIBCUDACXX_INT128_WARN(_Type) __atomic_exchange(&__atomic_force_align_host(__ptr)->__atom, &__op, &__dst, __atomic_order_to_int(__order)); } -template < - class _Type, - class _Operand, - enable_if_t && (_Operand::__op != __cuda_atomic_operand::_f) && (_Operand::__size <= 128), - bool> = false> +template > && (_Operand::__op != __cuda_atomic_operand::_f) + && (_Operand::__size <= 64), + bool> = false> _CCCL_HOST_API void __cuda_atomic_fetch_add( - __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { _LIBCUDACXX_INT128_WARN(_Type) __dst = __atomic_fetch_add(__ptr, __op, __atomic_order_to_int(__order)); } -template < - class _Type, - class _Operand, - enable_if_t && (_Operand::__op != __cuda_atomic_operand::_f) && (_Operand::__size <= 128), - bool> = false> +template > && (_Operand::__op != __cuda_atomic_operand::_f) + && (_Operand::__size <= 64), + bool> = false> _CCCL_HOST_API void __cuda_atomic_fetch_sub( - __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { _LIBCUDACXX_INT128_WARN(_Type) __dst = __atomic_fetch_sub(__ptr, __op, __atomic_order_to_int(__order)); @@ -192,9 +216,15 @@ _CCCL_HOST_API void __cuda_atomic_fetch_sub( template = false> + enable_if_t<(_Operand::__op == __cuda_atomic_operand::_b) && (_Operand::__size <= 64), bool> = false> _CCCL_HOST_API void __cuda_atomic_fetch_and( - __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { _LIBCUDACXX_INT128_WARN(_Type) __dst = __atomic_fetch_and(__ptr, __op, __atomic_order_to_int(__order)); @@ -202,9 +232,15 @@ _CCCL_HOST_API void __cuda_atomic_fetch_and( template = false> + enable_if_t<(_Operand::__op == __cuda_atomic_operand::_b) && (_Operand::__size <= 64), bool> = false> _CCCL_HOST_API void __cuda_atomic_fetch_or( - __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { _LIBCUDACXX_INT128_WARN(_Type) __dst = __atomic_fetch_or(__ptr, __op, __atomic_order_to_int(__order)); @@ -212,9 +248,15 @@ _CCCL_HOST_API void __cuda_atomic_fetch_or( template = false> + enable_if_t<(_Operand::__op == __cuda_atomic_operand::_b) && (_Operand::__size <= 64), bool> = false> _CCCL_HOST_API void __cuda_atomic_fetch_xor( - __cuda_atomic_host_backend, _Type* __ptr, _Type& __dst, _Type __op, memory_order __order, _Operand, __thread_scope_tag) + __cuda_atomic_host_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + memory_order __order, + _Operand, + __thread_scope_tag) { _LIBCUDACXX_INT128_WARN(_Type) __dst = __atomic_fetch_xor(__ptr, __op, __atomic_order_to_int(__order)); diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h index aab661378b2b..9057365125d3 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/host_backend.h @@ -21,7 +21,6 @@ # pragma system_header #endif // no system header -#include #include #include #include @@ -32,25 +31,6 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD struct __cuda_atomic_host_backend { - template - using __enable_if_direct_bitwise = enable_if_t<(sizeof(_Type) < 16), bool>; - - template - using __enable_if_direct_arithmetic = enable_if_t && (sizeof(_Type) < 16), bool>; - - template - using __enable_if_direct_minmax = enable_if_t && (sizeof(_Type) < 16), bool>; - - template - using __enable_if_fallback_bitwise = enable_if_t<(sizeof(_Type) == 16), bool>; - - template - using __enable_if_fallback_arithmetic = enable_if_t && (sizeof(_Type) == 16), bool>; - - template - using __enable_if_fallback_minmax = - enable_if_t || (is_scalar_v<_Type> && sizeof(_Type) == 16), bool>; - static constexpr bool __needs_constant_order = false; static constexpr bool __requires_local_memory_workaround = false; static constexpr size_t __smallest_cas = 8; diff --git a/libcudacxx/include/cuda/std/__atomic/types/common.h b/libcudacxx/include/cuda/std/__atomic/types/common.h index 6626a8adcb72..6d1fb157a9a5 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/common.h +++ b/libcudacxx/include/cuda/std/__atomic/types/common.h @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include #include @@ -31,6 +33,37 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD +template +_CCCL_HOST_DEVICE_API bool __cuda_atomic_less(_Tp __lhs, _Tp __rhs) +{ + if constexpr (__is_extended_floating_point_v<_Tp> && sizeof(_Tp) == 2) + { +#if _CCCL_HAS_CTK() && _CCCL_CTK_BELOW(12, 2) + // Before CTK 12.2, __hlt is device-only and its bfloat16 overload is unavailable before SM80. +# if _CCCL_HAS_NVBF16() + if constexpr (is_same_v<_Tp, __nv_bfloat16>) + { + // Intentionally unqualified to avoid including . + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_80, (return __hlt(__lhs, __rhs);), (return __bfloat162float(__lhs) < __bfloat162float(__rhs);)) + } + else +# endif // _CCCL_HAS_NVBF16() + { + // Intentionally unqualified to avoid including . + NV_IF_ELSE_TARGET(NV_IS_DEVICE, (return __hlt(__lhs, __rhs);), (return __half2float(__lhs) < __half2float(__rhs);)) + } +#else // ^^^ CTK below 12.2 ^^^ / vvv CTK 12.2 or newer vvv + // Intentionally unqualified to avoid including and . + return __hlt(__lhs, __rhs); +#endif // CTK 12.2 or newer + } + else + { + return __lhs < __rhs; + } +} + enum class __atomic_tag { __atomic_base_tag, diff --git a/libcudacxx/include/cuda/std/__atomic/types/small.h b/libcudacxx/include/cuda/std/__atomic/types/small.h index bd8cf9fa7ce4..7d6df4e56a4c 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/small.h +++ b/libcudacxx/include/cuda/std/__atomic/types/small.h @@ -28,7 +28,6 @@ #include #include #include -#include #include #include @@ -78,30 +77,6 @@ _CCCL_HOST_DEVICE_API _Tp __atomic_small_from_32(__atomic_small_proxy_t<_Tp> __v _CCCL_DIAG_POP -template -_CCCL_HOST_DEVICE_API bool __atomic_small_extended_floating_point_less(_Tp __lhs, _Tp __rhs) -{ -#if _CCCL_HAS_CTK() && _CCCL_CTK_BELOW(12, 2) - // Before CTK 12.2, __hlt is device-only and its bfloat16 overload is unavailable before SM80. -# if _CCCL_HAS_NVBF16() - if constexpr (is_same_v<_Tp, __nv_bfloat16>) - { - // Intentionally unqualified to avoid including . - NV_IF_ELSE_TARGET( - NV_PROVIDES_SM_80, (return __hlt(__lhs, __rhs);), (return __bfloat162float(__lhs) < __bfloat162float(__rhs);)) - } - else -# endif // _CCCL_HAS_NVBF16() - { - // Intentionally unqualified to avoid including . - NV_IF_ELSE_TARGET(NV_IS_DEVICE, (return __hlt(__lhs, __rhs);), (return __half2float(__lhs) < __half2float(__rhs);)) - } -#else // ^^^ CTK below 12.2 ^^^ / vvv CTK 12.2 or newer vvv - // Intentionally unqualified to avoid including and . - return __hlt(__lhs, __rhs); -#endif // CTK 12.2 or newer -} - template struct __atomic_small_storage { @@ -250,7 +225,7 @@ _CCCL_HOST_DEVICE_API auto __atomic_fetch_max_dispatch(_Sto* __a, _Up __val, mem while (true) { const auto __old = __atomic_small_from_32<_Tp>(__expected); - const auto __desired = __atomic_small_extended_floating_point_less(__old, _Tp(__val)) ? _Tp(__val) : __old; + const auto __desired = __cuda_atomic_less(__old, _Tp(__val)) ? _Tp(__val) : __old; if (__atomic_compare_exchange_strong_dispatch( &__a->__a_value, &__expected, __atomic_small_to_32(__desired), __order, __order, _Sco{})) { @@ -278,7 +253,7 @@ _CCCL_HOST_DEVICE_API auto __atomic_fetch_min_dispatch(_Sto* __a, _Up __val, mem while (true) { const auto __old = __atomic_small_from_32<_Tp>(__expected); - const auto __desired = __atomic_small_extended_floating_point_less(_Tp(__val), __old) ? _Tp(__val) : __old; + const auto __desired = __cuda_atomic_less(_Tp(__val), __old) ? _Tp(__val) : __old; if (__atomic_compare_exchange_strong_dispatch( &__a->__a_value, &__expected, __atomic_small_to_32(__desired), __order, __order, _Sco{})) { From bf76f303b12a9de1f6ffc52ba1dab8b1506f1010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Thu, 27 Aug 2026 14:43:20 -0700 Subject: [PATCH 06/24] Tighten sequentially consistent CAS-loop checks. --- .../test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu | 4 ++-- .../sass/bitwise_volatile_types_8_16_atomic_ref.cu | 4 ++-- .../sass/compare_exchange_types_8_16_atomic_ref.cu | 4 ++-- .../sass/compare_exchange_volatile_types_8_16_atomic_ref.cu | 4 ++-- .../atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu | 4 ++-- .../sass/exchange_volatile_types_8_16_atomic_ref.cu | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu index f649b373a4ec..1669524907a4 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu @@ -35,14 +35,14 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; SMXX: {{.*}}LOP3.LUT {{.*}} ; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu index ef4d3c9db3f0..a605c319d7fb 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_volatile_types_8_16_atomic_ref.cu @@ -35,8 +35,6 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}}{{.*\[}}[[ALIGNED_ADDR]]{{\].*}} diff --git a/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu index ca8f36a06998..3c26e9f2307c 100644 --- a/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/compare_exchange_volatile_types_8_16_atomic_ref.cu @@ -38,8 +38,6 @@ atomic_codegen_test(cuda::atomic_ref& atom, TYPE& expected ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-NOT: {{.*}}LD.E{{.*}}.STRONG{{.*}} @@ -47,6 +45,8 @@ atomic_codegen_test(cuda::atomic_ref& atom, TYPE& expected ; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; SMXX-NOT: {{.*}}LD.E{{.*}}.STRONG{{.*}} ; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}LD.E{{.*}}.STRONG{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu index 2d13f8cdb053..eb09aeeacc06 100644 --- a/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/exchange_types_8_16_atomic_ref.cu @@ -34,13 +34,13 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[A:R[0-9]+]], [[ATOM_ADDR]], 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[E:R[0-9]+]], {{.*\[}}[[A]]{{(\.64)?\].*}} ; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[E:R[0-9]+]], {{.*\[}}[[A]]{{(\.64)?\].*}} ; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[C:R[0-9]+]], {{\[}}[[A]]{{\]}}, [[E]], [[C]]{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu index 8685bcfa9626..7cc36bf3375b 100644 --- a/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/exchange_volatile_types_8_16_atomic_ref.cu @@ -35,8 +35,6 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref Date: Thu, 27 Aug 2026 15:01:45 -0700 Subject: [PATCH 07/24] Reduce atomic binder instantiations. --- .../codegen/generators/compare_and_swap.h | 10 +- libcudacxx/codegen/generators/exchange.h | 10 +- libcudacxx/codegen/generators/fetch_ops.h | 10 +- libcudacxx/codegen/generators/ld_st.h | 22 ++-- .../cuda/std/__atomic/functions/cuda_ptx.h | 21 ++-- .../std/__atomic/functions/cuda_ptx_backend.h | 7 +- .../__atomic/functions/cuda_ptx_generated.h | 102 +++++++++--------- .../cuda/std/__atomic/functions/dispatch.h | 95 ++++++++-------- 8 files changed, 141 insertions(+), 136 deletions(-) diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index f9b30d82d1b2..9cd7403a4db2 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -128,7 +128,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( << R"XXX( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_compare_exchange { _Backend __backend; _Type* __ptr; @@ -136,8 +136,8 @@ struct __cuda_atomic_bind_compare_exchange { __unv<_Type> __cmp; __unv<_Type> __des; - template - [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { + template + [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) { return __cuda_atomic_compare_exchange( __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); } @@ -165,10 +165,10 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( - __backend, __bound_compare_swap, __success, __failure, __scope); + __backend, __bound_compare_swap, __success, __failure, __scope, _Cas{}, __proxy_tag{}); } #if _CCCL_CUDA_COMPILATION() diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index ac5f92435dd8..5335e4bf7dd5 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -128,15 +128,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( << R"XXX( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_exchange { _Backend __backend; _Type* __ptr; __unv<_Type>* __old; __unv<_Type> __new; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); } }; @@ -160,9 +160,9 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( { if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } - __cuda_atomic_bind_exchange<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_swap{ + __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ __backend, __ptr_proxy, __old_proxy, *__new_proxy}; - __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); } template diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index 412094cd9cc6..d207deb8380f 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -70,15 +70,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_{0}( constexpr auto fetch_bind_invoke = R"XXX( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_fetch_{0} {{ _Backend __backend; _Type* __ptr; __unv<_Type>* __dst; __unv<_Type> __op; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) {{ + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) {{ __cuda_atomic_fetch_{0}(__backend, __ptr, *__dst, __op, __order, _Tag{{}}, _Sco{{}}); }} }}; @@ -101,9 +101,9 @@ template __bound_{0}{{ + __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_pointee> __bound_{0}{{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope, __proxy_tag{{}}); return __dst; }} diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index 60704eacfba2..71aff58f8776 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -190,14 +190,14 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( << R"XXX( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_load { _Backend __backend; const _Type* __ptr; __unv<_Type>* __dst; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { __cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); } }; @@ -220,9 +220,10 @@ __cuda_atomic_load_dispatch( { if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_load<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); + __cuda_atomic_load_order_dispatch( + __backend, __bound_load, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); } template @@ -388,14 +389,14 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( << R"XXX( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_store { _Backend __backend; _Type* __ptr; __unv<_Type> __val; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { __cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; @@ -414,9 +415,10 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory { if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_store<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ __backend, __ptr_proxy, *__val_proxy}; - __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); + __cuda_atomic_store_order_dispatch( + __backend, __bound_store, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); } #if _CCCL_CUDA_COMPILATION() diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h index 6a2b7bf882a4..dd2c4570f595 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -35,10 +35,9 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD -template -_CCCL_DEVICE_API auto -__cuda_atomic_ptx_backend::__with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope) - -> decltype(__fn(__order)) +template +_CCCL_DEVICE_API auto __cuda_atomic_ptx_backend::__with_transformed_order( + _Operation, _Fn& __fn, _Order __order, _Sco __scope, _Args... __args) -> decltype(__fn(__order, __args..., __scope)) { constexpr bool __is_load = is_same_v<_Operation, __cuda_atomic_operation_load>; constexpr bool __is_store = is_same_v<_Operation, __cuda_atomic_operation_store>; @@ -60,16 +59,16 @@ __cuda_atomic_ptx_backend::__with_transformed_order(_Operation, _Fn& __fn, _Orde { if constexpr (__is_store) { - return __fn(__cuda_atomic_ptx_order_relaxed{true}); + return __fn(__cuda_atomic_ptx_order_relaxed{true}, __args..., __scope); } else { - return __fn(__cuda_atomic_ptx_order_acquire{true}); + return __fn(__cuda_atomic_ptx_order_acquire{true}, __args..., __scope); } } else { - return __fn(__transform_order(__order)); + return __fn(__transform_order(__order), __args..., __scope); } }), NV_IS_DEVICE, @@ -80,22 +79,22 @@ __cuda_atomic_ptx_backend::__with_transformed_order(_Operation, _Fn& __fn, _Orde } if constexpr (__membar_after) { - if constexpr (is_void_v) + if constexpr (is_void_v) { - __fn(__cuda_atomic_order_volatile{}); + __fn(__cuda_atomic_order_volatile{}, __args..., __scope); __cuda_atomic_membar(__scope); return; } else { - auto __result = __fn(__cuda_atomic_order_volatile{}); + auto __result = __fn(__cuda_atomic_order_volatile{}, __args..., __scope); __cuda_atomic_membar(__scope); return __result; } } else { - return __fn(__cuda_atomic_order_volatile{}); + return __fn(__cuda_atomic_order_volatile{}, __args..., __scope); } })) } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h index 24de4ae2c91f..5112cb8f1bbb 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h @@ -88,9 +88,10 @@ struct __cuda_atomic_ptx_backend } } - template - _CCCL_DEVICE_API static auto __with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope) - -> decltype(__fn(__order)); + template + _CCCL_DEVICE_API static auto + __with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope, _Args... __args) + -> decltype(__fn(__order, __args..., __scope)); static constexpr bool __needs_constant_order = true; static constexpr bool __requires_local_memory_workaround = true; diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index d4bc435a58f9..79938916f87b 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -1277,14 +1277,14 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_load { _Backend __backend; const _Type* __ptr; __unv<_Type>* __dst; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { __cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); } }; @@ -1307,9 +1307,10 @@ __cuda_atomic_load_dispatch( { if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_load<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_load{ + __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch(__backend, __bound_load, __order, __scope); + __cuda_atomic_load_order_dispatch( + __backend, __bound_load, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); } template @@ -1820,14 +1821,14 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_store { _Backend __backend; _Type* __ptr; __unv<_Type> __val; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { __cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; @@ -1846,9 +1847,10 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory { if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } - __cuda_atomic_bind_store<_Backend, __proxy_pointee, __proxy_tag, _Sco, __cuda_atomic_mmio_disable> __bound_store{ + __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ __backend, __ptr_proxy, *__val_proxy}; - __cuda_atomic_store_order_dispatch(__backend, __bound_store, __order, __scope); + __cuda_atomic_store_order_dispatch( + __backend, __bound_store, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); } #if _CCCL_CUDA_COMPILATION() @@ -2416,7 +2418,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_compare_exchange { _Backend __backend; _Type* __ptr; @@ -2424,8 +2426,8 @@ struct __cuda_atomic_bind_compare_exchange { __unv<_Type> __cmp; __unv<_Type> __des; - template - [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order) { + template + [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) { return __cuda_atomic_compare_exchange( __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); } @@ -2453,10 +2455,10 @@ template { if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee, __proxy_tag, _Cas, _Sco> __bound_compare_swap{ + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( - __backend, __bound_compare_swap, __success, __failure, __scope); + __backend, __bound_compare_swap, __success, __failure, __scope, _Cas{}, __proxy_tag{}); } #if _CCCL_CUDA_COMPILATION() @@ -3024,15 +3026,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_exchange { _Backend __backend; _Type* __ptr; __unv<_Type>* __old; __unv<_Type> __new; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); } }; @@ -3056,9 +3058,9 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( { if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } - __cuda_atomic_bind_exchange<_Backend, __proxy_pointee, __proxy_tag, _Sco> __bound_swap{ + __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ __backend, __ptr_proxy, __old_proxy, *__new_proxy}; - __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope); + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); } template @@ -3476,15 +3478,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_fetch_add { _Backend __backend; _Type* __ptr; __unv<_Type>* __dst; __unv<_Type> __op; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_fetch_add(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; @@ -3508,9 +3510,9 @@ template __bound_add{ + __cuda_atomic_bind_fetch_add<_Backend, __proxy_pointee> __bound_add{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope, __proxy_tag{}); return __dst; } @@ -3680,15 +3682,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_fetch_and { _Backend __backend; _Type* __ptr; __unv<_Type>* __dst; __unv<_Type> __op; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_fetch_and(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; @@ -3711,9 +3713,9 @@ template __bound_and{ + __cuda_atomic_bind_fetch_and<_Backend, __proxy_pointee> __bound_and{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope, __proxy_tag{}); return __dst; } @@ -4043,15 +4045,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_fetch_max { _Backend __backend; _Type* __ptr; __unv<_Type>* __dst; __unv<_Type> __op; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_fetch_max(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; @@ -4074,9 +4076,9 @@ template __bound_max{ + __cuda_atomic_bind_fetch_max<_Backend, __proxy_pointee> __bound_max{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope, __proxy_tag{}); return __dst; } @@ -4406,15 +4408,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_fetch_min { _Backend __backend; _Type* __ptr; __unv<_Type>* __dst; __unv<_Type> __op; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_fetch_min(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; @@ -4437,9 +4439,9 @@ template __bound_min{ + __cuda_atomic_bind_fetch_min<_Backend, __proxy_pointee> __bound_min{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope, __proxy_tag{}); return __dst; } @@ -4609,15 +4611,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_fetch_or { _Backend __backend; _Type* __ptr; __unv<_Type>* __dst; __unv<_Type> __op; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_fetch_or(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; @@ -4640,9 +4642,9 @@ template __bound_or{ + __cuda_atomic_bind_fetch_or<_Backend, __proxy_pointee> __bound_or{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope, __proxy_tag{}); return __dst; } @@ -4812,15 +4814,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( #endif // _CCCL_CUDA_COMPILATION() -template +template struct __cuda_atomic_bind_fetch_xor { _Backend __backend; _Type* __ptr; __unv<_Type>* __dst; __unv<_Type> __op; - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order) { + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { __cuda_atomic_fetch_xor(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; @@ -4843,9 +4845,9 @@ template __bound_xor{ + __cuda_atomic_bind_fetch_xor<_Backend, __proxy_pointee> __bound_xor{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope, __proxy_tag{}); return __dst; } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index afc13633ca11..197eebfe26be 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -34,13 +34,13 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD -template +template _CCCL_HOST_DEVICE_API void __cuda_atomic_load_order_dispatch( - [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope) + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope, _Args... __args) { if constexpr (!_Backend::__needs_constant_order) { - __fn(__order); + __fn(__order, __args..., __scope); } else { @@ -48,28 +48,28 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_load_order_dispatch( { case __ATOMIC_RELAXED: return __backend.__with_transformed_order( - __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_relaxed{}, __scope); + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_relaxed{}, __scope, __args...); case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: return __backend.__with_transformed_order( - __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_acquire{}, __scope); + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_acquire{}, __scope, __args...); case __ATOMIC_SEQ_CST: return __backend.__with_transformed_order( - __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_seq_cst{}, __scope); + __cuda_atomic_operation_load{}, __fn, __cuda_atomic_order_seq_cst{}, __scope, __args...); default: _CCCL_ASSERT(false, "invalid load memory order"); } } } -template +template _CCCL_HOST_DEVICE_API void __cuda_atomic_store_order_dispatch( - [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope) + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope, _Args... __args) { if constexpr (!_Backend::__needs_constant_order) { - __fn(__order); + __fn(__order, __args..., __scope); } else { @@ -77,26 +77,26 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_store_order_dispatch( { case __ATOMIC_RELAXED: return __backend.__with_transformed_order( - __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_relaxed{}, __scope); + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_relaxed{}, __scope, __args...); case __ATOMIC_RELEASE: return __backend.__with_transformed_order( - __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_release{}, __scope); + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_release{}, __scope, __args...); case __ATOMIC_SEQ_CST: return __backend.__with_transformed_order( - __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_seq_cst{}, __scope); + __cuda_atomic_operation_store{}, __fn, __cuda_atomic_order_seq_cst{}, __scope, __args...); default: _CCCL_ASSERT(false, "invalid store memory order"); } } } -template +template _CCCL_HOST_DEVICE_API void __cuda_atomic_rmw_order_dispatch( - [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope) + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope, _Args... __args) { if constexpr (!_Backend::__needs_constant_order) { - __fn(__order); + __fn(__order, __args..., __scope); } else { @@ -104,44 +104,44 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_rmw_order_dispatch( { case __ATOMIC_RELAXED: return __backend.__with_transformed_order( - __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_relaxed{}, __scope); + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_relaxed{}, __scope, __args...); case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: return __backend.__with_transformed_order( - __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acquire{}, __scope); + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acquire{}, __scope, __args...); case __ATOMIC_RELEASE: return __backend.__with_transformed_order( - __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_release{}, __scope); + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_release{}, __scope, __args...); case __ATOMIC_ACQ_REL: return __backend.__with_transformed_order( - __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acq_rel{}, __scope); + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_acq_rel{}, __scope, __args...); case __ATOMIC_SEQ_CST: return __backend.__with_transformed_order( - __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_seq_cst{}, __scope); + __cuda_atomic_operation_rmw{}, __fn, __cuda_atomic_order_seq_cst{}, __scope, __args...); default: _CCCL_ASSERT(false, "invalid read-modify-write memory order"); } } } -template -_CCCL_HOST_DEVICE_API void -__cuda_atomic_exchange_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_order_dispatch( + _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) { - __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope); + __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); } -template +template _CCCL_HOST_DEVICE_API void -__cuda_atomic_fetch_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope) +__cuda_atomic_fetch_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) { - __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope); + __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); } -template -[[nodiscard]] _CCCL_HOST_DEVICE_API bool -__cuda_atomic_compare_exchange_failure_order_dispatch(_Backend __backend, _Fn& __fn, int __failure, _Sco __scope) +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_failure_order_dispatch( + _Backend __backend, _Fn& __fn, int __failure, _Sco __scope, _Args... __args) { switch (__failure) { @@ -150,7 +150,8 @@ __cuda_atomic_compare_exchange_failure_order_dispatch(_Backend __backend, _Fn& _ __cuda_atomic_operation_rmw{}, __fn, __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_relaxed>{}), - __scope); + __scope, + __args...); case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: @@ -158,30 +159,33 @@ __cuda_atomic_compare_exchange_failure_order_dispatch(_Backend __backend, _Fn& _ __cuda_atomic_operation_rmw{}, __fn, __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_acquire>{}), - __scope); + __scope, + __args...); case __ATOMIC_SEQ_CST: return __backend.__with_transformed_order( __cuda_atomic_operation_rmw{}, __fn, __backend.__collapse_cas_order(__cuda_atomic_cas_order<_Success, __cuda_atomic_order_seq_cst>{}), - __scope); + __scope, + __args...); default: _CCCL_ASSERT(false, "invalid compare-exchange failure memory order"); _CCCL_UNREACHABLE(); } } -template +template [[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch( [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __success, memory_order __failure, - [[maybe_unused]] _Sco __scope) + [[maybe_unused]] _Sco __scope, + _Args... __args) { if constexpr (!_Backend::__needs_constant_order) { - return __fn(__cuda_atomic_runtime_cas_order{__success, __failure}); + return __fn(__cuda_atomic_runtime_cas_order{__success, __failure}, __args..., __scope); } else { @@ -190,21 +194,21 @@ template { case __ATOMIC_RELAXED: return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_relaxed>( - __backend, __fn, __failure_order, __scope); + __backend, __fn, __failure_order, __scope, __args...); case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_acquire>( - __backend, __fn, __failure_order, __scope); + __backend, __fn, __failure_order, __scope, __args...); case __ATOMIC_RELEASE: return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_release>( - __backend, __fn, __failure_order, __scope); + __backend, __fn, __failure_order, __scope, __args...); case __ATOMIC_ACQ_REL: return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_acq_rel>( - __backend, __fn, __failure_order, __scope); + __backend, __fn, __failure_order, __scope, __args...); case __ATOMIC_SEQ_CST: return __cuda_atomic_compare_exchange_failure_order_dispatch<__cuda_atomic_order_seq_cst>( - __backend, __fn, __failure_order, __scope); + __backend, __fn, __failure_order, __scope, __args...); default: _CCCL_ASSERT(false, "invalid compare-exchange success memory order"); _CCCL_UNREACHABLE(); @@ -223,7 +227,7 @@ struct __cuda_atomic_bind_fetch_sub template _CCCL_HOST_DEVICE_API void operator()(_Order __order, _Operand, _Sco) { - ::cuda::std::__cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); + __cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); } }; @@ -241,19 +245,16 @@ __cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, mem auto* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); auto* __dst_proxy = reinterpret_cast<__proxy_type*>(&__dst); auto* __op_proxy = reinterpret_cast<__proxy_type*>(&__op); -#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { - if (::cuda::std::__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + if (__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) { return __dst; } } -#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_sub<_Backend, __proxy_pointee> __bound_fetch_sub{ __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - ::cuda::std::__cuda_atomic_fetch_order_dispatch( - __backend, __bound_fetch_sub, __order, __scope, __proxy_operand{}); + __cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_sub, __order, __scope, __proxy_operand{}); return __dst; } From 117a264672cb6daa04ead5a55827a5819f1845d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Thu, 27 Aug 2026 15:13:00 -0700 Subject: [PATCH 08/24] Avoid ADL in atomic backend dispatch. --- .../codegen/generators/compare_and_swap.h | 8 +- libcudacxx/codegen/generators/exchange.h | 11 +- libcudacxx/codegen/generators/fence.h | 8 +- libcudacxx/codegen/generators/fetch_ops.h | 6 +- libcudacxx/codegen/generators/header.h | 1 + libcudacxx/codegen/generators/ld_st.h | 22 +- .../cuda/std/__atomic/functions/backend.h | 2 +- .../cuda/std/__atomic/functions/cuda_local.h | 35 +- .../cuda/std/__atomic/functions/cuda_ptx.h | 9 +- .../__atomic/functions/cuda_ptx_generated.h | 1574 +++++++++-------- .../cuda/std/__atomic/functions/dispatch.h | 10 +- .../cuda/std/__atomic/functions/generic.h | 20 +- .../cuda/std/__atomic/functions/generic_rmw.h | 8 +- .../cuda/std/__atomic/functions/host.h | 6 +- 14 files changed, 866 insertions(+), 854 deletions(-) diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index 9cd7403a4db2..9929a19e0d2f 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -31,7 +31,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ - __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -51,7 +51,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) -{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; constexpr Operand supported_types[] = { Operand::Bit, @@ -138,7 +138,7 @@ struct __cuda_atomic_bind_compare_exchange { template [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) { - return __cuda_atomic_compare_exchange( + return ::cuda::std::__cuda_atomic_compare_exchange( __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); } }; @@ -163,7 +163,7 @@ template bool __res = false; if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} + if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index 5335e4bf7dd5..1e3932a7619c 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -31,7 +31,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ - __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -51,7 +51,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) -{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; constexpr Operand supported_types[] = { Operand::Bit, @@ -137,7 +137,7 @@ struct __cuda_atomic_bind_exchange { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); } }; template @@ -158,7 +158,7 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); if constexpr (_Backend::__requires_local_memory_workaround) { - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} + if(::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ __backend, __ptr_proxy, __old_proxy, *__new_proxy}; @@ -171,7 +171,8 @@ template { using __value_type = __unv<_Type>; __value_type __old; - __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); + ::cuda::std::__cuda_atomic_exchange_dispatch( + __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); return __old; } diff --git a/libcudacxx/codegen/generators/fence.h b/libcudacxx/codegen/generators/fence.h index 117b810c77f9..3072656e5155 100644 --- a/libcudacxx/codegen/generators/fence.h +++ b/libcudacxx/codegen/generators/fence.h @@ -83,7 +83,7 @@ __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco _ { if (__order.__was_seq_cst) { - __cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); + ::cuda::std::__cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); } } @@ -98,11 +98,11 @@ static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; + case __ATOMIC_SEQ_CST: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } @@ -113,7 +113,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_membar(_Sco{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index d207deb8380f..15ee6db1c810 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -63,7 +63,7 @@ inline void FormatFetchOps(std::ostream& out) template static inline _CCCL_DEVICE void __cuda_atomic_fetch_{0}( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, {5} __order, __cuda_atomic_operand_{1}{2}, {7}) -{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {7}{{}}); asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {7}{{}}); asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; // 0 - Atomic Operation // 1 - Operand type constraint // 2 - Operand transform @@ -79,7 +79,7 @@ struct __cuda_atomic_bind_fetch_{0} {{ template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) {{ - __cuda_atomic_fetch_{0}(__backend, __ptr, *__dst, __op, __order, _Tag{{}}, _Sco{{}}); + ::cuda::std::__cuda_atomic_fetch_{0}(__backend, __ptr, *__dst, __op, __order, _Tag{{}}, _Sco{{}}); }} }}; template (&__op); if constexpr (_Backend::__requires_local_memory_workaround) {{ - if (__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} + if (::cuda::std::__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} }} __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_pointee> __bound_{0}{{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}}; diff --git a/libcudacxx/codegen/generators/header.h b/libcudacxx/codegen/generators/header.h index c9710d384d16..672f278bb95f 100644 --- a/libcudacxx/codegen/generators/header.h +++ b/libcudacxx/codegen/generators/header.h @@ -54,6 +54,7 @@ inline void FormatHeader(std::ostream& out) #include #include #include +#include #include #include diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index 71aff58f8776..d5d55d82a7a3 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -53,7 +53,7 @@ inline void FormatLoad(std::ostream& out) static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ - __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -71,13 +71,13 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) -{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ - __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); uint16_t __tmp; asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -198,7 +198,7 @@ struct __cuda_atomic_bind_load { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - __cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); + ::cuda::std::__cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); } }; template @@ -218,7 +218,7 @@ __cuda_atomic_load_dispatch( __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} + if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; @@ -231,7 +231,7 @@ template __cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) { __unv<_Type> __dst; - __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + ::cuda::std::__cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); return __dst; } @@ -256,7 +256,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ - __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -274,13 +274,13 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) -{{ __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; +{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ - __cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); const uint16_t __tmp = static_cast(__val); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__tmp) : "memory"); }})XXX"; @@ -397,7 +397,7 @@ struct __cuda_atomic_bind_store { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - __cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); + ::cuda::std::__cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; template @@ -413,7 +413,7 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} + if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ __backend, __ptr_proxy, *__val_proxy}; diff --git a/libcudacxx/include/cuda/std/__atomic/functions/backend.h b/libcudacxx/include/cuda/std/__atomic/functions/backend.h index 398a617db39a..514c3a068acb 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/backend.h @@ -159,7 +159,7 @@ template } else { - return __cuda_atomic_initial_load_order(__order); + return ::cuda::std::__cuda_atomic_initial_load_order(__order); } } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h index bcbb8843d176..f69501afbedc 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h @@ -80,19 +80,19 @@ _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_sub(_Type __atom, _Type con template _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_max(_Type __atom, _Type const& __v) { - return __cuda_atomic_less(__atom, __v) ? __v : __atom; + return ::cuda::std::__cuda_atomic_less(__atom, __v) ? __v : __atom; } template _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_min(_Type __atom, _Type const& __v) { - return __cuda_atomic_less(__v, __atom) ? __v : __atom; + return ::cuda::std::__cuda_atomic_less(__v, __atom) ? __v : __atom; } template _CCCL_DEVICE_API bool __cuda_atomic_load_weak_if_local(const _Type* __ptr, __unv<_Type>* __ret, [[maybe_unused]] size_t __size) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } @@ -106,7 +106,7 @@ template _CCCL_DEVICE_API bool __cuda_atomic_store_weak_if_local(_Type* __ptr, const __unv<_Type>* __val, [[maybe_unused]] size_t __size) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } @@ -118,7 +118,7 @@ template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_weak_if_local( _Type* __ptr, __unv<_Type>* __expected, const __unv<_Type>* __desired, bool* __success) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } @@ -142,7 +142,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_weak_if_local( template _CCCL_DEVICE_API bool __cuda_atomic_exchange_weak_if_local(_Type* __ptr, __unv<_Type>* __val, __unv<_Type>* __ret) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } @@ -156,7 +156,7 @@ template _CCCL_DEVICE_API bool __cuda_atomic_fetch_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret, _BOp&& __bop) { - if (!__cuda_atomic_is_local(__ptr)) + if (!::cuda::std::__cuda_atomic_is_local(__ptr)) { return false; } @@ -174,49 +174,56 @@ template _CCCL_DEVICE_API bool __cuda_atomic_fetch_and_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { using _ValueType = __unv<_Type>; - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_and<_ValueType>); + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_and<_ValueType>); } template _CCCL_DEVICE_API bool __cuda_atomic_fetch_or_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { using _ValueType = __unv<_Type>; - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_or<_ValueType>); + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_or<_ValueType>); } template _CCCL_DEVICE_API bool __cuda_atomic_fetch_xor_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { using _ValueType = __unv<_Type>; - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_xor<_ValueType>); + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_xor<_ValueType>); } template _CCCL_DEVICE_API bool __cuda_atomic_fetch_add_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { using _ValueType = __unv<_Type>; - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_add<_ValueType>); + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_add<_ValueType>); } template _CCCL_DEVICE_API bool __cuda_atomic_fetch_sub_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { using _ValueType = __unv<_Type>; - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_sub<_ValueType>); + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_sub<_ValueType>); } template _CCCL_DEVICE_API bool __cuda_atomic_fetch_max_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { using _ValueType = __unv<_Type>; - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_max<_ValueType>); + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_max<_ValueType>); } template _CCCL_DEVICE_API bool __cuda_atomic_fetch_min_weak_if_local(_Type* __ptr, __unv<_Type> __val, __unv<_Type>* __ret) { using _ValueType = __unv<_Type>; - return __cuda_atomic_fetch_weak_if_local(__ptr, __val, __ret, __cuda_atomic_fetch_local_bop_min<_ValueType>); + return ::cuda::std::__cuda_atomic_fetch_weak_if_local( + __ptr, __val, __ret, ::cuda::std::__cuda_atomic_fetch_local_bop_min<_ValueType>); } #endif // _CCCL_CUDA_COMPILATION() diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h index dd2c4570f595..325db4f4d5c4 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -75,20 +75,20 @@ _CCCL_DEVICE_API auto __cuda_atomic_ptx_backend::__with_transformed_order( ({ if constexpr (__membar_before) { - __cuda_atomic_membar(__scope); + ::cuda::std::__cuda_atomic_membar(__scope); } if constexpr (__membar_after) { if constexpr (is_void_v) { __fn(__cuda_atomic_order_volatile{}, __args..., __scope); - __cuda_atomic_membar(__scope); + ::cuda::std::__cuda_atomic_membar(__scope); return; } else { auto __result = __fn(__cuda_atomic_order_volatile{}, __args..., __scope); - __cuda_atomic_membar(__scope); + ::cuda::std::__cuda_atomic_membar(__scope); return __result; } } @@ -128,7 +128,8 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_sub( _Operand, _Sco __scope) { - __cuda_atomic_fetch_add(__backend, __ptr, __dst, __cuda_atomic_ptx_negate(__op), __order, _Operand{}, __scope); + ::cuda::std::__cuda_atomic_fetch_add( + __backend, __ptr, __dst, ::cuda::std::__cuda_atomic_ptx_negate(__op), __order, _Operand{}, __scope); } _CCCL_END_NAMESPACE_CUDA_STD diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 79938916f87b..6eadb4711b01 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -79,7 +80,7 @@ __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco _ { if (__order.__was_seq_cst) { - __cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); + ::cuda::std::__cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{}); } } @@ -94,11 +95,11 @@ static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, ( switch (__memorder) { - case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; + case __ATOMIC_SEQ_CST: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break; case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } @@ -109,7 +110,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( case __ATOMIC_CONSUME: [[fallthrough]]; case __ATOMIC_ACQUIRE: [[fallthrough]]; case __ATOMIC_ACQ_REL: [[fallthrough]]; - case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); break; + case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_membar(_Sco{}); break; case __ATOMIC_RELAXED: break; default: _CCCL_ASSERT(false, "invalid memory order"); } @@ -121,7 +122,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -130,7 +131,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -139,7 +140,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -148,7 +149,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -157,7 +158,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -166,7 +167,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -175,7 +176,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -184,7 +185,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -193,7 +194,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -202,7 +203,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -211,7 +212,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -220,7 +221,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -229,7 +230,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.b8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -238,7 +239,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -247,7 +248,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -256,7 +257,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -265,7 +266,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -274,7 +275,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -283,7 +284,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -292,7 +293,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -301,7 +302,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -310,7 +311,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -319,7 +320,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -328,7 +329,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -337,7 +338,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -346,7 +347,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.u8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -355,7 +356,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -364,7 +365,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.acquire.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -373,7 +374,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.acquire.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -382,7 +383,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.acquire.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -391,7 +392,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cta.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -400,7 +401,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.cluster.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -409,7 +410,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.gpu.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -418,7 +419,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -427,7 +428,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.mmio.relaxed.sys.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -436,7 +437,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -445,7 +446,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -454,7 +455,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -463,7 +464,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); uint16_t __tmp; asm volatile("ld.volatile.s8 %0,[%1];" : "=h"(__tmp) : "l"(__ptr) : "memory"); __dst = static_cast<__unv<_Type>>(__tmp); @@ -471,580 +472,580 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1062,7 +1063,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1080,7 +1081,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1098,7 +1099,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1116,7 +1117,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1134,7 +1135,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1152,7 +1153,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1170,7 +1171,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1188,7 +1189,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1206,7 +1207,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1224,7 +1225,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1242,7 +1243,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1260,7 +1261,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( static inline _CCCL_DEVICE void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1285,7 +1286,7 @@ struct __cuda_atomic_bind_load { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - __cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); + ::cuda::std::__cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); } }; template @@ -1305,7 +1306,7 @@ __cuda_atomic_load_dispatch( __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} + if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; @@ -1318,7 +1319,7 @@ template __cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) { __unv<_Type> __dst; - __cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + ::cuda::std::__cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); return __dst; } @@ -1328,7 +1329,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1336,7 +1337,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1344,7 +1345,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1352,7 +1353,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.release.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1360,7 +1361,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1368,7 +1369,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1376,7 +1377,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1384,7 +1385,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1392,7 +1393,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.mmio.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1400,7 +1401,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1408,7 +1409,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1416,7 +1417,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } @@ -1424,171 +1425,171 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1606,7 +1607,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1624,7 +1625,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1642,7 +1643,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1660,7 +1661,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1678,7 +1679,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1696,7 +1697,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1714,7 +1715,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1732,7 +1733,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1750,7 +1751,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1768,7 +1769,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1786,7 +1787,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1804,7 +1805,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, (), @@ -1829,7 +1830,7 @@ struct __cuda_atomic_bind_store { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - __cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); + ::cuda::std::__cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); } }; template @@ -1845,7 +1846,7 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} + if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ __backend, __ptr_proxy, *__val_proxy}; @@ -1858,168 +1859,168 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2039,7 +2040,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2059,7 +2060,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2079,7 +2080,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2099,7 +2100,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2119,7 +2120,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2139,7 +2140,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2159,7 +2160,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2179,7 +2180,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2199,7 +2200,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2219,7 +2220,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2239,7 +2240,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2259,7 +2260,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2279,7 +2280,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2299,7 +2300,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2319,7 +2320,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2339,7 +2340,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2359,7 +2360,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2379,7 +2380,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2399,7 +2400,7 @@ template static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2428,7 +2429,7 @@ struct __cuda_atomic_bind_compare_exchange { template [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) { - return __cuda_atomic_compare_exchange( + return ::cuda::std::__cuda_atomic_compare_exchange( __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); } }; @@ -2453,7 +2454,7 @@ template bool __res = false; if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} + if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; @@ -2466,168 +2467,168 @@ template template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2647,7 +2648,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2667,7 +2668,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2687,7 +2688,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2707,7 +2708,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2727,7 +2728,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2747,7 +2748,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2767,7 +2768,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2787,7 +2788,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2807,7 +2808,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2827,7 +2828,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2847,7 +2848,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2867,7 +2868,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2887,7 +2888,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2907,7 +2908,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2927,7 +2928,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2947,7 +2948,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2967,7 +2968,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -2987,7 +2988,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -3007,7 +3008,7 @@ template static inline _CCCL_DEVICE void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { - __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840"); NV_DISPATCH_TARGET( NV_PROVIDES_SM_90, (), @@ -3035,7 +3036,7 @@ struct __cuda_atomic_bind_exchange { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); } }; template @@ -3056,7 +3057,7 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); if constexpr (_Backend::__requires_local_memory_workaround) { - if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} + if(::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ __backend, __ptr_proxy, __old_proxy, *__new_proxy}; @@ -3069,7 +3070,8 @@ template { using __value_type = __unv<_Type>; __value_type __old; - __cuda_atomic_exchange_dispatch(__backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); + ::cuda::std::__cuda_atomic_exchange_dispatch( + __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); return __old; } @@ -3078,403 +3080,403 @@ template template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -3487,7 +3489,7 @@ struct __cuda_atomic_bind_fetch_add { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_fetch_add(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_fetch_add(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template (&__op); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + if (::cuda::std::__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } __cuda_atomic_bind_fetch_add<_Backend, __proxy_pointee> __bound_add{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; @@ -3522,163 +3524,163 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -3691,7 +3693,7 @@ struct __cuda_atomic_bind_fetch_and { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_fetch_and(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_fetch_and(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template (&__op); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + if (::cuda::std::__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } __cuda_atomic_bind_fetch_and<_Backend, __proxy_pointee> __bound_and{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; @@ -3725,323 +3727,323 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -4054,7 +4056,7 @@ struct __cuda_atomic_bind_fetch_max { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_fetch_max(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_fetch_max(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template (&__op); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + if (::cuda::std::__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } __cuda_atomic_bind_fetch_max<_Backend, __proxy_pointee> __bound_max{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; @@ -4088,323 +4090,323 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -4417,7 +4419,7 @@ struct __cuda_atomic_bind_fetch_min { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_fetch_min(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_fetch_min(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template (&__op); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + if (::cuda::std::__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } __cuda_atomic_bind_fetch_min<_Backend, __proxy_pointee> __bound_min{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; @@ -4451,163 +4453,163 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -4620,7 +4622,7 @@ struct __cuda_atomic_bind_fetch_or { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_fetch_or(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_fetch_or(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template (&__op); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + if (::cuda::std::__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } __cuda_atomic_bind_fetch_or<_Backend, __proxy_pointee> __bound_or{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; @@ -4654,163 +4656,163 @@ template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) -{ __cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } +{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } #endif // _CCCL_CUDA_COMPILATION() @@ -4823,7 +4825,7 @@ struct __cuda_atomic_bind_fetch_xor { template _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - __cuda_atomic_fetch_xor(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + ::cuda::std::__cuda_atomic_fetch_xor(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); } }; template (&__op); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} + if (::cuda::std::__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } __cuda_atomic_bind_fetch_xor<_Backend, __proxy_pointee> __bound_xor{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index 197eebfe26be..6f13b2a8a379 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -129,14 +129,14 @@ template _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_order_dispatch( _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) { - __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); + ::cuda::std::__cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); } template _CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_order_dispatch(_Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) { - __cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); + ::cuda::std::__cuda_atomic_rmw_order_dispatch(__backend, __fn, __order, __scope, __args...); } template @@ -227,7 +227,7 @@ struct __cuda_atomic_bind_fetch_sub template _CCCL_HOST_DEVICE_API void operator()(_Order __order, _Operand, _Sco) { - __cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); + ::cuda::std::__cuda_atomic_fetch_sub(__backend, __ptr, *__dst, *__op, __order, _Operand{}, _Sco{}); } }; @@ -247,14 +247,14 @@ __cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, mem auto* __op_proxy = reinterpret_cast<__proxy_type*>(&__op); if constexpr (_Backend::__requires_local_memory_workaround) { - if (__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + if (::cuda::std::__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) { return __dst; } } __cuda_atomic_bind_fetch_sub<_Backend, __proxy_pointee> __bound_fetch_sub{ __backend, __ptr_proxy, __dst_proxy, __op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_sub, __order, __scope, __proxy_operand{}); + ::cuda::std::__cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_sub, __order, __scope, __proxy_operand{}); return __dst; } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic.h b/libcudacxx/include/cuda/std/__atomic/functions/generic.h index 922a26567fcc..7a2afd4a572c 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic.h @@ -55,12 +55,12 @@ _CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( static_assert(__rmw_size <= _Backend::__widest_cas, "atomic CAS cannot be widened beyond the backend's widest CAS"); using __rmw_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, __rmw_size>; - const auto __result = __cuda_atomic_rmw( + const auto __result = ::cuda::std::__cuda_atomic_rmw( __backend, __ptr, __cuda_atomic_compare_exchange_op<_Type>{__cmp, __op}, __order, - __cuda_atomic_compare_exchange_initial_load_order(__order), + ::cuda::std::__cuda_atomic_compare_exchange_initial_load_order(__order), _Operand{}, __rmw_operand{}, __scope); @@ -93,7 +93,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_add( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_add>{__op}, __order, _Operand{}, __scope); } @@ -107,7 +107,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_sub( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_sub>{__op}, __order, _Operand{}, __scope); } @@ -121,7 +121,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_and( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_and>{__op}, __order, _Operand{}, __scope); } @@ -135,7 +135,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_or( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_or>{__op}, __order, _Operand{}, __scope); } @@ -149,7 +149,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_xor( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, ::cuda::std::bit_xor>{__op}, __order, _Operand{}, __scope); } @@ -163,7 +163,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_min( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_min>{__op}, __order, _Operand{}, __scope); } @@ -177,7 +177,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_fetch_max( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_fetch_max>{__op}, __order, _Operand{}, __scope); } @@ -191,7 +191,7 @@ _CCCL_HOST_DEVICE static void __cuda_atomic_exchange( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; - __dst = __cuda_atomic_fetch_update( + __dst = ::cuda::std::__cuda_atomic_fetch_update( __backend, __ptr, __cuda_atomic_op_bind<_ValueType, __cuda_atomic_op_store>{__op}, __order, _Operand{}, __scope); } _CCCL_END_NAMESPACE_CUDA_STD diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h index f9c31d6a12de..1bbe73aa91b9 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h @@ -287,7 +287,7 @@ struct __cuda_atomic_op_fetch_min { [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const { - return __cuda_atomic_less(__op, __old) ? __op : __old; + return ::cuda::std::__cuda_atomic_less(__op, __old) ? __op : __old; } }; @@ -296,7 +296,7 @@ struct __cuda_atomic_op_fetch_max { [[nodiscard]] _CCCL_HOST_DEVICE_API _Type operator()(_Type __op, _Type __old) const { - return __cuda_atomic_less(__old, __op) ? __op : __old; + return ::cuda::std::__cuda_atomic_less(__old, __op) ? __op : __old; } }; @@ -312,12 +312,12 @@ __cuda_atomic_fetch_update(_Backend __backend, _Pointee* __ptr, const _Fn& __op, using __bitwise_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, _Operand::__size>; using __rmw_operand = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, __rmw_size>; - return __cuda_atomic_rmw( + return ::cuda::std::__cuda_atomic_rmw( __backend, __ptr, __cuda_atomic_rmw_op<_Type, _Fn>{__op}, __order, - __cuda_atomic_initial_load_order(__order), + ::cuda::std::__cuda_atomic_initial_load_order(__order), __bitwise_operand{}, __rmw_operand{}, __scope) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/host.h b/libcudacxx/include/cuda/std/__atomic/functions/host.h index 9ccfa03c991e..b8ca3782b9f1 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/host.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/host.h @@ -135,7 +135,7 @@ _CCCL_HOST_API bool __cuda_atomic_compare_exchange( // This is only alignment wrapped in order to prevent GCC-6 from triggering an unused warning. &__atomic_force_align_host(&__dst)->__atom, &__op, - __cuda_atomic_cas_is_weak(__cas), + ::cuda::std::__cuda_atomic_cas_is_weak(__cas), __atomic_order_to_int(__order.__success), __atomic_failure_order_to_int(__order.__failure)); } @@ -152,14 +152,14 @@ _CCCL_HOST_API bool __cuda_atomic_compare_exchange( _Operand __operand, __thread_scope_tag __scope) { - return __cuda_atomic_compare_exchange( + return ::cuda::std::__cuda_atomic_compare_exchange( __backend, __ptr, __dst, __cmp, __op, __cas, - __cuda_atomic_runtime_cas_order{__order, __cuda_atomic_failure_order(__order)}, + __cuda_atomic_runtime_cas_order{__order, ::cuda::std::__cuda_atomic_failure_order(__order)}, __operand, __scope); } From a263bb70b547381ecb1862cf0ce8d92620149b89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Thu, 27 Aug 2026 15:25:05 -0700 Subject: [PATCH 09/24] Limit PTX fence state to pre-SM70 lowering. --- .../include/cuda/std/__atomic/functions/cuda_ptx.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h index 325db4f4d5c4..fbfd356d1384 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -45,12 +45,6 @@ _CCCL_DEVICE_API auto __cuda_atomic_ptx_backend::__with_transformed_order( static_assert(__is_load || __is_store || __is_rmw, "invalid atomic operation class"); constexpr bool __is_seq_cst = is_same_v<_Order, __cuda_atomic_order_seq_cst>; - constexpr bool __is_release = is_same_v<_Order, __cuda_atomic_order_release>; - constexpr bool __is_acq_rel = is_same_v<_Order, __cuda_atomic_order_acq_rel>; - constexpr bool __is_acquire = is_same_v<_Order, __cuda_atomic_order_acquire>; - constexpr bool __membar_before = - __is_seq_cst || (__is_store && __is_release) || (__is_rmw && (__is_release || __is_acq_rel)); - constexpr bool __membar_after = (__is_load || __is_rmw) && (__is_acquire || __is_acq_rel || __is_seq_cst); NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, @@ -73,6 +67,13 @@ _CCCL_DEVICE_API auto __cuda_atomic_ptx_backend::__with_transformed_order( }), NV_IS_DEVICE, ({ + constexpr bool __is_release = is_same_v<_Order, __cuda_atomic_order_release>; + constexpr bool __is_acq_rel = is_same_v<_Order, __cuda_atomic_order_acq_rel>; + constexpr bool __is_acquire = is_same_v<_Order, __cuda_atomic_order_acquire>; + constexpr bool __membar_before = + __is_seq_cst || (__is_store && __is_release) || (__is_rmw && (__is_release || __is_acq_rel)); + constexpr bool __membar_after = (__is_load || __is_rmw) && (__is_acquire || __is_acq_rel || __is_seq_cst); + if constexpr (__membar_before) { ::cuda::std::__cuda_atomic_membar(__scope); From be10761a7ade18392f19b8cb02603acfcd13ebf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Thu, 27 Aug 2026 15:56:35 -0700 Subject: [PATCH 10/24] Guard CUDA-only local memory fallbacks. --- .../codegen/generators/compare_and_swap.h | 2 ++ libcudacxx/codegen/generators/exchange.h | 2 ++ libcudacxx/codegen/generators/fetch_ops.h | 2 ++ libcudacxx/codegen/generators/ld_st.h | 4 ++++ .../__atomic/functions/cuda_ptx_generated.h | 20 +++++++++++++++++++ .../cuda/std/__atomic/functions/dispatch.h | 2 ++ 6 files changed, 32 insertions(+) diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index 9929a19e0d2f..6b32ca61a6bb 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -160,11 +160,13 @@ template __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); +#if _CCCL_CUDA_COMPILATION() bool __res = false; if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index 1e3932a7619c..fa7f67a26c66 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -156,10 +156,12 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if(::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ __backend, __ptr_proxy, __old_proxy, *__new_proxy}; __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index 15ee6db1c810..f066dce35f53 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -97,10 +97,12 @@ template (__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) {{ if (::cuda::std::__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} }} +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_pointee> __bound_{0}{{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope, __proxy_tag{{}}); diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index d5d55d82a7a3..e29eab29c193 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -216,10 +216,12 @@ __cuda_atomic_load_dispatch( using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; __cuda_atomic_load_order_dispatch( @@ -411,10 +413,12 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __value_type __store = __val; __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ __backend, __ptr_proxy, *__val_proxy}; __cuda_atomic_store_order_dispatch( diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 6eadb4711b01..0bf14bb63bab 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -1304,10 +1304,12 @@ __cuda_atomic_load_dispatch( using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ __backend, __ptr_proxy, __dst_proxy}; __cuda_atomic_load_order_dispatch( @@ -1844,10 +1846,12 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __value_type __store = __val; __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ __backend, __ptr_proxy, *__val_proxy}; __cuda_atomic_store_order_dispatch( @@ -2451,11 +2455,13 @@ template __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); +#if _CCCL_CUDA_COMPILATION() bool __res = false; if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; return __cuda_atomic_compare_exchange_order_dispatch( @@ -3055,10 +3061,12 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if(::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ __backend, __ptr_proxy, __old_proxy, *__new_proxy}; __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); @@ -3508,10 +3516,12 @@ template (__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_add<_Backend, __proxy_pointee> __bound_add{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope, __proxy_tag{}); @@ -3711,10 +3721,12 @@ template (__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_and<_Backend, __proxy_pointee> __bound_and{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope, __proxy_tag{}); @@ -4074,10 +4086,12 @@ template (__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_max<_Backend, __proxy_pointee> __bound_max{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope, __proxy_tag{}); @@ -4437,10 +4451,12 @@ template (__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_min<_Backend, __proxy_pointee> __bound_min{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope, __proxy_tag{}); @@ -4640,10 +4656,12 @@ template (__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_or<_Backend, __proxy_pointee> __bound_or{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope, __proxy_tag{}); @@ -4843,10 +4861,12 @@ template (__ptr); __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_xor<_Backend, __proxy_pointee> __bound_xor{ __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope, __proxy_tag{}); diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index 6f13b2a8a379..ed55614d32e2 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -245,6 +245,7 @@ __cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, mem auto* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); auto* __dst_proxy = reinterpret_cast<__proxy_type*>(&__dst); auto* __op_proxy = reinterpret_cast<__proxy_type*>(&__op); +#if _CCCL_CUDA_COMPILATION() if constexpr (_Backend::__requires_local_memory_workaround) { if (::cuda::std::__cuda_atomic_fetch_sub_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) @@ -252,6 +253,7 @@ __cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, mem return __dst; } } +#endif // _CCCL_CUDA_COMPILATION() __cuda_atomic_bind_fetch_sub<_Backend, __proxy_pointee> __bound_fetch_sub{ __backend, __ptr_proxy, __dst_proxy, __op_proxy}; ::cuda::std::__cuda_atomic_fetch_order_dispatch(__backend, __bound_fetch_sub, __order, __scope, __proxy_operand{}); From c76731723766ffd35f3b1e461b83367d37dd1ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Fri, 28 Aug 2026 09:44:18 -0700 Subject: [PATCH 11/24] CI and coderabbit fixes. --- libcudacxx/codegen/generators/ld_st.h | 6 +- .../cuda/std/__atomic/functions/cuda_ptx.h | 5 +- .../std/__atomic/functions/cuda_ptx_backend.h | 3 +- .../__atomic/functions/cuda_ptx_generated.h | 130 +++++++++--------- 4 files changed, 73 insertions(+), 71 deletions(-) diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index e29eab29c193..48ffc4c53b03 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -256,7 +256,7 @@ inline void FormatStore(std::ostream& out) constexpr auto asm_intrinsic_format_128 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -275,12 +275,12 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( constexpr auto asm_intrinsic_format = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); const uint16_t __tmp = static_cast(__val); diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h index fbfd356d1384..783cd9d3a4b5 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -28,9 +28,12 @@ #include #include #include +#include #include #include +#include + #include _CCCL_BEGIN_NAMESPACE_CUDA_STD @@ -44,7 +47,7 @@ _CCCL_DEVICE_API auto __cuda_atomic_ptx_backend::__with_transformed_order( constexpr bool __is_rmw = is_same_v<_Operation, __cuda_atomic_operation_rmw>; static_assert(__is_load || __is_store || __is_rmw, "invalid atomic operation class"); - constexpr bool __is_seq_cst = is_same_v<_Order, __cuda_atomic_order_seq_cst>; + [[maybe_unused]] constexpr bool __is_seq_cst = is_same_v<_Order, __cuda_atomic_order_seq_cst>; NV_DISPATCH_TARGET( NV_PROVIDES_SM_70, diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h index 5112cb8f1bbb..e795a0c0ea92 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_backend.h @@ -22,8 +22,7 @@ #endif // no system header #include -#include -#include +#include #include #include diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 0bf14bb63bab..405e9fe739b3 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -1329,7 +1329,7 @@ __cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1337,7 +1337,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1345,7 +1345,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1353,7 +1353,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1361,7 +1361,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1369,7 +1369,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1377,7 +1377,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1385,7 +1385,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1393,7 +1393,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1401,7 +1401,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1409,7 +1409,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1417,7 +1417,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1425,7 +1425,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); const uint16_t __tmp = static_cast(__val); @@ -1433,163 +1433,163 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1607,7 +1607,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1625,7 +1625,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1643,7 +1643,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1661,7 +1661,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1679,7 +1679,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1697,7 +1697,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1715,7 +1715,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1733,7 +1733,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1751,7 +1751,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1769,7 +1769,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1787,7 +1787,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); @@ -1805,7 +1805,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( } template static inline _CCCL_DEVICE void __cuda_atomic_store( - __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) + __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840"); From 246e6dfdaba2bc015e4953b0e37802ffa9bcd788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Fri, 28 Aug 2026 09:47:48 -0700 Subject: [PATCH 12/24] Address review comments. --- libcudacxx/codegen/generators/exchange.h | 10 ++--- .../cuda/std/__atomic/functions/backend.h | 39 ++++++++++--------- .../cuda/std/__atomic/functions/cuda_local.h | 14 +++---- .../__atomic/functions/cuda_ptx_generated.h | 10 ++--- .../cuda/std/__atomic/functions/dispatch.h | 8 ++-- 5 files changed, 41 insertions(+), 40 deletions(-) diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index fa7f67a26c66..962c0a4e69f1 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -149,10 +149,10 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( memory_order __order, _Sco __scope) { - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + using __value_type _CCCL_NODEBUG = __unv<_Type>; + using __proxy_t _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee _CCCL_NODEBUG = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); @@ -171,7 +171,7 @@ template [[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_exchange_dispatch( _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) { - using __value_type = __unv<_Type>; + using __value_type _CCCL_NODEBUG = __unv<_Type>; __value_type __old; ::cuda::std::__cuda_atomic_exchange_dispatch( __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); diff --git a/libcudacxx/include/cuda/std/__atomic/functions/backend.h b/libcudacxx/include/cuda/std/__atomic/functions/backend.h index 514c3a068acb..a2476a286328 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/backend.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/backend.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -47,14 +48,14 @@ enum class __cuda_atomic_order }; template <__cuda_atomic_order _Order> -using __cuda_atomic_order_tag = integral_constant<__cuda_atomic_order, _Order>; +using __cuda_atomic_order_tag _CCCL_NODEBUG = integral_constant<__cuda_atomic_order, _Order>; -using __cuda_atomic_order_relaxed = __cuda_atomic_order_tag<__cuda_atomic_order::_relaxed>; -using __cuda_atomic_order_release = __cuda_atomic_order_tag<__cuda_atomic_order::_release>; -using __cuda_atomic_order_acquire = __cuda_atomic_order_tag<__cuda_atomic_order::_acquire>; -using __cuda_atomic_order_acq_rel = __cuda_atomic_order_tag<__cuda_atomic_order::_acq_rel>; -using __cuda_atomic_order_seq_cst = __cuda_atomic_order_tag<__cuda_atomic_order::_seq_cst>; -using __cuda_atomic_order_volatile = __cuda_atomic_order_tag<__cuda_atomic_order::_volatile>; +using __cuda_atomic_order_relaxed _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_relaxed>; +using __cuda_atomic_order_release _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_release>; +using __cuda_atomic_order_acquire _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_acquire>; +using __cuda_atomic_order_acq_rel _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_acq_rel>; +using __cuda_atomic_order_seq_cst _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_seq_cst>; +using __cuda_atomic_order_volatile _CCCL_NODEBUG = __cuda_atomic_order_tag<__cuda_atomic_order::_volatile>; template struct __cuda_atomic_ptx_order : _Order @@ -66,10 +67,10 @@ struct __cuda_atomic_ptx_order : _Order {} }; -using __cuda_atomic_ptx_order_relaxed = __cuda_atomic_ptx_order<__cuda_atomic_order_relaxed>; -using __cuda_atomic_ptx_order_release = __cuda_atomic_ptx_order<__cuda_atomic_order_release>; -using __cuda_atomic_ptx_order_acquire = __cuda_atomic_ptx_order<__cuda_atomic_order_acquire>; -using __cuda_atomic_ptx_order_acq_rel = __cuda_atomic_ptx_order<__cuda_atomic_order_acq_rel>; +using __cuda_atomic_ptx_order_relaxed _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_relaxed>; +using __cuda_atomic_ptx_order_release _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_release>; +using __cuda_atomic_ptx_order_acquire _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_acquire>; +using __cuda_atomic_ptx_order_acq_rel _CCCL_NODEBUG = __cuda_atomic_ptx_order<__cuda_atomic_order_acq_rel>; struct __cuda_atomic_operation_load {}; @@ -89,8 +90,8 @@ struct __cuda_atomic_runtime_cas_order template struct __cuda_atomic_cas_order { - using __success = _Success; - using __failure = _Failure; + using __success _CCCL_NODEBUG = _Success; + using __failure _CCCL_NODEBUG = _Failure; }; struct __cuda_atomic_cas_strong @@ -253,14 +254,14 @@ struct _CCCL_ALIGNAS(16) __cuda_atomic_longlong2 template [[nodiscard]] _CCCL_HOST_DEVICE_API _CCCL_CONSTEVAL auto __cuda_atomic_deduce_bitwise_impl() noexcept { - using __tag = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, sizeof(_Type) * 8>; + using __tag = __cuda_atomic_operand_tag<__cuda_atomic_operand::_b, sizeof(_Type) * CHAR_BIT>; if constexpr (sizeof(_Type) == 16) { return __cuda_atomic_operand_deduction<__cuda_atomic_longlong2, __tag>{}; } else { - return __cuda_atomic_operand_deduction<__make_nbit_uint_t, __tag>{}; + return __cuda_atomic_operand_deduction<__make_nbit_uint_t, __tag>{}; } } @@ -279,14 +280,14 @@ template constexpr auto __op = __is_floating ? __cuda_atomic_operand::_f : (is_signed_v<_Type> && sizeof(_Type) != 8 ? __cuda_atomic_operand::_s : __cuda_atomic_operand::_u); - using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * 8>; + using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * CHAR_BIT>; if constexpr (__is_floating || sizeof(_Type) == 16) { return __cuda_atomic_operand_deduction<_Type, __tag>{}; } else { - return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; + return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; } } @@ -302,14 +303,14 @@ template constexpr bool __is_floating = is_floating_point_v<_Type> || __is_extended_floating_point_v<_Type>; constexpr auto __op = __is_floating ? __cuda_atomic_operand::_f : (is_signed_v<_Type> ? __cuda_atomic_operand::_s : __cuda_atomic_operand::_u); - using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * 8>; + using __tag = __cuda_atomic_operand_tag<__op, sizeof(_Type) * CHAR_BIT>; if constexpr (__is_floating || sizeof(_Type) == 16) { return __cuda_atomic_operand_deduction<_Type, __tag>{}; } else { - return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; + return __cuda_atomic_operand_deduction<__make_nbit_int_t>, __tag>{}; } } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h index f69501afbedc..a377ef9e1093 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_local.h @@ -53,37 +53,37 @@ _CCCL_DEVICE_API inline bool __cuda_atomic_is_local(const volatile void* __ptr) } template -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_and(_Type __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_and(_Type __atom, _Type const& __v) { return __atom & __v; } template -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_or(_Type __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_or(_Type __atom, _Type const& __v) { return __atom | __v; } template -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_xor(_Type __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_xor(_Type __atom, _Type const& __v) { return __atom ^ __v; } template -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_add(_Type __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_add(_Type __atom, _Type const& __v) { return __atom + __v; } template -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_sub(_Type __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_sub(_Type __atom, _Type const& __v) { return __atom - __v; } template -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_max(_Type __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_max(_Type __atom, _Type const& __v) { return ::cuda::std::__cuda_atomic_less(__atom, __v) ? __v : __atom; } template -_CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_min(_Type __atom, _Type const& __v) +[[nodiscard]] _CCCL_DEVICE_API _Type __cuda_atomic_fetch_local_bop_min(_Type __atom, _Type const& __v) { return ::cuda::std::__cuda_atomic_less(__v, __atom) ? __v : __atom; } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 405e9fe739b3..3aa950a21c8d 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -3054,10 +3054,10 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( memory_order __order, _Sco __scope) { - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + using __value_type _CCCL_NODEBUG = __unv<_Type>; + using __proxy_t _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee _CCCL_NODEBUG = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); @@ -3076,7 +3076,7 @@ template [[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_exchange_dispatch( _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) { - using __value_type = __unv<_Type>; + using __value_type _CCCL_NODEBUG = __unv<_Type>; __value_type __old; ::cuda::std::__cuda_atomic_exchange_dispatch( __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index ed55614d32e2..34340c5d7100 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -36,7 +36,7 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD template _CCCL_HOST_DEVICE_API void __cuda_atomic_load_order_dispatch( - [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope, _Args... __args) + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) { if constexpr (!_Backend::__needs_constant_order) { @@ -65,7 +65,7 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_load_order_dispatch( template _CCCL_HOST_DEVICE_API void __cuda_atomic_store_order_dispatch( - [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope, _Args... __args) + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) { if constexpr (!_Backend::__needs_constant_order) { @@ -92,7 +92,7 @@ _CCCL_HOST_DEVICE_API void __cuda_atomic_store_order_dispatch( template _CCCL_HOST_DEVICE_API void __cuda_atomic_rmw_order_dispatch( - [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, [[maybe_unused]] _Sco __scope, _Args... __args) + [[maybe_unused]] _Backend __backend, _Fn& __fn, memory_order __order, _Sco __scope, _Args... __args) { if constexpr (!_Backend::__needs_constant_order) { @@ -180,7 +180,7 @@ template _Fn& __fn, memory_order __success, memory_order __failure, - [[maybe_unused]] _Sco __scope, + _Sco __scope, _Args... __args) { if constexpr (!_Backend::__needs_constant_order) From f0d69e2c9cca65d9bb84150cd1a3a27f51e01b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Fri, 28 Aug 2026 10:01:38 -0700 Subject: [PATCH 13/24] Use libcu++ API annotations throughout atomics. --- .../codegen/generators/compare_and_swap.h | 4 +- libcudacxx/codegen/generators/exchange.h | 4 +- libcudacxx/codegen/generators/fence.h | 10 +- libcudacxx/codegen/generators/fetch_ops.h | 2 +- libcudacxx/codegen/generators/ld_st.h | 12 +- .../__atomic/functions/cuda_ptx_generated.h | 1548 ++++++++--------- .../cuda/std/__atomic/functions/dispatch.h | 2 +- .../cuda/std/__atomic/functions/generic.h | 20 +- .../cuda/std/__atomic/functions/generic_rmw.h | 2 +- 9 files changed, 802 insertions(+), 802 deletions(-) diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index 6b32ca61a6bb..d1c42927274c 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -28,7 +28,7 @@ inline void FormatCompareAndSwap(std::ostream& out) // 6 - Scope function tag constexpr auto asm_intrinsic_format_128 = R"XXX( template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); @@ -49,7 +49,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; }})XXX"; constexpr auto asm_intrinsic_format = R"XXX( template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX"; diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index 962c0a4e69f1..ad5f9c96bb01 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -28,7 +28,7 @@ inline void FormatExchange(std::ostream& out) // 6 - Scope function tag constexpr auto asm_intrinsic_format_128 = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); @@ -49,7 +49,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( }})XXX"; constexpr auto asm_intrinsic_format = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX"; diff --git a/libcudacxx/codegen/generators/fence.h b/libcudacxx/codegen/generators/fence.h index 3072656e5155..928c7c8391e1 100644 --- a/libcudacxx/codegen/generators/fence.h +++ b/libcudacxx/codegen/generators/fence.h @@ -33,7 +33,7 @@ inline void FormatFence(std::ostream& out) // 0 - Membar scope tag // 1 - Membar scope constexpr auto intrinsic_membar = R"XXX( -static inline _CCCL_DEVICE void __cuda_atomic_membar({0}) +_CCCL_DEVICE_API inline void __cuda_atomic_membar({0}) {{ asm volatile("membar{1};" ::: "memory"); }})XXX"; const std::map membar_scopes{ @@ -53,7 +53,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_membar({0}) // 2 - Fence order tag // 3 - Fence order constexpr auto intrinsic_fence = R"XXX( -static inline _CCCL_DEVICE void __cuda_atomic_fence({0}, {2}) +_CCCL_DEVICE_API inline void __cuda_atomic_fence({0}, {2}) {{ asm volatile("fence{1}{3};" ::: "memory"); }})XXX"; const Scope fence_scopes[] = { @@ -78,7 +78,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_fence({0}, {2}) out << "\n" << R"XXX( template -static inline _CCCL_DEVICE void +_CCCL_DEVICE_API void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco __scope) { if (__order.__was_seq_cst) @@ -88,11 +88,11 @@ __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco _ } template -static inline _CCCL_DEVICE void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) +_CCCL_DEVICE_API void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) {} template -static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( +_CCCL_DEVICE_API void __cuda_atomic_thread_fence( __cuda_atomic_ptx_backend, memory_order __order, _Sco) { [[maybe_unused]] const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index f066dce35f53..695f75fd3b69 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -61,7 +61,7 @@ inline void FormatFetchOps(std::ostream& out) // 7 - Scope function tag constexpr auto asm_intrinsic_format = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_{0}( +_CCCL_DEVICE_API void __cuda_atomic_fetch_{0}( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, {5} __order, __cuda_atomic_operand_{1}{2}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {7}{{}}); asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; // 0 - Atomic Operation diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index 48ffc4c53b03..aa4753931fbf 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -50,7 +50,7 @@ inline void FormatLoad(std::ostream& out) // 8 - Mmio semantic constexpr auto asm_intrinsic_format_128 = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); @@ -69,12 +69,12 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( }})XXX"; constexpr auto asm_intrinsic_format = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); @@ -255,7 +255,7 @@ inline void FormatStore(std::ostream& out) // 8 - Mmio semantic constexpr auto asm_intrinsic_format_128 = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); @@ -274,12 +274,12 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( }})XXX"; constexpr auto asm_intrinsic_format = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX"; constexpr auto asm_intrinsic_format_8 = R"XXX( template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, {3} __order, __cuda_atomic_operand_{0}{1}, {5}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {5}{{}}); diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index 3aa950a21c8d..bf414d050e65 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -51,31 +51,31 @@ extern "C" _CCCL_DEVICE void __atomic_cas_128b_unsupported_before_SM_90(); extern "C" _CCCL_DEVICE void __atomic_exchange_128b_unsupported_before_SM_90(); extern "C" _CCCL_DEVICE void __atomic_ldst_128b_unsupported_before_SM_70(); -static inline _CCCL_DEVICE void __cuda_atomic_membar(__thread_scope_block_tag) +_CCCL_DEVICE_API inline void __cuda_atomic_membar(__thread_scope_block_tag) { asm volatile("membar.cta;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_membar(__thread_scope_device_tag) +_CCCL_DEVICE_API inline void __cuda_atomic_membar(__thread_scope_device_tag) { asm volatile("membar.gl;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_membar(__thread_scope_system_tag) +_CCCL_DEVICE_API inline void __cuda_atomic_membar(__thread_scope_system_tag) { asm volatile("membar.sys;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_block_tag, __cuda_atomic_order_acq_rel) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_block_tag, __cuda_atomic_order_acq_rel) { asm volatile("fence.acq_rel.cta;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_block_tag, __cuda_atomic_order_seq_cst) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_block_tag, __cuda_atomic_order_seq_cst) { asm volatile("fence.sc.cta;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_cluster_tag, __cuda_atomic_order_acq_rel) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_cluster_tag, __cuda_atomic_order_acq_rel) { asm volatile("fence.acq_rel.cluster;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_cluster_tag, __cuda_atomic_order_seq_cst) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_cluster_tag, __cuda_atomic_order_seq_cst) { asm volatile("fence.sc.cluster;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_device_tag, __cuda_atomic_order_acq_rel) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_device_tag, __cuda_atomic_order_acq_rel) { asm volatile("fence.acq_rel.gpu;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_device_tag, __cuda_atomic_order_seq_cst) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_device_tag, __cuda_atomic_order_seq_cst) { asm volatile("fence.sc.gpu;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_system_tag, __cuda_atomic_order_acq_rel) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_system_tag, __cuda_atomic_order_acq_rel) { asm volatile("fence.acq_rel.sys;" ::: "memory"); } -static inline _CCCL_DEVICE void __cuda_atomic_fence(__thread_scope_system_tag, __cuda_atomic_order_seq_cst) +_CCCL_DEVICE_API inline void __cuda_atomic_fence(__thread_scope_system_tag, __cuda_atomic_order_seq_cst) { asm volatile("fence.sc.sys;" ::: "memory"); } template -static inline _CCCL_DEVICE void +_CCCL_DEVICE_API void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco __scope) { if (__order.__was_seq_cst) @@ -85,11 +85,11 @@ __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco _ } template -static inline _CCCL_DEVICE void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) +_CCCL_DEVICE_API void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco) {} template -static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( +_CCCL_DEVICE_API void __cuda_atomic_thread_fence( __cuda_atomic_ptx_backend, memory_order __order, _Sco) { [[maybe_unused]] const int __memorder = __atomic_order_to_int(__order); NV_DISPATCH_TARGET( @@ -119,7 +119,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_thread_fence( } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -128,7 +128,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -137,7 +137,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -146,7 +146,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -155,7 +155,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -164,7 +164,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -173,7 +173,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -182,7 +182,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -191,7 +191,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -200,7 +200,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -209,7 +209,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -218,7 +218,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -227,7 +227,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -236,7 +236,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -245,7 +245,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -254,7 +254,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -263,7 +263,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -272,7 +272,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -281,7 +281,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -290,7 +290,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -299,7 +299,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -308,7 +308,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -317,7 +317,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -326,7 +326,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -335,7 +335,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -344,7 +344,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -353,7 +353,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -362,7 +362,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -371,7 +371,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -380,7 +380,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -389,7 +389,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -398,7 +398,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -407,7 +407,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -416,7 +416,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -425,7 +425,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -434,7 +434,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -443,7 +443,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -452,7 +452,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -461,7 +461,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -470,579 +470,579 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( __dst = static_cast<__unv<_Type>>(__tmp); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s16 %0,[%1];" : "=h"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f32 %0,[%1];" : "=f"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s32 %0,[%1];" : "=r"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.b64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.f64 %0,[%1];" : "=d"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.u64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.acquire.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.acquire.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.acquire.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.acquire.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.relaxed.cta.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.relaxed.cluster.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.relaxed.gpu.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.mmio.relaxed.sys.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("ld.volatile.s64 %0,[%1];" : "=l"(__dst) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1060,7 +1060,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1078,7 +1078,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1096,7 +1096,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1114,7 +1114,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1132,7 +1132,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1150,7 +1150,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1168,7 +1168,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1186,7 +1186,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1204,7 +1204,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1222,7 +1222,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1240,7 +1240,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1258,7 +1258,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_load( +_CCCL_DEVICE_API void __cuda_atomic_load( __cuda_atomic_ptx_backend, const _Type* __ptr, __unv<_Type>& __dst, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1328,7 +1328,7 @@ __cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order #if _CCCL_CUDA_COMPILATION() template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1336,7 +1336,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.release.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1344,7 +1344,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.release.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1352,7 +1352,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.release.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1360,7 +1360,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.release.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1368,7 +1368,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.relaxed.cta.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1376,7 +1376,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.relaxed.cluster.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1384,7 +1384,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.relaxed.gpu.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1392,7 +1392,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1400,7 +1400,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.mmio.relaxed.sys.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1408,7 +1408,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1416,7 +1416,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1424,7 +1424,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b8, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1432,163 +1432,163 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( asm volatile("st.volatile.b8 [%0],%1;" :: "l"(__ptr), "h"(__tmp) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b16, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b16 [%0],%1;" :: "l"(__ptr), "h"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b32 [%0],%1;" :: "l"(__ptr), "r"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.release.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.release.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.release.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.release.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.relaxed.cta.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.relaxed.cluster.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.relaxed.gpu.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.mmio.relaxed.sys.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("st.volatile.b64 [%0],%1;" :: "l"(__ptr), "l"(__val) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1606,7 +1606,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1624,7 +1624,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1642,7 +1642,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1660,7 +1660,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1678,7 +1678,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1696,7 +1696,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1714,7 +1714,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1732,7 +1732,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_enable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1750,7 +1750,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -1768,7 +1768,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -1786,7 +1786,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -1804,7 +1804,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_store( +_CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag, __cuda_atomic_mmio_disable) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -1861,167 +1861,167 @@ __cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory #if _CCCL_CUDA_COMPILATION() template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b32 %0,[%1],%2,%3;" : "=r"(__dst) : "l"(__ptr), "r"(__cmp), "r"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acquire.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acquire.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acquire.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acquire.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.relaxed.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.relaxed.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.relaxed.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.relaxed.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.release.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.release.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.release.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.release.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.acq_rel.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.acq_rel.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.acq_rel.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.acq_rel.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.cas.cta.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.cas.cluster.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.cas.gpu.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.cas.sys.b64 %0,[%1],%2,%3;" : "=l"(__dst) : "l"(__ptr), "l"(__cmp), "l"(__op) : "memory"); return __dst == __cmp; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2041,7 +2041,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2061,7 +2061,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2081,7 +2081,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2101,7 +2101,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2121,7 +2121,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2141,7 +2141,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2161,7 +2161,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2181,7 +2181,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2201,7 +2201,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2221,7 +2221,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2241,7 +2241,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2261,7 +2261,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2281,7 +2281,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2301,7 +2301,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2321,7 +2321,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2341,7 +2341,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2361,7 +2361,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2381,7 +2381,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2401,7 +2401,7 @@ static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } template -static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange( +_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2471,167 +2471,167 @@ template #if _CCCL_CUDA_COMPILATION() template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b32 %0,[%1],%2;" : "=r"(__old) : "l"(__ptr), "r"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acquire.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acquire.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.release.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.release.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.release.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.release.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.exch.cta.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.exch.cluster.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.exch.gpu.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.exch.sys.b64 %0,[%1],%2;" : "=l"(__old) : "l"(__ptr), "l"(__new) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2651,7 +2651,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2671,7 +2671,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2691,7 +2691,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2711,7 +2711,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2731,7 +2731,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2751,7 +2751,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2771,7 +2771,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2791,7 +2791,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2811,7 +2811,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2831,7 +2831,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2851,7 +2851,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2871,7 +2871,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2891,7 +2891,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2911,7 +2911,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -2931,7 +2931,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -2951,7 +2951,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); @@ -2971,7 +2971,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); @@ -2991,7 +2991,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); @@ -3011,7 +3011,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_exchange( +_CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b128, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); @@ -3086,403 +3086,403 @@ template #if _CCCL_CUDA_COMPILATION() template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f32 %0,[%1],%2;" : "=f"(__dst) : "l"(__ptr), "f"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_f64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.f64 %0,[%1],%2;" : "=d"(__dst) : "l"(__ptr), "d"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.add.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.add.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.add.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_add( +_CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } @@ -3532,163 +3532,163 @@ template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.and.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.and.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.and.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_and( +_CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } @@ -3737,323 +3737,323 @@ template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.max.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.max.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.max.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_max( +_CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } @@ -4102,323 +4102,323 @@ template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_u64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.u64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acquire.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.relaxed.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.release.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.acq_rel.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.min.cta.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acquire.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.relaxed.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.release.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.acq_rel.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.min.cluster.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acquire.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.relaxed.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.release.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.acq_rel.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.min.gpu.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acquire.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.relaxed.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.release.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.acq_rel.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_min( +_CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } @@ -4467,163 +4467,163 @@ template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.or.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.or.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.or.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_or( +_CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } @@ -4672,163 +4672,163 @@ template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acquire.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.relaxed.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.release.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.acq_rel.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_block_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_block_tag{}); asm volatile("atom.xor.cta.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acquire.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.relaxed.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.release.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.acq_rel.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_cluster_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_cluster_tag{}); asm volatile("atom.xor.cluster.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acquire.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.relaxed.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.release.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.acq_rel.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_device_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_device_tag{}); asm volatile("atom.xor.gpu.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acquire.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_relaxed __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.relaxed.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.release.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acq_rel __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.acq_rel.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } template -static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor( +_CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.xor.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index 34340c5d7100..c41a51379f8c 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -261,7 +261,7 @@ __cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, mem } #if _CCCL_CUDA_COMPILATION() -_CCCL_DEVICE static inline void __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend, memory_order) +_CCCL_DEVICE_API inline void __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend, memory_order) { asm volatile("" ::: "memory"); } diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic.h b/libcudacxx/include/cuda/std/__atomic/functions/generic.h index 7a2afd4a572c..7d4ecb3415b8 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic.h @@ -39,7 +39,7 @@ template = false> -_CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( +_CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange( _Backend __backend, _Pointee* __ptr, __unv<_Pointee>& __dst, @@ -76,7 +76,7 @@ template _Backend::__widest_cas), bool> = false> -_CCCL_HOST_DEVICE static bool __cuda_atomic_compare_exchange( +_CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange( _Backend, _Pointee*, __unv<_Pointee>&, __unv<_Pointee>, __unv<_Pointee>, _Cas, _Order, _Operand, _Sco) { static_assert(_Operand::__size < _Backend::__widest_cas, "the backend must provide its widest CAS operation"); @@ -89,7 +89,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_add( +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_add( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; @@ -103,7 +103,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_sub( +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_sub( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; @@ -117,7 +117,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_and( +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_and( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; @@ -131,7 +131,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_or( +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_or( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; @@ -145,7 +145,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_xor( +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_xor( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; @@ -159,7 +159,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_min( +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_min( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; @@ -173,7 +173,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_fetch_max( +_CCCL_HOST_DEVICE_API void __cuda_atomic_fetch_max( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; @@ -187,7 +187,7 @@ template = false> -_CCCL_HOST_DEVICE static void __cuda_atomic_exchange( +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange( _Backend __backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order __order, _Operand, _Sco __scope) { using _ValueType = __unv<_Type>; diff --git a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h index 1bbe73aa91b9..234d6b977d76 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/generic_rmw.h @@ -147,7 +147,7 @@ template -_CCCL_HOST_DEVICE static __cuda_atomic_rmw_result<__unv<_Pointee>> __cuda_atomic_rmw( +_CCCL_HOST_DEVICE_API __cuda_atomic_rmw_result<__unv<_Pointee>> __cuda_atomic_rmw( _Backend __backend, _Pointee* __ptr, const _Fn& __op, From 41ecfb04123e1c2eaf33ba7eb5b8ad3a32c5a522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Fri, 28 Aug 2026 10:10:18 -0700 Subject: [PATCH 14/24] Correct SC fence placement in small arithmetic SASS checks. --- .../sass/arithmetic_minmax_types_8_16_atomic_ref.cu | 4 ++-- .../sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu | 4 ++-- .../atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu | 4 ++-- .../sass/arithmetic_volatile_types_8_16_atomic_ref.cu | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu index ef9688720d2b..0b0eceba24e4 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_types_8_16_atomic_ref.cu @@ -47,11 +47,11 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SM90-PLUS-DAG: {{.*}}HSETP2{{.*}} ; BLOCK-DAG: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} ; NON_BLOCK-DAG: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; RELEASE-DAG: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu index bf43a59c126c..ecd4bc32eacf 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_types_8_16_atomic_ref.cu @@ -47,11 +47,11 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} @@ -46,6 +44,8 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; MIN: {{.*}}IMNMX{{.*}} ; MAX: {{.*}}IMNMX{{.*}} ; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu index b963b13c438c..fe3909804bf8 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_volatile_types_8_16_atomic_ref.cu @@ -33,8 +33,6 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref Date: Fri, 28 Aug 2026 12:57:26 -0700 Subject: [PATCH 15/24] Relax float minmax patterns to account for reg alloc variance. --- .../sass/arithmetic_minmax_floating_types.cu | 8 ++++---- .../sass/arithmetic_minmax_volatile_floating_types.cu | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu index d08d4fde5438..3be3efe80f1a 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_floating_types.cu @@ -31,8 +31,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} -; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} ; SMXX-DAG: {{.*}}[[SASS_CALC]]{{.*}} ; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} ; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} @@ -41,8 +41,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; RELEASE-DAG: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} -; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} +; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.CAS{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu index 8f3342cc1fa3..b166a1f2e513 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_minmax_volatile_floating_types.cu @@ -31,8 +31,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} -; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} +; NON_BLOCK-DAG: {{.*}}LD.E[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ATOM_ADDR:R[0-9]+]]{{(\.64)?\].*}} ; SMXX-DAG: {{.*}}[[SASS_CALC]]{{.*}} ; SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} ; NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} @@ -42,8 +42,8 @@ extern "C" __device__ auto atomic_codegen_test(TEMPLATE& atom, TYPE ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{MIN|MAX}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.CAS[[SASS_SIZE]]{{.*}} -; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.{{CTA|SM}} {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} +; NON_BLOCK: {{.*}}ATOM.E.CAS[[SASS_SIZE]].STRONG.[[SASS_SCOPE]] {{P(T|[0-9]+)}}, [[OLD:R[0-9]+]], {{.*\[}}[[ATOM_ADDR]]{{(\.64)?\].*}}, {{R[0-9]+}}, {{R[0-9]+}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.CAS[[SASS_SIZE]]{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} From e42d75e9a439a9519947c2e4fb5c08c1639d3373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Fri, 28 Aug 2026 16:47:24 -0700 Subject: [PATCH 16/24] Move atomic dispatch machinery out of the PTX generator. --- .../codegen/generators/compare_and_swap.h | 52 +- libcudacxx/codegen/generators/exchange.h | 57 +-- libcudacxx/codegen/generators/fetch_ops.h | 58 +-- libcudacxx/codegen/generators/header.h | 10 +- libcudacxx/codegen/generators/ld_st.h | 95 +--- .../__atomic/functions/cuda_ptx_generated.h | 461 ------------------ .../cuda/std/__atomic/functions/dispatch.h | 424 ++++++++++++++++ 7 files changed, 432 insertions(+), 725 deletions(-) diff --git a/libcudacxx/codegen/generators/compare_and_swap.h b/libcudacxx/codegen/generators/compare_and_swap.h index d1c42927274c..51070cd3963e 100644 --- a/libcudacxx/codegen/generators/compare_and_swap.h +++ b/libcudacxx/codegen/generators/compare_and_swap.h @@ -124,57 +124,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( } } - out << "\n" - << R"XXX( -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_compare_exchange { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __exp; - __unv<_Type> __cmp; - __unv<_Type> __des; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) { - return ::cuda::std::__cuda_atomic_compare_exchange( - __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( - _Backend __backend, - _Type* __ptr, - __unv<_Type>* __exp, - __unv<_Type> __des, - _Cas, - memory_order __success, - memory_order __failure, - _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); -#if _CCCL_CUDA_COMPILATION() - bool __res = false; - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch( - __backend, __bound_compare_swap, __success, __failure, __scope, _Cas{}, __proxy_tag{}); -} - -#if _CCCL_CUDA_COMPILATION() -)XXX"; + out << "\n"; } #endif // COMPARED_AND_SWAP_H diff --git a/libcudacxx/codegen/generators/exchange.h b/libcudacxx/codegen/generators/exchange.h index ad5f9c96bb01..d6cdaa2cc5ad 100644 --- a/libcudacxx/codegen/generators/exchange.h +++ b/libcudacxx/codegen/generators/exchange.h @@ -124,62 +124,7 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( } } - out << "\n" - << R"XXX( -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_exchange { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __old; - __unv<_Type> __new; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); - } -}; -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( - _Backend __backend, - _Type* __ptr, - __unv<_Type>& __old, - __unv<_Type> __new, - memory_order __order, - _Sco __scope) -{ - using __value_type _CCCL_NODEBUG = __unv<_Type>; - using __proxy_t _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee _CCCL_NODEBUG = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if(::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ - __backend, __ptr_proxy, __old_proxy, *__new_proxy}; - __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_exchange_dispatch( - _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) -{ - using __value_type _CCCL_NODEBUG = __unv<_Type>; - __value_type __old; - ::cuda::std::__cuda_atomic_exchange_dispatch( - __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); - return __old; -} - -#if _CCCL_CUDA_COMPILATION() -)XXX"; + out << "\n"; } #endif // EXCHANGE_H diff --git a/libcudacxx/codegen/generators/fetch_ops.h b/libcudacxx/codegen/generators/fetch_ops.h index 695f75fd3b69..a7fcd0c4fd76 100644 --- a/libcudacxx/codegen/generators/fetch_ops.h +++ b/libcudacxx/codegen/generators/fetch_ops.h @@ -64,54 +64,7 @@ template _CCCL_DEVICE_API void __cuda_atomic_fetch_{0}( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, {5} __order, __cuda_atomic_operand_{1}{2}, {7}) {{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {7}{{}}); asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX"; - // 0 - Atomic Operation - // 1 - Operand type constraint - // 2 - Operand transform - constexpr auto fetch_bind_invoke = R"XXX( -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_fetch_{0} {{ - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __dst; - __unv<_Type> __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) {{ - ::cuda::std::__cuda_atomic_fetch_{0}(__backend, __ptr, *__dst, __op, __order, _Tag{{}}, _Sco{{}}); - }} -}}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_{0}_dispatch( - _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{{{2} - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_{1}_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_{1}_tag_t<__value_type>; - __value_type __dst{{}}; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - {{ - if (::cuda::std::__cuda_atomic_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}} - }} -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_fetch_{0}<_Backend, __proxy_pointee> __bound_{0}{{ - __backend, __ptr_proxy, __dst_proxy, *__op_proxy}}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_{0}, __order, __scope, __proxy_tag{{}}); - return __dst; -}} - -)XXX"; - - constexpr size_t supported_sizes[] = { + constexpr size_t supported_sizes[] = { 32, 64, }; @@ -131,15 +84,8 @@ template #include -#include -#include -#include -#include - #include #include #include #include -#include -#include #include -#include #include @@ -75,6 +67,8 @@ extern "C" _CCCL_DEVICE void __atomic_ldst_128b_unsupported_before_SM_70(); inline void FormatTail(std::ostream& out) { constexpr auto tail = R"XXX( +#endif // _CCCL_CUDA_COMPILATION() + _CCCL_END_NAMESPACE_CUDA_STD #include diff --git a/libcudacxx/codegen/generators/ld_st.h b/libcudacxx/codegen/generators/ld_st.h index aa4753931fbf..dbc5d38ef6ab 100644 --- a/libcudacxx/codegen/generators/ld_st.h +++ b/libcudacxx/codegen/generators/ld_st.h @@ -186,59 +186,7 @@ _CCCL_DEVICE_API void __cuda_atomic_load( } } } - out << "\n" - << R"XXX( -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_load { - _Backend __backend; - const _Type* __ptr; - __unv<_Type>* __dst; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - ::cuda::std::__cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_HOST_DEVICE_API void -__cuda_atomic_load_dispatch( - _Backend __backend, - const _Type* __ptr, - __unv<_Type>& __dst, - memory_order __order, - _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ - __backend, __ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch( - __backend, __bound_load, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> -__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) -{ - __unv<_Type> __dst; - ::cuda::std::__cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); - return __dst; -} - -#if _CCCL_CUDA_COMPILATION() -)XXX"; + out << "\n"; } inline void FormatStore(std::ostream& out) @@ -387,46 +335,7 @@ _CCCL_DEVICE_API void __cuda_atomic_store( } } } - out << "\n" - << R"XXX( -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_store { - _Backend __backend; - _Type* __ptr; - __unv<_Type> __val; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - ::cuda::std::__cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_HOST_DEVICE_API void -__cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __value_type __store = __val; - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ - __backend, __ptr_proxy, *__val_proxy}; - __cuda_atomic_store_order_dispatch( - __backend, __bound_store, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); -} - -#if _CCCL_CUDA_COMPILATION() -)XXX"; + out << "\n"; } #endif // LD_ST_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h index bf414d050e65..cab331b147c1 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx_generated.h @@ -27,19 +27,11 @@ #include #include -#include -#include -#include -#include - #include #include #include #include -#include -#include #include -#include #include @@ -1276,57 +1268,6 @@ _CCCL_DEVICE_API void __cuda_atomic_load( )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_load { - _Backend __backend; - const _Type* __ptr; - __unv<_Type>* __dst; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - ::cuda::std::__cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_HOST_DEVICE_API void -__cuda_atomic_load_dispatch( - _Backend __backend, - const _Type* __ptr, - __unv<_Type>& __dst, - memory_order __order, - _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {return;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{ - __backend, __ptr_proxy, __dst_proxy}; - __cuda_atomic_load_order_dispatch( - __backend, __bound_load, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> -__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) -{ - __unv<_Type> __dst; - ::cuda::std::__cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); - return __dst; -} - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_store( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type> __val, __cuda_atomic_ptx_order_release __order, __cuda_atomic_operand_b8, __thread_scope_block_tag, __cuda_atomic_mmio_disable) @@ -1822,44 +1763,6 @@ _CCCL_DEVICE_API void __cuda_atomic_store( )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_store { - _Backend __backend; - _Type* __ptr; - __unv<_Type> __val; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) { - ::cuda::std::__cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); - } -}; -template -_CCCL_HOST_DEVICE_API void -__cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __value_type __store = __val; - __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {return;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{ - __backend, __ptr_proxy, *__val_proxy}; - __cuda_atomic_store_order_dispatch( - __backend, __bound_store, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); -} - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) @@ -2421,55 +2324,6 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( } )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_compare_exchange { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __exp; - __unv<_Type> __cmp; - __unv<_Type> __des; - - template - [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) { - return ::cuda::std::__cuda_atomic_compare_exchange( - __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( - _Backend __backend, - _Type* __ptr, - __unv<_Type>* __exp, - __unv<_Type> __des, - _Cas, - memory_order __success, - memory_order __failure, - _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); - __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); -#if _CCCL_CUDA_COMPILATION() - bool __res = false; - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ - __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; - return __cuda_atomic_compare_exchange_order_dispatch( - __backend, __bound_compare_swap, __success, __failure, __scope, _Cas{}, __proxy_tag{}); -} - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_exchange( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) @@ -3031,60 +2885,6 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_exchange { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __old; - __unv<_Type> __new; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); - } -}; -template -_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( - _Backend __backend, - _Type* __ptr, - __unv<_Type>& __old, - __unv<_Type> __new, - memory_order __order, - _Sco __scope) -{ - using __value_type _CCCL_NODEBUG = __unv<_Type>; - using __proxy_t _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee _CCCL_NODEBUG = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); - __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if(::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {return;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{ - __backend, __ptr_proxy, __old_proxy, *__new_proxy}; - __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); -} - -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_exchange_dispatch( - _Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) -{ - using __value_type _CCCL_NODEBUG = __unv<_Type>; - __value_type __old; - ::cuda::std::__cuda_atomic_exchange_dispatch( - __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); - return __old; -} - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_f32, __thread_scope_block_tag) @@ -3486,51 +3286,6 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_add( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s32, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.add.sys.s32 %0,[%1],%2;" : "=r"(__dst) : "l"(__ptr), "r"(__op) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_fetch_add { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __dst; - __unv<_Type> __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_fetch_add(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_add_dispatch( - _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - __op = __op * __atomic_ptr_skip_t<_Type>::__skip; - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_arithmetic_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<__value_type>; - __value_type __dst{}; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_fetch_add<_Backend, __proxy_pointee> __bound_add{ - __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope, __proxy_tag{}); - return __dst; -} - - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) @@ -3692,50 +3447,6 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_and( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.and.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_fetch_and { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __dst; - __unv<_Type> __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_fetch_and(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_and_dispatch( - _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __value_type __dst{}; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_fetch_and<_Backend, __proxy_pointee> __bound_and{ - __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope, __proxy_tag{}); - return __dst; -} - - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) @@ -4057,50 +3768,6 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_max( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.max.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_fetch_max { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __dst; - __unv<_Type> __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_fetch_max(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_max_dispatch( - _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; - __value_type __dst{}; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_fetch_max<_Backend, __proxy_pointee> __bound_max{ - __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope, __proxy_tag{}); - return __dst; -} - - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_u32, __thread_scope_block_tag) @@ -4422,50 +4089,6 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_min( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_s64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.min.sys.s64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_fetch_min { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __dst; - __unv<_Type> __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_fetch_min(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_min_dispatch( - _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; - __value_type __dst{}; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_fetch_min<_Backend, __proxy_pointee> __bound_min{ - __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope, __proxy_tag{}); - return __dst; -} - - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) @@ -4627,50 +4250,6 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_or( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_order_volatile __order, __cuda_atomic_operand_b64, __thread_scope_system_tag) { ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, __thread_scope_system_tag{}); asm volatile("atom.or.sys.b64 %0,[%1],%2;" : "=l"(__dst) : "l"(__ptr), "l"(__op) : "memory"); } -#endif // _CCCL_CUDA_COMPILATION() - -template -struct __cuda_atomic_bind_fetch_or { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __dst; - __unv<_Type> __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_fetch_or(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_or_dispatch( - _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __value_type __dst{}; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_fetch_or<_Backend, __proxy_pointee> __bound_or{ - __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope, __proxy_tag{}); - return __dst; -} - - -#if _CCCL_CUDA_COMPILATION() - template _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( __cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, __cuda_atomic_ptx_order_acquire __order, __cuda_atomic_operand_b32, __thread_scope_block_tag) @@ -4834,46 +4413,6 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_xor( #endif // _CCCL_CUDA_COMPILATION() -template -struct __cuda_atomic_bind_fetch_xor { - _Backend __backend; - _Type* __ptr; - __unv<_Type>* __dst; - __unv<_Type> __op; - - template - _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) { - ::cuda::std::__cuda_atomic_fetch_xor(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); - } -}; -template -[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> __cuda_atomic_fetch_xor_dispatch( - _Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) -{ - using __value_type = __unv<_Type>; - using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; - using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; - using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; - __value_type __dst{}; - __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); - __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); - __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); -#if _CCCL_CUDA_COMPILATION() - if constexpr (_Backend::__requires_local_memory_workaround) - { - if (::cuda::std::__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {return __dst;} - } -#endif // _CCCL_CUDA_COMPILATION() - __cuda_atomic_bind_fetch_xor<_Backend, __proxy_pointee> __bound_xor{ - __backend, __ptr_proxy, __dst_proxy, *__op_proxy}; - __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope, __proxy_tag{}); - return __dst; -} - - _CCCL_END_NAMESPACE_CUDA_STD #include diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index c41a51379f8c..7d97c79be54e 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -216,6 +216,430 @@ template } } +template +struct __cuda_atomic_bind_load +{ + _Backend __backend; + const _Type* __ptr; + __unv<_Type>* __dst; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) + { + ::cuda::std::__cuda_atomic_load(__backend, __ptr, *__dst, __order, _Tag{}, _Sco{}, _Mmio{}); + } +}; +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_load_dispatch( + _Backend __backend, const _Type* __ptr, __unv<_Type>& __dst, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + const __proxy_pointee* __ptr_proxy = reinterpret_cast(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) + { + return; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_load<_Backend, __proxy_pointee> __bound_load{__backend, __ptr_proxy, __dst_proxy}; + __cuda_atomic_load_order_dispatch( + __backend, __bound_load, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_load_dispatch(_Backend __backend, const _Type* __ptr, memory_order __order, _Sco __scope) +{ + __unv<_Type> __dst; + ::cuda::std::__cuda_atomic_load_dispatch(__backend, __ptr, __dst, __order, __scope); + return __dst; +} + +template +struct __cuda_atomic_bind_store +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type> __val; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Mmio, _Sco) + { + ::cuda::std::__cuda_atomic_store(__backend, __ptr, __val, __order, _Tag{}, _Sco{}, _Mmio{}); + } +}; +template +_CCCL_HOST_DEVICE_API void +__cuda_atomic_store_dispatch(_Backend __backend, _Type* __ptr, _Up __val, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __value_type __store = __val; + __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__store); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) + { + return; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_store<_Backend, __proxy_pointee> __bound_store{__backend, __ptr_proxy, *__val_proxy}; + __cuda_atomic_store_order_dispatch( + __backend, __bound_store, __order, __scope, __proxy_tag{}, __cuda_atomic_mmio_disable{}); +} + +template +struct __cuda_atomic_bind_compare_exchange +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __exp; + __unv<_Type> __cmp; + __unv<_Type> __des; + + template + [[nodiscard]] _CCCL_HOST_DEVICE_API bool operator()(_Atomic_Memorder __order, _Cas, _Tag, _Sco) + { + return ::cuda::std::__cuda_atomic_compare_exchange( + __backend, __ptr, *__exp, __cmp, __des, _Cas{}, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch( + _Backend __backend, + _Type* __ptr, + __unv<_Type>* __exp, + __unv<_Type> __des, + _Cas, + memory_order __success, + memory_order __failure, + _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp); + __proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des); +#if _CCCL_CUDA_COMPILATION() + bool __res = false; + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) + { + return __res; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_compare_exchange<_Backend, __proxy_pointee> __bound_compare_swap{ + __backend, __ptr_proxy, __exp_proxy, *__exp_proxy, *__des_proxy}; + return __cuda_atomic_compare_exchange_order_dispatch( + __backend, __bound_compare_swap, __success, __failure, __scope, _Cas{}, __proxy_tag{}); +} + +template +struct __cuda_atomic_bind_exchange +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __old; + __unv<_Type> __new; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_exchange(__backend, __ptr, *__old, __new, __order, _Tag{}, _Sco{}); + } +}; +template +_CCCL_HOST_DEVICE_API void __cuda_atomic_exchange_dispatch( + _Backend __backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, memory_order __order, _Sco __scope) +{ + using __value_type _CCCL_NODEBUG = __unv<_Type>; + using __proxy_t _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee _CCCL_NODEBUG = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag _CCCL_NODEBUG = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old); + __proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) + { + return; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_exchange<_Backend, __proxy_pointee> __bound_swap{__backend, __ptr_proxy, __old_proxy, *__new_proxy}; + __cuda_atomic_exchange_order_dispatch(__backend, __bound_swap, __order, __scope, __proxy_tag{}); +} + +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_exchange_dispatch(_Backend __backend, _Type* __ptr, _Up __new, memory_order __order, _Sco __scope) +{ + using __value_type _CCCL_NODEBUG = __unv<_Type>; + __value_type __old; + ::cuda::std::__cuda_atomic_exchange_dispatch( + __backend, __ptr, __old, static_cast<__value_type>(__new), __order, __scope); + return __old; +} + +template +struct __cuda_atomic_bind_fetch_add +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_add(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_add_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + __op = __op * __atomic_ptr_skip_t<_Type>::__skip; + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_arithmetic_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_arithmetic_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_add_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_add<_Backend, __proxy_pointee> __bound_add{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_add, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_and +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_and(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_and_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_and_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_and<_Backend, __proxy_pointee> __bound_and{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_and, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_max +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_max(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_max_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_max_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_max<_Backend, __proxy_pointee> __bound_max{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_max, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_min +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_min(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_min_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_minmax_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_minmax_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_min_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_min<_Backend, __proxy_pointee> __bound_min{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_min, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_or +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_or(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_or_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_or_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_or<_Backend, __proxy_pointee> __bound_or{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_or, __order, __scope, __proxy_tag{}); + return __dst; +} + +template +struct __cuda_atomic_bind_fetch_xor +{ + _Backend __backend; + _Type* __ptr; + __unv<_Type>* __dst; + __unv<_Type> __op; + + template + _CCCL_HOST_DEVICE_API void operator()(_Atomic_Memorder __order, _Tag, _Sco) + { + ::cuda::std::__cuda_atomic_fetch_xor(__backend, __ptr, *__dst, __op, __order, _Tag{}, _Sco{}); + } +}; +template +[[nodiscard]] _CCCL_HOST_DEVICE_API __unv<_Type> +__cuda_atomic_fetch_xor_dispatch(_Backend __backend, _Type* __ptr, _Up __op, memory_order __order, _Sco __scope) +{ + using __value_type = __unv<_Type>; + using __proxy_t = __cuda_atomic_deduce_bitwise_t<__value_type>; + using __proxy_pointee = __copy_cv_t<_Type, __proxy_t>; + using __proxy_tag = __cuda_atomic_deduce_bitwise_tag_t<__value_type>; + __value_type __dst{}; + __proxy_pointee* __ptr_proxy = reinterpret_cast<__proxy_pointee*>(__ptr); + __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst); + __proxy_t* __op_proxy = reinterpret_cast<__proxy_t*>(&__op); +#if _CCCL_CUDA_COMPILATION() + if constexpr (_Backend::__requires_local_memory_workaround) + { + if (::cuda::std::__cuda_atomic_fetch_xor_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) + { + return __dst; + } + } +#endif // _CCCL_CUDA_COMPILATION() + __cuda_atomic_bind_fetch_xor<_Backend, __proxy_pointee> __bound_xor{__backend, __ptr_proxy, __dst_proxy, *__op_proxy}; + __cuda_atomic_fetch_order_dispatch(__backend, __bound_xor, __order, __scope, __proxy_tag{}); + return __dst; +} + template struct __cuda_atomic_bind_fetch_sub { From eb3a1c3a840a98adc6f27ed97afcee9e167f06de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Fri, 28 Aug 2026 17:06:00 -0700 Subject: [PATCH 17/24] Move the signal fence into the PTX backend. --- libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h | 7 +++++++ libcudacxx/include/cuda/std/__atomic/functions/dispatch.h | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h index 783cd9d3a4b5..a1655e596092 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_ptx.h @@ -136,6 +136,13 @@ _CCCL_DEVICE_API void __cuda_atomic_fetch_sub( __backend, __ptr, __dst, ::cuda::std::__cuda_atomic_ptx_negate(__op), __order, _Operand{}, __scope); } +#if _CCCL_CUDA_COMPILATION() +_CCCL_DEVICE_API inline void __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend, memory_order) +{ + asm volatile("" ::: "memory"); +} +#endif // _CCCL_CUDA_COMPILATION() + _CCCL_END_NAMESPACE_CUDA_STD #include diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index 7d97c79be54e..c43e3bab57f5 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -684,13 +684,6 @@ __cuda_atomic_fetch_sub_dispatch(_Backend __backend, _Type* __ptr, _Up __op, mem return __dst; } -#if _CCCL_CUDA_COMPILATION() -_CCCL_DEVICE_API inline void __cuda_atomic_signal_fence(__cuda_atomic_ptx_backend, memory_order) -{ - asm volatile("" ::: "memory"); -} -#endif // _CCCL_CUDA_COMPILATION() - _CCCL_END_NAMESPACE_CUDA_STD #include From ed4d0d54e4472f88643e8466a78b7106d808de94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Tue, 25 Aug 2026 22:06:35 -0700 Subject: [PATCH 18/24] Add an NVVM atomic backend. --- libcudacxx/include/cuda/__atomic/atomic.h | 6 +- .../cuda/std/__atomic/functions/cuda_nvvm.h | 382 ++++++++++++++++++ .../__atomic/functions/cuda_nvvm_backend.h | 86 ++++ .../std/__atomic/functions/device_backend.h | 44 ++ .../cuda/std/__atomic/functions/dispatch.h | 2 +- .../include/cuda/std/__atomic/types/base.h | 16 +- 6 files changed, 524 insertions(+), 12 deletions(-) create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm_backend.h create mode 100644 libcudacxx/include/cuda/std/__atomic/functions/device_backend.h diff --git a/libcudacxx/include/cuda/__atomic/atomic.h b/libcudacxx/include/cuda/__atomic/atomic.h index 9188122ea2e1..c5ecc576134d 100644 --- a/libcudacxx/include/cuda/__atomic/atomic.h +++ b/libcudacxx/include/cuda/__atomic/atomic.h @@ -145,15 +145,15 @@ atomic_thread_fence(memory_order __m, [[maybe_unused]] thread_scope _Scope = thr (switch (_Scope) { case thread_scope::thread_scope_system: ::cuda::std::__cuda_atomic_thread_fence( - ::cuda::std::__cuda_atomic_ptx_backend{}, __m, __thread_scope_system_tag{}); + ::cuda::std::__cuda_atomic_device_backend{}, __m, __thread_scope_system_tag{}); break; case thread_scope::thread_scope_device: ::cuda::std::__cuda_atomic_thread_fence( - ::cuda::std::__cuda_atomic_ptx_backend{}, __m, __thread_scope_device_tag{}); + ::cuda::std::__cuda_atomic_device_backend{}, __m, __thread_scope_device_tag{}); break; case thread_scope::thread_scope_block: ::cuda::std::__cuda_atomic_thread_fence( - ::cuda::std::__cuda_atomic_ptx_backend{}, __m, __thread_scope_block_tag{}); + ::cuda::std::__cuda_atomic_device_backend{}, __m, __thread_scope_block_tag{}); break; // Atomics scoped to themselves do not require fencing case thread_scope::thread_scope_thread: diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h new file mode 100644 index 000000000000..c7ef7e4f153a --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h @@ -0,0 +1,382 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() + +extern "C" _CCCL_DEVICE void __cuda_atomic_nvvm_cas_128b_unsupported_before_SM_90(); +extern "C" _CCCL_DEVICE void __cuda_atomic_nvvm_exchange_128b_unsupported_before_SM_90(); + +template +struct __cuda_atomic_nvvm_order; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_relaxed> +{ + static constexpr int __value = __NV_ATOMIC_RELAXED; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_release> +{ + static constexpr int __value = __NV_ATOMIC_RELEASE; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_acquire> +{ + static constexpr int __value = __NV_ATOMIC_ACQUIRE; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_acq_rel> +{ + static constexpr int __value = __NV_ATOMIC_ACQ_REL; +}; + +template <> +struct __cuda_atomic_nvvm_order<__cuda_atomic_order_seq_cst> +{ + static constexpr int __value = __NV_ATOMIC_SEQ_CST; +}; + +template +struct __cuda_atomic_nvvm_scope; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_block_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_BLOCK; +}; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_cluster_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_CLUSTER; +}; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_device_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_DEVICE; +}; + +template <> +struct __cuda_atomic_nvvm_scope<__thread_scope_system_tag> +{ + static constexpr int __value = __NV_THREAD_SCOPE_SYSTEM; +}; + +template +[[nodiscard]] _CCCL_DEVICE_API __unv<_Type>* __cuda_atomic_nvvm_ptr(_Type* __ptr) +{ + return const_cast<__unv<_Type>*>(__ptr); +} + +template +struct __cuda_atomic_nvvm_failure_order +{ + using type = _Order; +}; + +template <> +struct __cuda_atomic_nvvm_failure_order<__cuda_atomic_order_release> +{ + using type = __cuda_atomic_order_relaxed; +}; + +template <> +struct __cuda_atomic_nvvm_failure_order<__cuda_atomic_order_acq_rel> +{ + using type = __cuda_atomic_order_acquire; +}; + +template +struct __cuda_atomic_nvvm_cas_orders +{ + using __success = _Order; + using __failure = typename __cuda_atomic_nvvm_failure_order<_Order>::type; +}; + +template +struct __cuda_atomic_nvvm_cas_orders<__cuda_atomic_cas_order<_Success, _Failure>> +{ + using __success = _Success; + using __failure = _Failure; +}; + +template +_CCCL_DEVICE_API void __cuda_atomic_load( + __cuda_atomic_nvvm_backend, + const _Type* __ptr, + __unv<_Type>& __dst, + _Order, + _Operand, + _Scope __scope, + __cuda_atomic_mmio_disable) +{ + ::__nv_atomic_load(__cuda_atomic_nvvm_ptr(__ptr), + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); +} + +template +_CCCL_DEVICE_API void __cuda_atomic_store( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type> __val, + _Order, + _Operand, + _Scope __scope, + __cuda_atomic_mmio_disable) +{ + ::__nv_atomic_store(__cuda_atomic_nvvm_ptr(__ptr), + &__val, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); +} + +template = 1000), bool> = false> +[[nodiscard]] _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __cmp, + __unv<_Type> __op, + _Cas, + _Order, + _Operand, + _Scope __scope) +{ + using __orders = __cuda_atomic_nvvm_cas_orders<_Order>; + using __success = typename __orders::__success; + using __failure = typename __orders::__failure; + __dst = __cmp; + + if constexpr (_Operand::__size == 128) + { + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_90, + (return ::__nv_atomic_compare_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__dst, + &__op, + __cuda_atomic_cas_is_weak(_Cas{}), + +__cuda_atomic_nvvm_order<__success>::__value, + +__cuda_atomic_nvvm_order<__failure>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value);), + (__cuda_atomic_nvvm_cas_128b_unsupported_before_SM_90(); return false;)) + } + else + { + return ::__nv_atomic_compare_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__dst, + &__op, + __cuda_atomic_cas_is_weak(_Cas{}), + +__cuda_atomic_nvvm_order<__success>::__value, + +__cuda_atomic_nvvm_order<__failure>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); + } +} + +template +_CCCL_DEVICE_API void __cuda_atomic_exchange( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + _Order __order, + _Operand, + _Scope __scope) +{ + if constexpr (_Operand::__size < 32) + { + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_100, + (::__nv_atomic_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__op, + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value);), + (__dst = __cuda_atomic_fetch_update( + __cuda_atomic_nvvm_backend{}, + __ptr, + __cuda_atomic_op_bind<__unv<_Type>, __cuda_atomic_op_store>{__op}, + __order, + _Operand{}, + __scope);)) + } + else if constexpr (_Operand::__size == 128) + { + NV_IF_ELSE_TARGET( + NV_PROVIDES_SM_90, + (::__nv_atomic_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__op, + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value);), + (__cuda_atomic_nvvm_exchange_128b_unsupported_before_SM_90();)) + } + else + { + ::__nv_atomic_exchange( + __cuda_atomic_nvvm_ptr(__ptr), + &__op, + &__dst, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); + } +} + +# define _CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(_Name) \ + template = 32) || (_CCCL_PTX_ARCH() >= 1000)) \ + && !(is_integral_v<_Type> && is_signed_v<_Type> && sizeof(_Type) == 8), \ + bool> = false> \ + _CCCL_DEVICE_API void __cuda_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order, _Operand, _Scope) \ + { \ + __dst = ::__nv_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_ptr(__ptr), \ + __op, \ + +__cuda_atomic_nvvm_order<_Order>::__value, \ + +__cuda_atomic_nvvm_scope<_Scope>::__value); \ + } \ + \ + template && is_signed_v<_Type> && sizeof(_Type) == 8 && _Operand::__size == 64, \ + bool> = false> \ + _CCCL_DEVICE_API void __cuda_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order, _Operand, _Scope) \ + { \ + const auto __result = ::__nv_atomic_fetch_##_Name( \ + reinterpret_cast(__cuda_atomic_nvvm_ptr(__ptr)), \ + ::cuda::std::bit_cast(__op), \ + +__cuda_atomic_nvvm_order<_Order>::__value, \ + +__cuda_atomic_nvvm_scope<_Scope>::__value); \ + __dst = ::cuda::std::bit_cast<__unv<_Type>>(__result); \ + } + +# define _CCCL_DEFINE_NVVM_FETCH_OP(_Name, _TypeConstraint) \ + template = 32) || (_CCCL_PTX_ARCH() >= 1000)) \ + && (_TypeConstraint), \ + bool> = false> \ + _CCCL_DEVICE_API void __cuda_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __op, _Order, _Operand, _Scope) \ + { \ + __dst = ::__nv_atomic_fetch_##_Name( \ + __cuda_atomic_nvvm_ptr(__ptr), \ + __op, \ + +__cuda_atomic_nvvm_order<_Order>::__value, \ + +__cuda_atomic_nvvm_scope<_Scope>::__value); \ + } + +_CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(add) +_CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(sub) +_CCCL_DEFINE_NVVM_FETCH_OP(and, true) +_CCCL_DEFINE_NVVM_FETCH_OP(or, true) +_CCCL_DEFINE_NVVM_FETCH_OP(xor, true) +_CCCL_DEFINE_NVVM_FETCH_OP(min, is_integral_v<_Type>) +_CCCL_DEFINE_NVVM_FETCH_OP(max, is_integral_v<_Type>) + +# undef _CCCL_DEFINE_NVVM_FETCH_ARITHMETIC +# undef _CCCL_DEFINE_NVVM_FETCH_OP + +template +struct __cuda_atomic_nvvm_fence +{ + template + _CCCL_DEVICE_API void operator()(_Order) const + { + ::__nv_atomic_thread_fence(+__cuda_atomic_nvvm_order<_Order>::__value, +__cuda_atomic_nvvm_scope<_Scope>::__value); + } +}; + +template +_CCCL_DEVICE_API void +__cuda_atomic_thread_fence(__cuda_atomic_nvvm_backend __backend, memory_order __order, _Scope __scope) +{ + (void) __backend; + (void) __scope; + __cuda_atomic_nvvm_fence<_Scope> __fence; + switch (__atomic_order_to_int(__order)) + { + case __ATOMIC_RELAXED: + return; + case __ATOMIC_CONSUME: + [[fallthrough]]; + case __ATOMIC_ACQUIRE: + return __fence(__cuda_atomic_order_acquire{}); + case __ATOMIC_RELEASE: + return __fence(__cuda_atomic_order_release{}); + case __ATOMIC_ACQ_REL: + return __fence(__cuda_atomic_order_acq_rel{}); + case __ATOMIC_SEQ_CST: + return __fence(__cuda_atomic_order_seq_cst{}); + default: + _CCCL_ASSERT(false, "invalid fence memory order"); + } +} + +_CCCL_DEVICE_API void __cuda_atomic_signal_fence(__cuda_atomic_nvvm_backend, memory_order) +{ + asm volatile("" ::: "memory"); +} + +#endif // _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm_backend.h new file mode 100644 index 000000000000..e9c9bed3a5d8 --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm_backend.h @@ -0,0 +1,86 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include +#include +#include +#include + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +struct __cuda_atomic_nvvm_backend +{ +private: + template + static constexpr bool __has_builtin_subword_rmw = sizeof(_Type) >= 4 || _CCCL_PTX_ARCH() >= 1000; + +public: + template + _CCCL_HOST_DEVICE_API static constexpr _Order __collapse_cas_order(_Order __order) + { + return __order; + } + + template + _CCCL_DEVICE_API static auto + __with_transformed_order(_Operation, _Fn& __fn, _Order __order, _Sco __scope, _Args... __args) + -> decltype(__fn(__order, __args..., __scope)) + { + return __fn(__order, __args..., __scope); + } + + template + static constexpr bool __use_direct_bitwise = sizeof(_Type) < 16 && __has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_direct_arithmetic = + is_scalar_v<_Type> && sizeof(_Type) < 16 && __has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_direct_minmax = + is_integral_v<_Type> && sizeof(_Type) < 16 && __has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_fallback_bitwise = sizeof(_Type) == 16 || !__has_builtin_subword_rmw<_Type>; + + template + static constexpr bool __use_fallback_arithmetic = + is_scalar_v<_Type> && (sizeof(_Type) == 16 || !__has_builtin_subword_rmw<_Type>); + + template + static constexpr bool __use_fallback_minmax = + !is_integral_v<_Type> || (is_scalar_v<_Type> && (sizeof(_Type) == 16 || !__has_builtin_subword_rmw<_Type>) ); + + static constexpr bool __needs_constant_order = true; + static constexpr bool __requires_local_memory_workaround = true; + static constexpr size_t __smallest_cas = 32; + static constexpr size_t __widest_cas = 128; +}; + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_NVVM_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/device_backend.h b/libcudacxx/include/cuda/std/__atomic/functions/device_backend.h new file mode 100644 index 000000000000..07bf2bbcb01e --- /dev/null +++ b/libcudacxx/include/cuda/std/__atomic/functions/device_backend.h @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// +// Part of libcu++, the C++ Standard Library for your entire system, +// under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +// +//===----------------------------------------------------------------------===// + +#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_DEVICE_BACKEND_H +#define _CUDA_STD___ATOMIC_FUNCTIONS_DEVICE_BACKEND_H + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#if _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() +# include +#else +# include +#endif + +#include + +_CCCL_BEGIN_NAMESPACE_CUDA_STD + +#if _CCCL_CTK_AT_LEAST(13, 5) && _CCCL_HAS_NV_ATOMIC_BUILTINS() +using __cuda_atomic_device_backend = __cuda_atomic_nvvm_backend; +#else +using __cuda_atomic_device_backend = __cuda_atomic_ptx_backend; +#endif + +_CCCL_END_NAMESPACE_CUDA_STD + +#include + +#endif // _CUDA_STD___ATOMIC_FUNCTIONS_DEVICE_BACKEND_H diff --git a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h index c43e3bab57f5..fdd50df8b4ec 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/dispatch.h @@ -23,7 +23,7 @@ #include #include -#include +#include #include #include #include diff --git a/libcudacxx/include/cuda/std/__atomic/types/base.h b/libcudacxx/include/cuda/std/__atomic/types/base.h index 17259c9014a6..f01b62b96938 100644 --- a/libcudacxx/include/cuda/std/__atomic/types/base.h +++ b/libcudacxx/include/cuda/std/__atomic/types/base.h @@ -69,16 +69,16 @@ struct __atomic_storage } }; -#define _CCCL_DISPATCH_ATOMIC_BACKEND(_Fn, ...) \ - NV_DISPATCH_TARGET(NV_IS_DEVICE, \ - (return _Fn(__cuda_atomic_ptx_backend{}, __VA_ARGS__);), \ - NV_IS_HOST, \ +#define _CCCL_DISPATCH_ATOMIC_BACKEND(_Fn, ...) \ + NV_DISPATCH_TARGET(NV_IS_DEVICE, \ + (return _Fn(__cuda_atomic_device_backend{}, __VA_ARGS__);), \ + NV_IS_HOST, \ (return _Fn(__cuda_atomic_host_backend{}, __VA_ARGS__);)) -#define _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(_Fn, _Scope, ...) \ - NV_DISPATCH_TARGET(NV_IS_DEVICE, \ - (return _Fn(__cuda_atomic_ptx_backend{}, __VA_ARGS__, _Scope);), \ - NV_IS_HOST, \ +#define _CCCL_DISPATCH_SCOPED_ATOMIC_BACKEND(_Fn, _Scope, ...) \ + NV_DISPATCH_TARGET(NV_IS_DEVICE, \ + (return _Fn(__cuda_atomic_device_backend{}, __VA_ARGS__, _Scope);), \ + NV_IS_HOST, \ (return _Fn(__cuda_atomic_host_backend{}, __VA_ARGS__, __thread_scope_tag{});)) _CCCL_HOST_DEVICE_API inline void __atomic_thread_fence_dispatch(memory_order __order) From 6980c89c5fbe2ebd9eeeb8dcb2faed0b5280a6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Mon, 31 Aug 2026 11:11:13 -0700 Subject: [PATCH 19/24] Use native addition for floating-point NVVM fetch-sub. --- .../cuda/std/__atomic/functions/cuda_nvvm.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h index c7ef7e4f153a..22c03a0f37b1 100644 --- a/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h +++ b/libcudacxx/include/cuda/std/__atomic/functions/cuda_nvvm.h @@ -323,6 +323,24 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange( _CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(add) _CCCL_DEFINE_NVVM_FETCH_ARITHMETIC(sub) + +template = false> +_CCCL_DEVICE_API void __cuda_atomic_fetch_sub( + __cuda_atomic_nvvm_backend, + _Type* __ptr, + __unv<_Type>& __dst, + __unv<_Type> __op, + _Order, + __cuda_atomic_operand_tag<__cuda_atomic_operand::_f, _Size>, + _Scope) +{ + __dst = ::__nv_atomic_fetch_add( + __cuda_atomic_nvvm_ptr(__ptr), + -__op, + +__cuda_atomic_nvvm_order<_Order>::__value, + +__cuda_atomic_nvvm_scope<_Scope>::__value); +} + _CCCL_DEFINE_NVVM_FETCH_OP(and, true) _CCCL_DEFINE_NVVM_FETCH_OP(or, true) _CCCL_DEFINE_NVVM_FETCH_OP(xor, true) From 4f1b2cfaf7976efc61052b09e1be3692eeea0ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Mon, 31 Aug 2026 11:29:10 -0700 Subject: [PATCH 20/24] Account for NVVM release-fence placement in SASS tests. --- .../test/atomic_codegen/sass/CMakeLists.txt | 25 ++++- .../sass/arithmetic_types_8_16_atomic_ref.cu | 48 +++++++++- .../compare_exchange_types_8_16_atomic_ref.cu | 95 +++++++++++++++++-- libcudacxx/test/cmake/CodegenTest.cmake | 10 +- 4 files changed, 163 insertions(+), 15 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt b/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt index df44cd85bc05..ff74c152ab89 100644 --- a/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt +++ b/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt @@ -61,8 +61,29 @@ check_source_compiles( ]] libcudacxx_atomic_codegen_has_int128 ) +check_source_compiles( + CUDA + [[ + #include + + #if defined(__CUDA_ARCH__) \ + && (!_CCCL_CTK_AT_LEAST(13, 5) || !_CCCL_HAS_NV_ATOMIC_BUILTINS()) + # error "the NVVM atomic backend is unavailable" + #endif + + __global__ void test() {} + int main() { test<<<1, 1>>>(); } + ]] + libcudacxx_atomic_codegen_uses_nvvm_backend +) cmake_pop_check_state() +if (libcudacxx_atomic_codegen_uses_nvvm_backend) + set(atomic_codegen_sass_backend_prefix NVVM) +else() + set(atomic_codegen_sass_backend_prefix PTX) +endif() + if (NOT libcudacxx_atomic_codegen_has_int128) message( STATUS @@ -88,7 +109,9 @@ libcudacxx_codegen_add_sass_tests( TARGET_PREFIX atomic_codegen ARCHITECTURES ${atomic_codegen_sass_cuda_archs} DUMP_FUNCTIONS atomic_codegen_test - CHECK_PREFIXES ${atomic_codegen_sass_cuda_version_prefix} + CHECK_PREFIXES + ${atomic_codegen_sass_cuda_version_prefix} + ${atomic_codegen_sass_backend_prefix} TESTS ${libcudacxx_atomic_codegen_tests} COMPILE_DEFINITIONS _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1 ) diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu index 6cd682a07336..65f59fb95bf8 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_8_16_atomic_ref.cu @@ -13,8 +13,28 @@ // %PARAM% TYPE type int8_t:uint8_t:int16_t:uint16_t // %PARAM% OP,FILECHECK_PREFIX_OP op add=fetch_add,add:sub=fetch_sub,sub:min=fetch_min,min:max=fetch_max,max // %PARAM% ORDER,FILECHECK_PREFIX_SEQ_CST,FILECHECK_PREFIX_ACQUIRE,FILECHECK_PREFIX_ORDER order relaxed=mor,non_seq_cst,no_acquire,no_membar:acquire=moa,non_seq_cst,acquire,no_membar:release=more,non_seq_cst,no_acquire,release:acq_rel=moar,non_seq_cst,acquire,release:seq_cst=mosc,seq_cst,acquire,seq_cst -// %FILECHECK% PREFIX_COMBINE non_block,seq_cst // %FILECHECK% PREFIX_COMBINE non_block,acquire +// %FILECHECK% PREFIX_COMBINE ptx,seq_cst +// %FILECHECK% PREFIX_COMBINE ptx,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE ptx,release +// %FILECHECK% PREFIX_COMBINE sm75,nvvm +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm80,nvvm +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm90,nvvm +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,release // clang-format on #include "atomic_codegen_helpers.h" @@ -30,6 +50,12 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SMXX-LABEL: {{[[:space:]]*}}Function : atomic_codegen_test ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; SMXX: {{.*}}LD.E.64{{(\.SYS)?}} [[ATOM_ADDR:R[0-9]+]], {{.*}} +; SM100_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM120_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} @@ -43,16 +69,28 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SUB: {{.*}}{{IADD3|IADD|IMAD\.IADD|PRMT|LOP3\.LUT}} {{R[0-9]+}}, {{.*}}[[SUB_VALUE]]{{.*}} ; MIN: {{.*}}IMNMX{{.*}} ; MAX: {{.*}}IMNMX{{.*}} -; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} -; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; PTX_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; PTX_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; PTX_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM80_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM90_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} ; NON_BLOCK: {{.*}}ATOM.E.CAS.STRONG.[[SASS_SCOPE]] PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]], [[EXPECTED]], {{.*}} +; PTX-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]]{{(\.reuse)?}}, [[EXPECTED]]{{(\.reuse)?}}, {{.*}} +; SM75_NVVM-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]]{{(\.reuse)?}}, [[OLD]]{{(\.reuse)?}}, {{.*}} +; SM80_NVVM-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]]{{(\.reuse)?}}, [[OLD]]{{(\.reuse)?}}, {{.*}} +; SM90_NVVM-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]]{{(\.reuse)?}}, [[OLD]]{{(\.reuse)?}}, {{.*}} ; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/compare_exchange_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/compare_exchange_types_8_16_atomic_ref.cu index 129e5cae59b9..f2dc1f01278c 100644 --- a/libcudacxx/test/atomic_codegen/sass/compare_exchange_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/compare_exchange_types_8_16_atomic_ref.cu @@ -10,11 +10,51 @@ // clang-format off // %PARAM% SCOPE,SASS_SCOPE,FILECHECK_PREFIX_SCOPE scope block=tsb,CTA,block:device=tsd,GPU,non_block:system=tss,SYS,non_block -// %PARAM% TYPE type int8_t:uint8_t:int16_t:uint16_t:f16:bf16 +// %PARAM% TYPE,FILECHECK_PREFIX_WIDTH type int8_t=int8_t,byte:uint8_t=uint8_t,byte:int16_t=int16_t,halfword:uint16_t=uint16_t,halfword:f16=f16,halfword:bf16=bf16,halfword // %PARAM% CAS cas compare_exchange_weak:compare_exchange_strong // %PARAM% SUCCESS_ORDER,FAILURE_ORDER,FILECHECK_PREFIX_SEQ_CST,FILECHECK_PREFIX_ACQUIRE,FILECHECK_PREFIX_ORDER order rr=mor,mor,non_seq_cst,no_acquire,no_membar:ar=moa,mor,non_seq_cst,acquire,no_membar:aa=moa,moa,non_seq_cst,acquire,no_membar:er=more,mor,non_seq_cst,no_acquire,release:br=moar,mor,non_seq_cst,acquire,release:ba=moar,moa,non_seq_cst,acquire,release:sr=mosc,mor,seq_cst,acquire,seq_cst:sa=mosc,moa,seq_cst,acquire,seq_cst:ss=mosc,mosc,seq_cst,acquire,seq_cst -// %FILECHECK% PREFIX_COMBINE non_block,seq_cst // %FILECHECK% PREFIX_COMBINE non_block,acquire +// %FILECHECK% PREFIX_COMBINE nvvm,byte +// %FILECHECK% PREFIX_COMBINE nvvm,byte,block +// %FILECHECK% PREFIX_COMBINE nvvm,byte,non_block +// %FILECHECK% PREFIX_COMBINE ptx,block +// %FILECHECK% PREFIX_COMBINE ptx,non_block +// %FILECHECK% PREFIX_COMBINE ptx,seq_cst +// %FILECHECK% PREFIX_COMBINE ptx,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE ptx,release +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,halfword +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,halfword,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,halfword,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,halfword,release +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,byte,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,byte,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm75,nvvm,byte,release +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,halfword +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,halfword,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,halfword,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,halfword,release +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,byte,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,byte,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm80,nvvm,byte,release +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,halfword +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,halfword,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,halfword,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,halfword,release +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,byte,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,byte,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm90,nvvm,byte,release +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,halfword +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,halfword,block +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,halfword,non_block +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm100,nvvm,release +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,halfword +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,halfword,block +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,halfword,non_block +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,seq_cst +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE sm120,nvvm,release // clang-format on #include @@ -32,16 +72,55 @@ extern "C" __device__ bool atomic_codegen_test(cuda::atomic_ref& at ; SMXX-LABEL: {{[[:space:]]*}}Function : atomic_codegen_test ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} ; SMXX: {{.*}}LD.E.64{{(\.SYS)?}} [[ATOM_ADDR:R[0-9]+]], {{.*}} +; SM100_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM120_NVVM_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM100_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM120_NVVM_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_HALFWORD_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_HALFWORD_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_HALFWORD_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_HALFWORD_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; PTX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; NVVM_BYTE-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; SM75_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffd, {{.*}} +; SM80_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffd, {{.*}} +; SM90_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffd, {{.*}} +; SM100_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; SM120_NVVM_HALFWORD-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; SM75_NVVM_HALFWORD_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM80_NVVM_HALFWORD_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_HALFWORD_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_HALFWORD_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} -; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; PTX_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; PTX_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; NVVM_BYTE_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; NVVM_BYTE_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM75_NVVM_HALFWORD: {{.*}}LD.E.SYS {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{\].*}} +; SM80_NVVM_HALFWORD: {{.*}}LD.E {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{\].*}} +; SM90_NVVM_HALFWORD: {{.*}}LD.E {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{\].*}} +; SM100_NVVM_HALFWORD_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM100_NVVM_HALFWORD_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM120_NVVM_HALFWORD_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; SM120_NVVM_HALFWORD_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] {{R[0-9]+}}, {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} +; PTX_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_BYTE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_BYTE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_BYTE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_HALFWORD_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; PTX_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM75_NVVM_BYTE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM80_NVVM_BYTE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; SM90_NVVM_BYTE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; PTX_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM75_NVVM_BYTE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM80_NVVM_BYTE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; SM90_NVVM_BYTE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} ; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.EXCH{{.*}} ; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}}{{.*\[}}[[ALIGNED_ADDR]]{{\].*}} diff --git a/libcudacxx/test/cmake/CodegenTest.cmake b/libcudacxx/test/cmake/CodegenTest.cmake index 090c4bd167e0..fc8f613cc40d 100644 --- a/libcudacxx/test/cmake/CodegenTest.cmake +++ b/libcudacxx/test/cmake/CodegenTest.cmake @@ -596,14 +596,22 @@ function(libcudacxx_codegen_add_sass_tests) "${arch}" ) string(REPLACE "," ";" common_check_prefixes "${check_prefixes}") + string(TOUPPER "${test_contents}" uppercase_test_contents) foreach (check_prefix IN LISTS arg_CHECK_PREFIXES) + string(TOUPPER "${check_prefix}" uppercase_check_prefix) string( REGEX MATCH "; ${check_prefix}(:|-[A-Z]+:)" has_check_prefix "${test_contents}" ) - if (has_check_prefix) + string( + REGEX MATCH + "%FILECHECK%[ ]+PREFIX_COMBINE[ ]+([A-Z][A-Z0-9_-]*[ ]*,[ ]*)*${uppercase_check_prefix}[ ]*(,|$)" + has_combined_check_prefix + "${uppercase_test_contents}" + ) + if (has_check_prefix OR has_combined_check_prefix) list(APPEND common_check_prefixes "${check_prefix}") endif() endforeach() From 8a15af049a49c07f5aeaf8aab5f1bf7c3dfc6b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Mon, 31 Aug 2026 11:39:14 -0700 Subject: [PATCH 21/24] Accept equivalent 128-bit retry predicates in SASS tests. --- .../atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu | 4 ++-- .../test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu index 57fa75f98e91..171e99a7b2e1 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu @@ -56,13 +56,13 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}ISETP.NE{{.*}} +; SMXX-DAG: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} -; SMXX: {{.*}}@{{!?P[0-9]+}} BRA{{.*}} +; SMXX: {{.*}}@[[RETRY_PRED]] BRA{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{ADD|MIN|MAX}}{{.*}} ; SMXX: {{.*}}RET.ABS.NODEC{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu index 32f0e03ed34b..b15b949be695 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu @@ -48,9 +48,10 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; NON_BLOCK_ACQUIRE: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX: {{.*}}ISETP.NE{{.*}} +; SMXX: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{AND|OR|XOR}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} +; SMXX: {{.*}}@[[RETRY_PRED]] BRA{{.*}} ; SMXX: {{.*}}RET.ABS.NODEC{{.*}} */ From b2bed6d83354ca0f51a64d3375aeb46afd5ec865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Mon, 31 Aug 2026 12:14:00 -0700 Subject: [PATCH 22/24] Check native NVVM subword bitwise atomics on SM100. --- .../test/atomic_codegen/sass/CMakeLists.txt | 53 +++++++-- .../sass/bitwise_types_8_16_atomic_ref.cu | 106 +++++++++++++----- 2 files changed, 118 insertions(+), 41 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt b/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt index ff74c152ab89..e6c0093fa787 100644 --- a/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt +++ b/libcudacxx/test/atomic_codegen/sass/CMakeLists.txt @@ -78,10 +78,23 @@ check_source_compiles( ) cmake_pop_check_state() +set(atomic_codegen_sass_cas_subword_bitwise_archs) +set(atomic_codegen_sass_native_subword_bitwise_archs) if (libcudacxx_atomic_codegen_uses_nvvm_backend) set(atomic_codegen_sass_backend_prefix NVVM) + foreach (arch IN LISTS atomic_codegen_sass_cuda_archs) + if (arch GREATER_EQUAL 100) + list(APPEND atomic_codegen_sass_native_subword_bitwise_archs ${arch}) + else() + list(APPEND atomic_codegen_sass_cas_subword_bitwise_archs ${arch}) + endif() + endforeach() else() set(atomic_codegen_sass_backend_prefix PTX) + set( + atomic_codegen_sass_cas_subword_bitwise_archs + ${atomic_codegen_sass_cuda_archs} + ) endif() if (NOT libcudacxx_atomic_codegen_has_int128) @@ -104,14 +117,32 @@ endif() # Restrict FileCheck to the wrapper so instructions in outlined helpers cannot # mask an inlining regression. -libcudacxx_codegen_add_sass_tests( - AGGREGATE_TARGET libcudacxx.test.atomics.sass - TARGET_PREFIX atomic_codegen - ARCHITECTURES ${atomic_codegen_sass_cuda_archs} - DUMP_FUNCTIONS atomic_codegen_test - CHECK_PREFIXES - ${atomic_codegen_sass_cuda_version_prefix} - ${atomic_codegen_sass_backend_prefix} - TESTS ${libcudacxx_atomic_codegen_tests} - COMPILE_DEFINITIONS _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1 -) +if (atomic_codegen_sass_cas_subword_bitwise_archs) + libcudacxx_codegen_add_sass_tests( + AGGREGATE_TARGET libcudacxx.test.atomics.sass + TARGET_PREFIX atomic_codegen + ARCHITECTURES ${atomic_codegen_sass_cas_subword_bitwise_archs} + DUMP_FUNCTIONS atomic_codegen_test + CHECK_PREFIXES + ${atomic_codegen_sass_cuda_version_prefix} + ${atomic_codegen_sass_backend_prefix} + CAS_SUBWORD_BITWISE + TESTS ${libcudacxx_atomic_codegen_tests} + COMPILE_DEFINITIONS _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1 + ) +endif() + +if (atomic_codegen_sass_native_subword_bitwise_archs) + libcudacxx_codegen_add_sass_tests( + AGGREGATE_TARGET libcudacxx.test.atomics.sass + TARGET_PREFIX atomic_codegen + ARCHITECTURES ${atomic_codegen_sass_native_subword_bitwise_archs} + DUMP_FUNCTIONS atomic_codegen_test + CHECK_PREFIXES + ${atomic_codegen_sass_cuda_version_prefix} + ${atomic_codegen_sass_backend_prefix} + NATIVE_SUBWORD_BITWISE + TESTS ${libcudacxx_atomic_codegen_tests} + COMPILE_DEFINITIONS _CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1 + ) +endif() diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu index 1669524907a4..e0ce1a11242a 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_types_8_16_atomic_ref.cu @@ -11,10 +11,30 @@ // clang-format off // %PARAM% SCOPE,SASS_SCOPE,FILECHECK_PREFIX_SCOPE scope block=tsb,CTA,block:device=tsd,GPU,non_block:system=tss,SYS,non_block // %PARAM% TYPE type int8_t:uint8_t:int16_t:uint16_t -// %PARAM% OP,SASS_OP op and=fetch_and,AND:or=fetch_or,OR:xor=fetch_xor,XOR +// %PARAM% OP,SASS_OP,FILECHECK_PREFIX_BITWISE_OP op and=fetch_and,AND,and:or=fetch_or,OR,or_xor:xor=fetch_xor,XOR,or_xor // %PARAM% ORDER,FILECHECK_PREFIX_SEQ_CST,FILECHECK_PREFIX_ACQUIRE,FILECHECK_PREFIX_ORDER order relaxed=mor,non_seq_cst,no_acquire,no_membar:acquire=moa,non_seq_cst,acquire,no_membar:release=more,non_seq_cst,no_acquire,release:acq_rel=moar,non_seq_cst,acquire,release:seq_cst=mosc,seq_cst,acquire,seq_cst -// %FILECHECK% PREFIX_COMBINE non_block,seq_cst -// %FILECHECK% PREFIX_COMBINE non_block,acquire +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,block +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_block +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_seq_cst +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,seq_cst +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,release +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,no_membar +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,non_block,acquire +// %FILECHECK% PREFIX_COMBINE cas_subword_bitwise,no_acquire +// %FILECHECK% PREFIX_COMBINE ptx,cas_subword_bitwise +// %FILECHECK% PREFIX_COMBINE nvvm,cas_subword_bitwise +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,block +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_block +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_seq_cst +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,seq_cst +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_block,seq_cst +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,release +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,no_membar +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,non_block,acquire +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,no_acquire +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,and +// %FILECHECK% PREFIX_COMBINE native_subword_bitwise,or_xor // clang-format on #include "atomic_codegen_helpers.h" @@ -24,38 +44,64 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at return atom.OP(value, ORDER); } -// The compiler may fuse the bitwise operation with subword packing, so check -// the CAS protocol without fixing a particular LOP3 truth table. +// The compiler may fuse the bitwise operation with subword packing, so avoid +// fixing a particular LOP3 truth table. +// TODO: Improve PTX codegen for SM100+ to use the native widened and masked +// subword bitwise operations too. // clang-format off /* ; SMXX-LABEL: {{[[:space:]]*}}Function : atomic_codegen_test -; SMXX-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} ; SMXX: {{.*}}LD.E.64{{(\.SYS)?}} [[ATOM_ADDR:R[0-9]+]], {{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}LOP3.LUT [[ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} -; BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[ALIGNED_ADDR]]{{(\.64)?\].*}} -; SMXX: {{.*}}LOP3.LUT {{.*}} -; RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} -; SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} -; NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} -; NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} -; SMXX-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} -; BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; NON_BLOCK: {{.*}}ATOM.E.CAS.STRONG.[[SASS_SCOPE]] PT, [[OLD:R[0-9]+]], {{\[}}[[ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]], [[EXPECTED]], {{.*}} -; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} -; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} -; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} -; SMXX-NOT: {{.*}}ATOM.E.CAS{{.*}} -; SMXX: {{.*}}RET.ABS.NODEC{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE: {{.*}}LOP3.LUT [[CAS_ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; CAS_SUBWORD_BITWISE_BLOCK: {{.*}}LD.E.STRONG.{{CTA|SM}} [[EXPECTED:R[0-9]+]], {{.*\[}}[[CAS_ALIGNED_ADDR]]{{(\.64)?\].*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK: {{.*}}LD.E.STRONG.[[SASS_SCOPE]] [[EXPECTED:R[0-9]+]], {{.*\[}}[[CAS_ALIGNED_ADDR]]{{(\.64)?\].*}} +; CAS_SUBWORD_BITWISE: {{.*}}LOP3.LUT {{.*}} +; CAS_SUBWORD_BITWISE_RELEASE: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; CAS_SUBWORD_BITWISE_SEQ_CST: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK_SEQ_CST: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK: {{.*}}ATOM.E.CAS.STRONG.{{CTA|SM}} PT, [[OLD:R[0-9]+]], {{\[}}[[CAS_ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK: {{.*}}ATOM.E.CAS.STRONG.[[SASS_SCOPE]] PT, [[OLD:R[0-9]+]], {{\[}}[[CAS_ALIGNED_ADDR]]{{\]}}, [[EXPECTED]], {{R[0-9]+}}{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} +; PTX_CAS_SUBWORD_BITWISE-DAG: {{.*}}ISETP.NE{{.*}} [[OLD]], [[EXPECTED]], {{.*}} +; NVVM_CAS_SUBWORD_BITWISE-DAG: {{.*}}ISETP.NE{{.*}} [[EXPECTED]], [[OLD]], {{.*}} +; CAS_SUBWORD_BITWISE_NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE_NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; CAS_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; CAS_SUBWORD_BITWISE: {{.*}}RET.ABS.NODEC{{.*}} + +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; NATIVE_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_SEQ_CST-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_RELEASE-DAG: {{.*}}MEMBAR.ALL.[[SASS_SCOPE]]{{.*}} +; NATIVE_SUBWORD_BITWISE_SEQ_CST-DAG: {{.*}}MEMBAR.SC.[[SASS_SCOPE]]{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_BLOCK_SEQ_CST-DAG: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NO_MEMBAR-NOT: {{.*}}MEMBAR.{{.*}} +; NATIVE_SUBWORD_BITWISE-DAG: {{.*}}LOP3.LUT [[NATIVE_ALIGNED_ADDR:R[0-9]+]], [[ATOM_ADDR]]{{(\.reuse)?}}, 0xfffffffc, {{.*}} +; NATIVE_SUBWORD_BITWISE-DAG: {{.*}}LOP3.LUT [[SHIFT:R[0-9]+]], {{.*}}, 0x18, {{.*}} +; NATIVE_SUBWORD_BITWISE_AND-DAG: {{.*}}SHF.L.U32 [[SHIFTED_OPERAND:R[0-9]+]], {{.*}}, [[SHIFT]]{{(\.reuse)?}}, RZ{{.*}} +; NATIVE_SUBWORD_BITWISE_AND-DAG: {{.*}}LOP3.LUT [[NATIVE_OPERAND:R[0-9]+]], [[SHIFTED_OPERAND]], {{.*}} +; NATIVE_SUBWORD_BITWISE_OR_XOR-DAG: {{.*}}SHF.L.U32 [[NATIVE_OPERAND:R[0-9]+]], {{.*}}, [[SHIFT]]{{(\.reuse)?}}, RZ{{.*}} +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; NATIVE_SUBWORD_BITWISE_BLOCK: {{.*}}ATOM.E.[[SASS_OP]].STRONG.{{CTA|SM}} PT, [[NATIVE_OLD:R[0-9]+]], {{.*\[}}[[NATIVE_ALIGNED_ADDR]]{{(\.64)?\].*}}, [[NATIVE_OPERAND]]{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_BLOCK: {{.*}}ATOM.E.[[SASS_OP]].STRONG.[[SASS_SCOPE]] PT, [[NATIVE_OLD:R[0-9]+]], {{.*\[}}[[NATIVE_ALIGNED_ADDR]]{{(\.64)?\].*}}, [[NATIVE_OPERAND]]{{.*}} +; NATIVE_SUBWORD_BITWISE_BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE_NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} +; NATIVE_SUBWORD_BITWISE-DAG: {{.*}}SHF.R.U32.HI {{R[0-9]+}}, RZ, [[SHIFT]]{{(\.reuse)?}}, [[NATIVE_OLD]]{{.*}} +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.[[SASS_OP]]{{.*}} +; NATIVE_SUBWORD_BITWISE-NOT: {{.*}}ATOM.E.CAS{{.*}} +; NATIVE_SUBWORD_BITWISE: {{.*}}RET.ABS.NODEC{{.*}} */ // clang-format on From 5b72d8e618787d494760ea40a3ad2bbadbe4f0db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Mon, 31 Aug 2026 18:17:30 -0700 Subject: [PATCH 23/24] Accept older 128-bit retry predicate lowering. --- .../test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu | 2 +- .../test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu index 171e99a7b2e1..b987ca0c50bb 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu @@ -56,7 +56,7 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} +; SMXX-DAG: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.U64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu index b15b949be695..d0bc2c3aea26 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu @@ -48,7 +48,7 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; NON_BLOCK_ACQUIRE: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} +; SMXX: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.U64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{AND|OR|XOR}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} ; SMXX: {{.*}}@[[RETRY_PRED]] BRA{{.*}} From f29671e415f35d5055377677de82b1ec33f6b28a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Griwes=27=20Dominiak?= Date: Mon, 31 Aug 2026 19:11:22 -0700 Subject: [PATCH 24/24] Accept signed 64-bit retry predicate lowering. --- .../test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu | 2 +- .../test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu index b987ca0c50bb..8761b16af7f5 100644 --- a/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/arithmetic_types_128_atomic_ref.cu @@ -56,7 +56,7 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX-DAG: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.U64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} +; SMXX-DAG: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.[SU]64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; NON_BLOCK_ACQUIRE-DAG: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} diff --git a/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu b/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu index d0bc2c3aea26..a9d951144237 100644 --- a/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu +++ b/libcudacxx/test/atomic_codegen/sass/bitwise_types_128_atomic_ref.cu @@ -48,7 +48,7 @@ extern "C" __device__ auto atomic_codegen_test(cuda::atomic_ref& at ; NON_BLOCK_ACQUIRE: {{.*}}CCTL.IVALL{{.*}} ; BLOCK-NOT: {{.*}}CCTL.IVALL{{.*}} ; NO_ACQUIRE-NOT: {{.*}}CCTL.IVALL{{.*}} -; SMXX: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.U64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} +; SMXX: {{.*}}{{ISETP\.NE(\.U32)?\.OR\.EX|ISETP\.NE\.[SU]64\.OR|LOP3\.LUT}} [[RETRY_PRED:P[0-9]+]], {{.*}}[[RETRY_PRED]]{{.*}} ; SMXX-NOT: {{.*}}ATOM.E.{{AND|OR|XOR}}{{.*}} ; SMXX-NOT: {{.*}}ATOM.{{.*}}CAS{{.*}} ; SMXX: {{.*}}@[[RETRY_PRED]] BRA{{.*}}