Skip to content

libcu++: __nv_atomic based atomics backend. #11075

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
griwes wants to merge 27 commits into
base: main
Choose a base branch
from
Draft

libcu++: __nv_atomic based atomics backend. #11075

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
6a883e8
Reorganize atomic backend implementation headers.
griwes Aug 19, 2026
5443ead
Refactor the PTX atomic backend.
griwes Aug 27, 2026
3c1bedf
Unify atomic dispatch across backends.
griwes Aug 19, 2026
1800050
Preserve compare-exchange strength through backend dispatch.
griwes Aug 19, 2026
1dde28f
Complete atomic backend dispatch.
griwes Aug 27, 2026
bf76f30
Tighten sequentially consistent CAS-loop checks.
griwes Aug 27, 2026
36e22bf
Reduce atomic binder instantiations.
griwes Aug 27, 2026
117a264
Avoid ADL in atomic backend dispatch.
griwes Aug 27, 2026
a263bb7
Limit PTX fence state to pre-SM70 lowering.
griwes Aug 27, 2026
be10761
Guard CUDA-only local memory fallbacks.
griwes Aug 27, 2026
c767317
CI and coderabbit fixes.
griwes Aug 28, 2026
246e6df
Address review comments.
griwes Aug 28, 2026
f0d69e2
Use libcu++ API annotations throughout atomics.
griwes Aug 28, 2026
f3efa4a
Merge remote-tracking branch 'origin/main' into feature/atomic-backen…
griwes Aug 28, 2026
41ecfb0
Correct SC fence placement in small arithmetic SASS checks.
griwes Aug 28, 2026
411482f
Relax float minmax patterns to account for reg alloc variance.
griwes Aug 28, 2026
b9deae5
Merge branch 'main' into feature/atomic-backend-refactor
griwes Aug 28, 2026
e42d75e
Move atomic dispatch machinery out of the PTX generator.
griwes Aug 28, 2026
eb3a1c3
Move the signal fence into the PTX backend.
griwes Aug 29, 2026
fafbed6
Merge remote-tracking branch 'origin/main' into feature/atomic-backen…
griwes Aug 31, 2026
ed4d0d5
Add an NVVM atomic backend.
griwes Aug 26, 2026
6980c89
Use native addition for floating-point NVVM fetch-sub.
griwes Aug 31, 2026
4f1b2cf
Account for NVVM release-fence placement in SASS tests.
griwes Aug 31, 2026
8a15af0
Accept equivalent 128-bit retry predicates in SASS tests.
griwes Aug 31, 2026
b2bed6d
Check native NVVM subword bitwise atomics on SM100.
griwes Aug 31, 2026
5b72d8e
Accept older 128-bit retry predicate lowering.
griwes Sep 1, 2026
f29671e
Accept signed 64-bit retry predicate lowering.
griwes Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 8 additions & 79 deletions libcudacxx/codegen/generators/compare_and_swap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,38 +18,6 @@

inline void FormatCompareAndSwap(std::ostream& out)
{
out << R"XXX(
template <class _Fn, class _Sco>
_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch(_Fn& __cuda_cas, int __success_memorder, int __failure_memorder, _Sco) {
bool __res = false;
NV_DISPATCH_TARGET(
NV_PROVIDES_SM_70, (
switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) {
case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]];
case __ATOMIC_CONSUME: [[fallthrough]];
case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_acquire{}); break;
case __ATOMIC_ACQ_REL: __res = __cuda_cas(__cuda_atomic_order_acq_rel{}); break;
case __ATOMIC_RELEASE: __res = __cuda_cas(__cuda_atomic_order_release{}); break;
case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_relaxed{}); break;
default: _CCCL_ASSERT(false, "invalid memory order");
}
),
NV_IS_DEVICE, (
switch (__cuda_atomic_stronger_order(__success_memorder, __failure_memorder)) {
case __ATOMIC_SEQ_CST: [[fallthrough]];
case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
case __ATOMIC_CONSUME: [[fallthrough]];
case __ATOMIC_ACQUIRE: __res = __cuda_cas(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break;
case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __res = __cuda_cas(__cuda_atomic_order_volatile{}); break;
case __ATOMIC_RELAXED: __res = __cuda_cas(__cuda_atomic_order_volatile{}); break;
default: _CCCL_ASSERT(false, "invalid memory order");
}
)
)
return __res;
}
)XXX";

// Argument ID Reference
// 0 - Operand Type
// 1 - Operand Size
Expand All @@ -61,8 +29,9 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_order_dispatch(_Fn& __cuda_
constexpr auto asm_intrinsic_format_128 = R"XXX(
template <class _Type>
_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange(
_Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __cuda_atomic_operand_{0}{1}, {6})
__cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6})
{{
::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}});
static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840");
NV_DISPATCH_TARGET(
NV_PROVIDES_SM_90, (),
Expand All @@ -78,12 +47,11 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange(
mov.b128 {{%0, %1}}, _d;
}}
)YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; }})XXX";

constexpr auto asm_intrinsic_format = R"XXX(
constexpr auto asm_intrinsic_format = R"XXX(
template <class _Type>
_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange(
_Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __cuda_atomic_operand_{0}{1}, {6})
{{ asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX";
__cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __dst, __unv<_Type> __cmp, __unv<_Type> __op, __cuda_atomic_cas_strong, {4} __order, __cuda_atomic_operand_{0}{1}, {6})
{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX";

constexpr Operand supported_types[] = {
Operand::Bit,
Expand Down Expand Up @@ -135,7 +103,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange(
size,
constraints(type, size),
semantic(sem),
semantic_tag(sem),
ptx_semantic_tag(sem),
scope(sco),
scope_tag(sco));
}
Expand All @@ -147,7 +115,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange(
size,
constraints(type, size),
semantic(sem),
semantic_tag(sem),
ptx_semantic_tag(sem),
scope(sco),
scope_tag(sco));
}
Expand All @@ -156,46 +124,7 @@ _CCCL_DEVICE_API bool __cuda_atomic_compare_exchange(
}
}

out << "\n"
<< R"XXX(
template <typename _Type, typename _Tag, typename _Sco>
struct __cuda_atomic_bind_compare_exchange {
_Type* __ptr;
_Type* __exp;
_Type* __des;

template <typename _Atomic_Memorder>
_CCCL_DEVICE_API bool operator()(_Atomic_Memorder) {
return __cuda_atomic_compare_exchange(__ptr, *__exp, *__exp, *__des, _Atomic_Memorder{}, _Tag{}, _Sco{});
}
};
template <class _Type, class _Sco>
_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch(_Type* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco)
{
using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type;
using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag;
__proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
__proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp);
__proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des);
bool __res = false;
if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;}
__cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy};
return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{});
}
template <class _Type, class _Sco>
_CCCL_DEVICE_API bool __cuda_atomic_compare_exchange_dispatch(_Type volatile* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco)
{
using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type;
using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag;
__proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
__proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp);
__proxy_t* __des_proxy = reinterpret_cast<__proxy_t*>(&__des);
bool __res = false;
if (__cuda_atomic_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;}
__cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy};
return __cuda_atomic_compare_exchange_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{});
}
)XXX";
out << "\n";
}

#endif // COMPARED_AND_SWAP_H
12 changes: 12 additions & 0 deletions libcudacxx/codegen/generators/definitions.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,18 @@ inline std::string semantic_tag(Semantic sem)
return sem_map[sem];
}

inline std::string ptx_semantic_tag(Semantic sem)
{
static std::map sem_map = {
std::pair{Semantic::Relaxed, "__cuda_atomic_ptx_order_relaxed"},
std::pair{Semantic::Release, "__cuda_atomic_ptx_order_release"},
std::pair{Semantic::Acquire, "__cuda_atomic_ptx_order_acquire"},
std::pair{Semantic::Acq_Rel, "__cuda_atomic_ptx_order_acq_rel"},
std::pair{Semantic::Volatile, "__cuda_atomic_order_volatile"},
};
return sem_map[sem];
}

enum class Scope
{
Thread,
Expand Down
83 changes: 8 additions & 75 deletions libcudacxx/codegen/generators/exchange.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,36 +18,6 @@

inline void FormatExchange(std::ostream& out)
{
out << R"XXX(
template <class _Fn, class _Sco>
_CCCL_DEVICE_API void __cuda_atomic_exchange_order_dispatch(_Fn& __cuda_exch, int __memorder, _Sco) {
NV_DISPATCH_TARGET(
NV_PROVIDES_SM_70, (
switch (__memorder) {
case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); [[fallthrough]];
case __ATOMIC_CONSUME: [[fallthrough]];
case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_acquire{}); break;
case __ATOMIC_ACQ_REL: __cuda_exch(__cuda_atomic_order_acq_rel{}); break;
case __ATOMIC_RELEASE: __cuda_exch(__cuda_atomic_order_release{}); break;
case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_relaxed{}); break;
default: _CCCL_ASSERT(false, "invalid memory order");
}
),
NV_IS_DEVICE, (
switch (__memorder) {
case __ATOMIC_SEQ_CST: [[fallthrough]];
case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
case __ATOMIC_CONSUME: [[fallthrough]];
case __ATOMIC_ACQUIRE: __cuda_exch(__cuda_atomic_order_volatile{}); __cuda_atomic_membar(_Sco{}); break;
case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_exch(__cuda_atomic_order_volatile{}); break;
case __ATOMIC_RELAXED: __cuda_exch(__cuda_atomic_order_volatile{}); break;
default: _CCCL_ASSERT(false, "invalid memory order");
}
)
)
}
)XXX";

// Argument ID Reference
// 0 - Operand Type
// 1 - Operand Size
Expand All @@ -59,8 +29,9 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange_order_dispatch(_Fn& __cuda_exch, in
constexpr auto asm_intrinsic_format_128 = R"XXX(
template <class _Type>
_CCCL_DEVICE_API void __cuda_atomic_exchange(
_Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6})
__cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6})
{{
::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}});
static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840");
NV_DISPATCH_TARGET(
NV_PROVIDES_SM_90, (),
Expand All @@ -76,12 +47,11 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange(
}}
)YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory");
}})XXX";

constexpr auto asm_intrinsic_format = R"XXX(
constexpr auto asm_intrinsic_format = R"XXX(
template <class _Type>
_CCCL_DEVICE_API void __cuda_atomic_exchange(
_Type* __ptr, _Type& __old, _Type __new, {4}, __cuda_atomic_operand_{0}{1}, {6})
{{ asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX";
__cuda_atomic_ptx_backend, _Type* __ptr, __unv<_Type>& __old, __unv<_Type> __new, {4} __order, __cuda_atomic_operand_{0}{1}, {6})
{{ ::cuda::std::__cuda_atomic_ptx_maybe_sc_fence(__order, {6}{{}}); asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX";

constexpr Operand supported_types[] = {
Operand::Bit,
Expand Down Expand Up @@ -133,7 +103,7 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange(
size,
constraints(type, size),
semantic(sem),
semantic_tag(sem),
ptx_semantic_tag(sem),
scope(sco),
scope_tag(sco));
}
Expand All @@ -145,7 +115,7 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange(
size,
constraints(type, size),
semantic(sem),
semantic_tag(sem),
ptx_semantic_tag(sem),
scope(sco),
scope_tag(sco));
}
Expand All @@ -154,44 +124,7 @@ _CCCL_DEVICE_API void __cuda_atomic_exchange(
}
}

out << "\n"
<< R"XXX(
template <typename _Type, typename _Tag, typename _Sco>
struct __cuda_atomic_bind_exchange {
_Type* __ptr;
_Type* __old;
_Type* __new;

template <typename _Atomic_Memorder>
_CCCL_DEVICE_API void operator()(_Atomic_Memorder) {
__cuda_atomic_exchange(__ptr, *__old, *__new, _Atomic_Memorder{}, _Tag{}, _Sco{});
}
};
template <class _Type, class _Sco>
_CCCL_DEVICE_API void __atomic_exchange_cuda(_Type* __ptr, _Type& __old, _Type __new, int __memorder, _Sco)
{
using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type;
using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag;
__proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
__proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old);
__proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new);
if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}}
__cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy};
__cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{});
}
template <class _Type, class _Sco>
_CCCL_DEVICE_API void __atomic_exchange_cuda(_Type volatile* __ptr, _Type& __old, _Type __new, int __memorder, _Sco)
{
using __proxy_t = typename __cuda_atomic_deduce_bitwise<_Type>::__type;
using __proxy_tag = typename __cuda_atomic_deduce_bitwise<_Type>::__tag;
__proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
__proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old);
__proxy_t* __new_proxy = reinterpret_cast<__proxy_t*>(&__new);
if(__cuda_atomic_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}}
__cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy};
__cuda_atomic_exchange_order_dispatch(__bound_swap, __memorder, _Sco{});
}
)XXX";
out << "\n";
}

#endif // EXCHANGE_H
24 changes: 20 additions & 4 deletions libcudacxx/codegen/generators/fence.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,32 @@ _CCCL_DEVICE_API inline void __cuda_atomic_fence({0}, {2})
}
out << "\n"
<< R"XXX(
template <class _Order, class _Sco>
_CCCL_DEVICE_API void
__cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_ptx_order<_Order> __order, _Sco __scope)
{
if (__order.__was_seq_cst)
{
::cuda::std::__cuda_atomic_fence(__scope, __cuda_atomic_order_seq_cst{});
}
}

template <class _Sco>
_CCCL_DEVICE_API void __cuda_atomic_ptx_maybe_sc_fence(__cuda_atomic_order_volatile, _Sco)
{}

template <typename _Sco>
_CCCL_DEVICE_API void __cuda_atomic_thread_fence(int __memorder, _Sco) {
_CCCL_DEVICE_API void __cuda_atomic_thread_fence(
__cuda_atomic_ptx_backend, memory_order __order, _Sco) {
[[maybe_unused]] const int __memorder = __atomic_order_to_int(__order);
NV_DISPATCH_TARGET(
NV_PROVIDES_SM_70, (
switch (__memorder) {
case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break;
case __ATOMIC_SEQ_CST: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_seq_cst{}); break;
case __ATOMIC_CONSUME: [[fallthrough]];
case __ATOMIC_ACQUIRE: [[fallthrough]];
case __ATOMIC_ACQ_REL: [[fallthrough]];
case __ATOMIC_RELEASE: __cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break;
case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_fence(_Sco{}, __cuda_atomic_order_acq_rel{}); break;
case __ATOMIC_RELAXED: break;
default: _CCCL_ASSERT(false, "invalid memory order");
}
Expand All @@ -97,7 +113,7 @@ _CCCL_DEVICE_API void __cuda_atomic_thread_fence(int __memorder, _Sco) {
case __ATOMIC_CONSUME: [[fallthrough]];
case __ATOMIC_ACQUIRE: [[fallthrough]];
case __ATOMIC_ACQ_REL: [[fallthrough]];
case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); break;
case __ATOMIC_RELEASE: ::cuda::std::__cuda_atomic_membar(_Sco{}); break;
case __ATOMIC_RELAXED: break;
default: _CCCL_ASSERT(false, "invalid memory order");
}
Expand Down
Loading
Loading