Skip to content

[libc++] Floating Point Atomic #67799

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 36 commits into from
Nov 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
8d780a8
[libc++][In progress] Floating Point Atomic
huixie90 Jun 28, 2023
d67e085
more tests
huixie90 Oct 2, 2023
523ce08
address feedback and rebase
huixie90 Oct 9, 2023
703df84
more tests
huixie90 Oct 11, 2023
64064ef
more tests
huixie90 Oct 11, 2023
a8b6621
shadow
huixie90 Oct 11, 2023
b260f7b
CI
huixie90 Oct 13, 2023
3fb8e40
add linker flag
huixie90 Oct 13, 2023
7c42f0c
CI
huixie90 Oct 13, 2023
1000d3e
volatile
huixie90 Oct 13, 2023
069cf2a
gcc
huixie90 Oct 13, 2023
7c8b0fc
clang format
huixie90 Oct 13, 2023
839532b
gcc warning
huixie90 Oct 14, 2023
0c3c9a1
relax floating point comparison
huixie90 Oct 14, 2023
b0bac87
test which test hangs in CI
huixie90 Oct 14, 2023
dfbd52f
try to figure out which one
huixie90 Oct 14, 2023
d80b44c
using compare_exchange_strong for long double
huixie90 Oct 15, 2023
dac3b3f
try again
huixie90 Oct 15, 2023
ef9a9d4
more try
huixie90 Oct 15, 2023
a5ff0b4
address feedback
huixie90 Oct 16, 2023
4c90193
latomic
huixie90 Oct 16, 2023
44ce10e
another try
huixie90 Oct 19, 2023
e75e2be
try again
huixie90 Oct 19, 2023
ef76292
fix clang tidy
huixie90 Oct 20, 2023
0e51ee3
another try
huixie90 Oct 21, 2023
b84119b
format
huixie90 Oct 21, 2023
02c08cc
another try
huixie90 Oct 21, 2023
fa3a632
fix gcc
huixie90 Oct 22, 2023
5f41f1f
remove stdio
huixie90 Oct 22, 2023
9cad749
clean up
huixie90 Oct 22, 2023
00cf684
refactor
huixie90 Oct 23, 2023
28b3ca2
gcc
huixie90 Oct 23, 2023
76f6b19
gcc
huixie90 Oct 23, 2023
f0a0a7e
CI
huixie90 Oct 23, 2023
89cf41a
no thread
huixie90 Oct 23, 2023
33de815
rebase + xfail some tests
huixie90 Nov 20, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions libcxx/docs/ReleaseNotes/18.rst
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ Implemented Papers
- P2614R2 - Deprecate ``numeric_limits::has_denorm``
- P0053R7 - C++ Synchronized Buffered Ostream (in the experimental library)
- P2467R1 - Support exclusive mode for fstreams
- P0020R6 - Floating Point Atomic


Improvements and New Features
Expand Down
2 changes: 1 addition & 1 deletion libcxx/docs/Status/Cxx20Papers.csv
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"`P0463R1 <https://wg21.link/P0463R1>`__","LWG","Endian just Endian","Toronto","|Complete|","7.0"
"`P0674R1 <https://wg21.link/P0674R1>`__","LWG","Extending make_shared to Support Arrays","Toronto","|Complete|","15.0"
"","","","","","",""
"`P0020R6 <https://wg21.link/P0020R6>`__","LWG","Floating Point Atomic","Albuquerque","",""
"`P0020R6 <https://wg21.link/P0020R6>`__","LWG","Floating Point Atomic","Albuquerque","|Complete|","18.0"
"`P0053R7 <https://wg21.link/P0053R7>`__","LWG","C++ Synchronized Buffered Ostream","Albuquerque","|Complete|","18.0"
"`P0202R3 <https://wg21.link/P0202R3>`__","LWG","Add constexpr modifiers to functions in <algorithm> and <utility> Headers","Albuquerque","|Complete|","12.0"
"`P0415R1 <https://wg21.link/P0415R1>`__","LWG","Constexpr for ``std::complex``\ ","Albuquerque","|Complete|","16.0"
Expand Down
136 changes: 136 additions & 0 deletions libcxx/include/__atomic/atomic.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,17 @@
#include <__atomic/cxx_atomic_impl.h>
#include <__atomic/memory_order.h>
#include <__config>
#include <__functional/operations.h>
#include <__memory/addressof.h>
#include <__type_traits/is_floating_point.h>
#include <__type_traits/is_function.h>
#include <__type_traits/is_same.h>
#include <__type_traits/remove_const.h>
#include <__type_traits/remove_pointer.h>
#include <__type_traits/remove_volatile.h>
#include <__utility/forward.h>
#include <cstddef>
#include <cstring>

#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
Expand Down Expand Up @@ -136,6 +142,136 @@ struct atomic<_Tp*>
atomic& operator=(const atomic&) volatile = delete;
};

#if _LIBCPP_STD_VER >= 20
template <class _Tp>
requires is_floating_point_v<_Tp>
struct atomic<_Tp> : __atomic_base<_Tp> {
private:
_LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() {
// Only x87-fp80 long double has 64-bit mantissa
return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>;
}

_LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() {
# ifndef _LIBCPP_COMPILER_CLANG_BASED
return false;
# else
// The builtin __cxx_atomic_fetch_add errors during compilation for
// long double on platforms with fp80 format.
// For more details, see
// lib/Sema/SemaChecking.cpp function IsAllowedValueType
// LLVM Parser does not allow atomicrmw with x86_fp80 type.
// if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) &&
// &Context.getTargetInfo().getLongDoubleFormat() ==
// &llvm::APFloat::x87DoubleExtended())
// For more info
// https://github.com/llvm/llvm-project/issues/68602
// https://reviews.llvm.org/D53965
return !__is_fp80_long_double();
# endif
}

template <class _This, class _Operation, class _BuiltinOp>
_LIBCPP_HIDE_FROM_ABI static _Tp
__rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
if constexpr (__has_rmw_builtin()) {
return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
} else {
_Tp __old = __self.load(memory_order_relaxed);
_Tp __new = __operation(__old, __operand);
while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
# ifdef _LIBCPP_COMPILER_CLANG_BASED
if constexpr (__is_fp80_long_double()) {
// https://github.com/llvm/llvm-project/issues/47978
// clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
// Note __old = __self.load(memory_order_relaxed) will not work
std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed);
}
# endif
__new = __operation(__old, __operand);
}
return __old;
}
}

template <class _This>
_LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
};
return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op);
}

template <class _This>
_LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
};
return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op);
}

public:
using __base = __atomic_base<_Tp>;
using value_type = _Tp;
using difference_type = value_type;

_LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;
_LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}

atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;

_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept
requires __base::is_always_lock_free
{
__base::store(__d);
return __d;
}
_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {
__base::store(__d);
return __d;
}

_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
requires __base::is_always_lock_free
{
return __fetch_add(*this, __op, __m);
}

_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
return __fetch_add(*this, __op, __m);
}

_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
requires __base::is_always_lock_free
{
return __fetch_sub(*this, __op, __m);
}

_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
return __fetch_sub(*this, __op, __m);
}

_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept
requires __base::is_always_lock_free
{
return fetch_add(__op) + __op;
}

_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }

_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept
requires __base::is_always_lock_free
{
return fetch_sub(__op) - __op;
}

_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
};

#endif // _LIBCPP_STD_VER >= 20

// atomic_is_lock_free

template <class _Tp>
Expand Down
47 changes: 47 additions & 0 deletions libcxx/include/__atomic/cxx_atomic_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,18 @@ _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a,
return __ret;
}

template <typename _Tp>
_LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
__atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
}

template <typename _Tp>
_LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
__atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
}

template <typename _Tp>
_LIBCPP_HIDE_FROM_ABI
_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
Expand Down Expand Up @@ -362,6 +374,21 @@ _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __ord
const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
}

template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
*__dst = __c11_atomic_load(
const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
}
template <class _Tp>
_LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
*__dst = __c11_atomic_load(
const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
}

template<class _Tp>
_LIBCPP_HIDE_FROM_ABI
_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
Expand Down Expand Up @@ -558,6 +585,16 @@ struct __cxx_atomic_lock_impl {
__unlock();
return __old;
}
_LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const volatile {
__lock();
__cxx_atomic_assign_volatile(*__dst, __a_value);
__unlock();
}
_LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const {
__lock();
*__dst = __a_value;
__unlock();
}
};

template <typename _Tp>
Expand Down Expand Up @@ -597,6 +634,16 @@ _Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
return __a->__read();
}

template <typename _Tp>
_LIBCPP_HIDE_FROM_ABI void
__cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
__a->__read_inplace(__dst);
}
template <typename _Tp>
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
__a->__read_inplace(__dst);
}

template <typename _Tp>
_LIBCPP_HIDE_FROM_ABI
_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
Expand Down
66 changes: 66 additions & 0 deletions libcxx/include/atomic
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,72 @@ struct atomic<T*>
void notify_all() noexcept;
};

template<>
struct atomic<floating-point-type> { // since C++20
using value_type = floating-point-type;
using difference_type = value_type;

static constexpr bool is_always_lock_free = implementation-defined;
bool is_lock_free() const volatile noexcept;
bool is_lock_free() const noexcept;

constexpr atomic() noexcept;
constexpr atomic(floating-point-type) noexcept;
atomic(const atomic&) = delete;
atomic& operator=(const atomic&) = delete;
atomic& operator=(const atomic&) volatile = delete;

void store(floating-point-type, memory_order = memory_order::seq_cst) volatile noexcept;
void store(floating-point-type, memory_order = memory_order::seq_cst) noexcept;
floating-point-type operator=(floating-point-type) volatile noexcept;
floating-point-type operator=(floating-point-type) noexcept;
floating-point-type load(memory_order = memory_order::seq_cst) volatile noexcept;
floating-point-type load(memory_order = memory_order::seq_cst) noexcept;
operator floating-point-type() volatile noexcept;
operator floating-point-type() noexcept;

floating-point-type exchange(floating-point-type,
memory_order = memory_order::seq_cst) volatile noexcept;
floating-point-type exchange(floating-point-type,
memory_order = memory_order::seq_cst) noexcept;
bool compare_exchange_weak(floating-point-type&, floating-point-type,
memory_order, memory_order) volatile noexcept;
bool compare_exchange_weak(floating-point-type&, floating-point-type,
memory_order, memory_order) noexcept;
bool compare_exchange_strong(floating-point-type&, floating-point-type,
memory_order, memory_order) volatile noexcept;
bool compare_exchange_strong(floating-point-type&, floating-point-type,
memory_order, memory_order) noexcept;
bool compare_exchange_weak(floating-point-type&, floating-point-type,
memory_order = memory_order::seq_cst) volatile noexcept;
bool compare_exchange_weak(floating-point-type&, floating-point-type,
memory_order = memory_order::seq_cst) noexcept;
bool compare_exchange_strong(floating-point-type&, floating-point-type,
memory_order = memory_order::seq_cst) volatile noexcept;
bool compare_exchange_strong(floating-point-type&, floating-point-type,
memory_order = memory_order::seq_cst) noexcept;

floating-point-type fetch_add(floating-point-type,
memory_order = memory_order::seq_cst) volatile noexcept;
floating-point-type fetch_add(floating-point-type,
memory_order = memory_order::seq_cst) noexcept;
floating-point-type fetch_sub(floating-point-type,
memory_order = memory_order::seq_cst) volatile noexcept;
floating-point-type fetch_sub(floating-point-type,
memory_order = memory_order::seq_cst) noexcept;

floating-point-type operator+=(floating-point-type) volatile noexcept;
floating-point-type operator+=(floating-point-type) noexcept;
floating-point-type operator-=(floating-point-type) volatile noexcept;
floating-point-type operator-=(floating-point-type) noexcept;

void wait(floating-point-type, memory_order = memory_order::seq_cst) const volatile noexcept;
void wait(floating-point-type, memory_order = memory_order::seq_cst) const noexcept;
void notify_one() volatile noexcept;
void notify_one() noexcept;
void notify_all() volatile noexcept;
void notify_all() noexcept;
};

// [atomics.nonmembers], non-member functions
template<class T>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// UNSUPPORTED: c++03, c++11, c++14, c++17
// UNSUPPORTED: target={{.+}}-windows-gnu
// ADDITIONAL_COMPILE_FLAGS(has-latomic): -latomic

// static constexpr bool is_always_lock_free = implementation-defined;
// bool is_lock_free() const volatile noexcept;
// bool is_lock_free() const noexcept;

#include <atomic>
#include <cassert>
#include <concepts>

#include "test_macros.h"

template <class T>
void test() {
// static constexpr bool is_always_lock_free = implementation-defined;
{
bool r = std::atomic<T>::is_always_lock_free;
assert(r == __atomic_always_lock_free(sizeof(std::__cxx_atomic_impl<T>), 0));
}

// bool is_lock_free() const volatile noexcept;
{
const volatile std::atomic<T> a;
bool r = a.is_lock_free();
assert(r == __cxx_atomic_is_lock_free(sizeof(std::__cxx_atomic_impl<T>)));
}

// bool is_lock_free() const noexcept;
{
const std::atomic<T> a;
bool r = a.is_lock_free();
assert(r == __cxx_atomic_is_lock_free(sizeof(std::__cxx_atomic_impl<T>)));
}
}

int main(int, char**) {
test<float>();
test<double>();
test<long double>();

return 0;
}
Loading