[libc++] Floating Point Atomic

libcxx/docs/ReleaseNotes/18.rst

@@ -49,6 +49,7 @@ Implemented Papers
 - P2614R2 - Deprecate ``numeric_limits::has_denorm``
 - P0053R7 - C++ Synchronized Buffered Ostream (in the experimental library)
 - P2467R1 - Support exclusive mode for fstreams
+- P0020R6 - Floating Point Atomic
 
 
 Improvements and New Features

libcxx/docs/Status/Cxx20Papers.csv

@@ -2,7 +2,7 @@
 "`P0463R1 <https://wg21.link/P0463R1>`__","LWG","Endian just Endian","Toronto","|Complete|","7.0"
 "`P0674R1 <https://wg21.link/P0674R1>`__","LWG","Extending make_shared to Support Arrays","Toronto","|Complete|","15.0"
 "","","","","","",""
-"`P0020R6 <https://wg21.link/P0020R6>`__","LWG","Floating Point Atomic","Albuquerque","",""
+"`P0020R6 <https://wg21.link/P0020R6>`__","LWG","Floating Point Atomic","Albuquerque","|Complete|","18.0"
 "`P0053R7 <https://wg21.link/P0053R7>`__","LWG","C++ Synchronized Buffered Ostream","Albuquerque","|Complete|","18.0"
 "`P0202R3 <https://wg21.link/P0202R3>`__","LWG","Add constexpr modifiers to functions in <algorithm> and <utility> Headers","Albuquerque","|Complete|","12.0"
 "`P0415R1 <https://wg21.link/P0415R1>`__","LWG","Constexpr for ``std::complex``\ ","Albuquerque","|Complete|","16.0"

libcxx/include/__atomic/atomic.h

@@ -14,11 +14,17 @@
 #include <__atomic/cxx_atomic_impl.h>
 #include <__atomic/memory_order.h>
 #include <__config>
+#include <__functional/operations.h>
 #include <__memory/addressof.h>
+#include <__type_traits/is_floating_point.h>
 #include <__type_traits/is_function.h>
 #include <__type_traits/is_same.h>
+#include <__type_traits/remove_const.h>
 #include <__type_traits/remove_pointer.h>
+#include <__type_traits/remove_volatile.h>
+#include <__utility/forward.h>
 #include <cstddef>
+#include <cstring>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -136,6 +142,136 @@ struct atomic<_Tp*>
     atomic& operator=(const atomic&) volatile = delete;
 };
 
+#if _LIBCPP_STD_VER >= 20
+template <class _Tp>
+  requires is_floating_point_v<_Tp>
+struct atomic<_Tp> : __atomic_base<_Tp> {
+  private:
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() {
+        // Only x87-fp80 long double has 64-bit mantissa
+        return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() {
+#  ifndef _LIBCPP_COMPILER_CLANG_BASED
+        return false;
+#  else
+        // The builtin __cxx_atomic_fetch_add errors during compilation for
+        // long double on platforms with fp80 format.
+        // For more details, see
+        // lib/Sema/SemaChecking.cpp function IsAllowedValueType
+        // LLVM Parser does not allow atomicrmw with x86_fp80 type.
+        // if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) &&
+        //    &Context.getTargetInfo().getLongDoubleFormat() ==
+        //        &llvm::APFloat::x87DoubleExtended())
+        // For more info
+        // https://github.com/llvm/llvm-project/issues/68602
+        // https://reviews.llvm.org/D53965
+        return !__is_fp80_long_double();
+#  endif
+    }
+
+    template <class _This, class _Operation, class _BuiltinOp>
+    _LIBCPP_HIDE_FROM_ABI static _Tp
+    __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
+        if constexpr (__has_rmw_builtin()) {
+          return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
+        } else {
+          _Tp __old = __self.load(memory_order_relaxed);
+          _Tp __new = __operation(__old, __operand);
+          while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
+#  ifdef _LIBCPP_COMPILER_CLANG_BASED
+            if constexpr (__is_fp80_long_double()) {
+              // https://github.com/llvm/llvm-project/issues/47978
+              // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
+              // Note __old = __self.load(memory_order_relaxed) will not work
+              std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed);
+            }
+#  endif
+            __new = __operation(__old, __operand);
+          }
+          return __old;
+        }
+    }
+
+    template <class _This>
+    _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
+        auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
+          return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
+        };
+        return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op);
+    }
+
+    template <class _This>
+    _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
+        auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
+          return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
+        };
+        return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op);
+    }
+
+  public:
+    using __base          = __atomic_base<_Tp>;
+    using value_type      = _Tp;
+    using difference_type = value_type;
+
+    _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;
+    _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}
+
+    atomic(const atomic&)                     = delete;
+    atomic& operator=(const atomic&)          = delete;
+    atomic& operator=(const atomic&) volatile = delete;
+
+    _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept
+      requires __base::is_always_lock_free
+    {
+        __base::store(__d);
+        return __d;
+    }
+    _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {
+        __base::store(__d);
+        return __d;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
+      requires __base::is_always_lock_free
+    {
+        return __fetch_add(*this, __op, __m);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
+        return __fetch_add(*this, __op, __m);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
+      requires __base::is_always_lock_free
+    {
+        return __fetch_sub(*this, __op, __m);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
+        return __fetch_sub(*this, __op, __m);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept
+      requires __base::is_always_lock_free
+    {
+        return fetch_add(__op) + __op;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept
+      requires __base::is_always_lock_free
+    {
+        return fetch_sub(__op) - __op;
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
+};
+
+#endif // _LIBCPP_STD_VER >= 20
+
 // atomic_is_lock_free
 
 template <class _Tp>

libcxx/include/__atomic/cxx_atomic_impl.h

@@ -128,6 +128,18 @@ _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a,
   return __ret;
 }
 
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
+}
+
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
+  __atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
+}
+
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI
 _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
@@ -362,6 +374,21 @@ _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __ord
         const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
 }
 
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
+    using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+    *__dst           = __c11_atomic_load(
+        const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
+    using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
+    *__dst           = __c11_atomic_load(
+        const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
+}
+
 template<class _Tp>
 _LIBCPP_HIDE_FROM_ABI
 _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
@@ -558,6 +585,16 @@ struct __cxx_atomic_lock_impl {
     __unlock();
     return __old;
   }
+  _LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const volatile {
+    __lock();
+    __cxx_atomic_assign_volatile(*__dst, __a_value);
+    __unlock();
+  }
+  _LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const {
+    __lock();
+    *__dst = __a_value;
+    __unlock();
+  }
 };
 
 template <typename _Tp>
@@ -597,6 +634,16 @@ _Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
   return __a->__read();
 }
 
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void
+__cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
+  __a->__read_inplace(__dst);
+}
+template <typename _Tp>
+_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
+  __a->__read_inplace(__dst);
+}
+
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI
 _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {

libcxx/include/atomic

@@ -262,6 +262,72 @@ struct atomic<T*>
     void notify_all() noexcept;
 };
 
+template<>
+struct atomic<floating-point-type> {  // since C++20
+  using value_type = floating-point-type;
+  using difference_type = value_type;
+
+  static constexpr bool is_always_lock_free = implementation-defined;
+  bool is_lock_free() const volatile noexcept;
+  bool is_lock_free() const noexcept;
+
+  constexpr atomic() noexcept;
+  constexpr atomic(floating-point-type) noexcept;
+  atomic(const atomic&) = delete;
+  atomic& operator=(const atomic&) = delete;
+  atomic& operator=(const atomic&) volatile = delete;
+
+  void store(floating-point-type, memory_order = memory_order::seq_cst) volatile noexcept;
+  void store(floating-point-type, memory_order = memory_order::seq_cst) noexcept;
+  floating-point-type operator=(floating-point-type) volatile noexcept;
+  floating-point-type operator=(floating-point-type) noexcept;
+  floating-point-type load(memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type load(memory_order = memory_order::seq_cst) noexcept;
+  operator floating-point-type() volatile noexcept;
+  operator floating-point-type() noexcept;
+
+  floating-point-type exchange(floating-point-type,
+                               memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type exchange(floating-point-type,
+                               memory_order = memory_order::seq_cst) noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order, memory_order) volatile noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order, memory_order) noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order, memory_order) volatile noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order, memory_order) noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order = memory_order::seq_cst) volatile noexcept;
+  bool compare_exchange_weak(floating-point-type&, floating-point-type,
+                             memory_order = memory_order::seq_cst) noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order = memory_order::seq_cst) volatile noexcept;
+  bool compare_exchange_strong(floating-point-type&, floating-point-type,
+                               memory_order = memory_order::seq_cst) noexcept;
+
+  floating-point-type fetch_add(floating-point-type,
+                                memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type fetch_add(floating-point-type,
+                                memory_order = memory_order::seq_cst) noexcept;
+  floating-point-type fetch_sub(floating-point-type,
+                                memory_order = memory_order::seq_cst) volatile noexcept;
+  floating-point-type fetch_sub(floating-point-type,
+                                memory_order = memory_order::seq_cst) noexcept;
+
+  floating-point-type operator+=(floating-point-type) volatile noexcept;
+  floating-point-type operator+=(floating-point-type) noexcept;
+  floating-point-type operator-=(floating-point-type) volatile noexcept;
+  floating-point-type operator-=(floating-point-type) noexcept;
+
+  void wait(floating-point-type, memory_order = memory_order::seq_cst) const volatile noexcept;
+  void wait(floating-point-type, memory_order = memory_order::seq_cst) const noexcept;
+  void notify_one() volatile noexcept;
+  void notify_one() noexcept;
+  void notify_all() volatile noexcept;
+  void notify_all() noexcept;
+};
 
 // [atomics.nonmembers], non-member functions
 template<class T>

libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: target={{.+}}-windows-gnu
+// ADDITIONAL_COMPILE_FLAGS(has-latomic): -latomic
+
+//   static constexpr bool is_always_lock_free = implementation-defined;
+//   bool is_lock_free() const volatile noexcept;
+//   bool is_lock_free() const noexcept;
+
+#include <atomic>
+#include <cassert>
+#include <concepts>
+
+#include "test_macros.h"
+
+template <class T>
+void test() {
+  //   static constexpr bool is_always_lock_free = implementation-defined;
+  {
+    bool r = std::atomic<T>::is_always_lock_free;
+    assert(r == __atomic_always_lock_free(sizeof(std::__cxx_atomic_impl<T>), 0));
+  }
+
+  //   bool is_lock_free() const volatile noexcept;
+  {
+    const volatile std::atomic<T> a;
+    bool r = a.is_lock_free();
+    assert(r == __cxx_atomic_is_lock_free(sizeof(std::__cxx_atomic_impl<T>)));
+  }
+
+  //   bool is_lock_free() const noexcept;
+  {
+    const std::atomic<T> a;
+    bool r = a.is_lock_free();
+    assert(r == __cxx_atomic_is_lock_free(sizeof(std::__cxx_atomic_impl<T>)));
+  }
+}
+
+int main(int, char**) {
+  test<float>();
+  test<double>();
+  test<long double>();
+
+  return 0;
+}