Skip to content

Commit f75c846

Browse files
SchrodingerZhuSchrodingerZhu
authored
[libc] fix atomic and apply an explicit check on unique object representations (#119715)
1 parent b21fa18 commit f75c846

File tree

4 files changed

+99
-30
lines changed

4 files changed

+99
-30
lines changed

‎libc/src/__support/CPP/atomic.h

Lines changed: 46 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_ATOMIC_H
1010
#define LLVM_LIBC_SRC___SUPPORT_CPP_ATOMIC_H
1111

12+
#include "src/__support/CPP/type_traits/has_unique_object_representations.h"
1213
#include "src/__support/macros/attributes.h"
1314
#include "src/__support/macros/config.h"
1415
#include "src/__support/macros/properties/architectures.h"
@@ -47,12 +48,11 @@ template <typename T> struct Atomic {
4748
"constructible, move constructible, copy assignable, "
4849
"and move assignable.");
4950

51+
static_assert(cpp::has_unique_object_representations_v<T>,
52+
"atomic<T> in libc only support types whose values has unique "
53+
"object representations.");
54+
5055
private:
51-
// The value stored should be appropriately aligned so that
52-
// hardware instructions used to perform atomic operations work
53-
// correctly.
54-
static constexpr int ALIGNMENT = sizeof(T) > alignof(T) ? sizeof(T)
55-
: alignof(T);
5656
// type conversion helper to avoid long c++ style casts
5757
LIBC_INLINE static int order(MemoryOrder mem_ord) {
5858
return static_cast<int>(mem_ord);
@@ -62,6 +62,17 @@ template <typename T> struct Atomic {
6262
return static_cast<int>(mem_scope);
6363
}
6464

65+
LIBC_INLINE static T *addressof(T &ref) { return __builtin_addressof(ref); }
66+
67+
// Require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to
68+
// at least their size to be potentially used lock-free.
69+
LIBC_INLINE_VAR static constexpr size_t MIN_ALIGNMENT =
70+
(sizeof(T) & (sizeof(T) - 1)) || (sizeof(T) > 16) ? 0 : sizeof(T);
71+
72+
LIBC_INLINE_VAR static constexpr size_t ALIGNMENT = alignof(T) > MIN_ALIGNMENT
73+
? alignof(T)
74+
: MIN_ALIGNMENT;
75+
6576
public:
6677
using value_type = T;
6778

@@ -87,9 +98,10 @@ template <typename T> struct Atomic {
8798
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
8899
T res;
89100
#if __has_builtin(__scoped_atomic_load)
90-
__scoped_atomic_load(&val, &res, order(mem_ord), scope(mem_scope));
101+
__scoped_atomic_load(addressof(val), addressof(res), order(mem_ord),
102+
scope(mem_scope));
91103
#else
92-
__atomic_load(&val, &res, order(mem_ord));
104+
__atomic_load(addressof(val), addressof(res), order(mem_ord));
93105
#endif
94106
return res;
95107
}
@@ -104,36 +116,39 @@ template <typename T> struct Atomic {
104116
store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
105117
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
106118
#if __has_builtin(__scoped_atomic_store)
107-
__scoped_atomic_store(&val, &rhs, order(mem_ord), scope(mem_scope));
119+
__scoped_atomic_store(addressof(val), addressof(rhs), order(mem_ord),
120+
scope(mem_scope));
108121
#else
109-
__atomic_store(&val, &rhs, order(mem_ord));
122+
__atomic_store(addressof(val), addressof(rhs), order(mem_ord));
110123
#endif
111124
}
112125

113126
// Atomic compare exchange
114127
LIBC_INLINE bool compare_exchange_strong(
115128
T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
116129
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
117-
return __atomic_compare_exchange(&val, &expected, &desired, false,
118-
order(mem_ord), order(mem_ord));
130+
return __atomic_compare_exchange(addressof(val), addressof(expected),
131+
addressof(desired), false, order(mem_ord),
132+
order(mem_ord));
119133
}
120134

121135
// Atomic compare exchange (separate success and failure memory orders)
122136
LIBC_INLINE bool compare_exchange_strong(
123137
T &expected, T desired, MemoryOrder success_order,
124138
MemoryOrder failure_order,
125139
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
126-
return __atomic_compare_exchange(&val, &expected, &desired, false,
127-
order(success_order),
128-
order(failure_order));
140+
return __atomic_compare_exchange(
141+
addressof(val), addressof(expected), addressof(desired), false,
142+
order(success_order), order(failure_order));
129143
}
130144

131145
// Atomic compare exchange (weak version)
132146
LIBC_INLINE bool compare_exchange_weak(
133147
T &expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
134148
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
135-
return __atomic_compare_exchange(&val, &expected, &desired, true,
136-
order(mem_ord), order(mem_ord));
149+
return __atomic_compare_exchange(addressof(val), addressof(expected),
150+
addressof(desired), true, order(mem_ord),
151+
order(mem_ord));
137152
}
138153

139154
// Atomic compare exchange (weak version with separate success and failure
@@ -142,20 +157,21 @@ template <typename T> struct Atomic {
142157
T &expected, T desired, MemoryOrder success_order,
143158
MemoryOrder failure_order,
144159
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
145-
return __atomic_compare_exchange(&val, &expected, &desired, true,
146-
order(success_order),
147-
order(failure_order));
160+
return __atomic_compare_exchange(
161+
addressof(val), addressof(expected), addressof(desired), true,
162+
order(success_order), order(failure_order));
148163
}
149164

150165
LIBC_INLINE T
151166
exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST,
152167
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
153168
T ret;
154169
#if __has_builtin(__scoped_atomic_exchange)
155-
__scoped_atomic_exchange(&val, &desired, &ret, order(mem_ord),
156-
scope(mem_scope));
170+
__scoped_atomic_exchange(addressof(val), addressof(desired), addressof(ret),
171+
order(mem_ord), scope(mem_scope));
157172
#else
158-
__atomic_exchange(&val, &desired, &ret, order(mem_ord));
173+
__atomic_exchange(addressof(val), addressof(desired), addressof(ret),
174+
order(mem_ord));
159175
#endif
160176
return ret;
161177
}
@@ -165,10 +181,10 @@ template <typename T> struct Atomic {
165181
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
166182
static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
167183
#if __has_builtin(__scoped_atomic_fetch_add)
168-
return __scoped_atomic_fetch_add(&val, increment, order(mem_ord),
184+
return __scoped_atomic_fetch_add(addressof(val), increment, order(mem_ord),
169185
scope(mem_scope));
170186
#else
171-
return __atomic_fetch_add(&val, increment, order(mem_ord));
187+
return __atomic_fetch_add(addressof(val), increment, order(mem_ord));
172188
#endif
173189
}
174190

@@ -177,10 +193,10 @@ template <typename T> struct Atomic {
177193
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
178194
static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
179195
#if __has_builtin(__scoped_atomic_fetch_or)
180-
return __scoped_atomic_fetch_or(&val, mask, order(mem_ord),
196+
return __scoped_atomic_fetch_or(addressof(val), mask, order(mem_ord),
181197
scope(mem_scope));
182198
#else
183-
return __atomic_fetch_or(&val, mask, order(mem_ord));
199+
return __atomic_fetch_or(addressof(val), mask, order(mem_ord));
184200
#endif
185201
}
186202

@@ -189,10 +205,10 @@ template <typename T> struct Atomic {
189205
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
190206
static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
191207
#if __has_builtin(__scoped_atomic_fetch_and)
192-
return __scoped_atomic_fetch_and(&val, mask, order(mem_ord),
208+
return __scoped_atomic_fetch_and(addressof(val), mask, order(mem_ord),
193209
scope(mem_scope));
194210
#else
195-
return __atomic_fetch_and(&val, mask, order(mem_ord));
211+
return __atomic_fetch_and(addressof(val), mask, order(mem_ord));
196212
#endif
197213
}
198214

@@ -201,10 +217,10 @@ template <typename T> struct Atomic {
201217
[[maybe_unused]] MemoryScope mem_scope = MemoryScope::DEVICE) {
202218
static_assert(cpp::is_integral_v<T>, "T must be an integral type.");
203219
#if __has_builtin(__scoped_atomic_fetch_sub)
204-
return __scoped_atomic_fetch_sub(&val, decrement, order(mem_ord),
220+
return __scoped_atomic_fetch_sub(addressof(val), decrement, order(mem_ord),
205221
scope(mem_scope));
206222
#else
207-
return __atomic_fetch_sub(&val, decrement, order(mem_ord));
223+
return __atomic_fetch_sub(addressof(val), decrement, order(mem_ord));
208224
#endif
209225
}
210226

‎libc/src/__support/CPP/type_traits.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "src/__support/CPP/type_traits/decay.h"
1919
#include "src/__support/CPP/type_traits/enable_if.h"
2020
#include "src/__support/CPP/type_traits/false_type.h"
21+
#include "src/__support/CPP/type_traits/has_unique_object_representations.h"
2122
#include "src/__support/CPP/type_traits/integral_constant.h"
2223
#include "src/__support/CPP/type_traits/invoke.h"
2324
#include "src/__support/CPP/type_traits/invoke_result.h"

‎libc/src/__support/CPP/type_traits/has_unique_object_representations.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
//===-- has_unique_object_representations type_traits ------------*- C++-*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
#ifndef LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATIONS_H
9+
#define LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATIONS_H
10+
11+
#include "src/__support/CPP/type_traits/integral_constant.h"
12+
#include "src/__support/CPP/type_traits/remove_all_extents.h"
13+
#include "src/__support/macros/config.h"
14+
15+
namespace LIBC_NAMESPACE_DECL {
16+
namespace cpp {
17+
18+
template <class T>
19+
struct has_unique_object_representations
20+
: public integral_constant<bool, __has_unique_object_representations(
21+
remove_all_extents_t<T>)> {};
22+
23+
template <class T>
24+
LIBC_INLINE_VAR constexpr bool has_unique_object_representations_v =
25+
has_unique_object_representations<T>::value;
26+
27+
} // namespace cpp
28+
} // namespace LIBC_NAMESPACE_DECL
29+
30+
#endif // LLVM_LIBC_SRC___SUPPORT_CPP_TYPE_TRAITS_HAS_UNIQUE_OBJECT_REPRESENTATIONS_H

‎libc/test/src/__support/CPP/type_traits_test.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,28 @@ TEST(LlvmLibcTypeTraitsTest, is_object) {
439439

440440
TEST(LlvmLibcTypeTraitsTest, true_type) { EXPECT_TRUE((true_type::value)); }
441441

442+
struct CompilerLeadingPadded {
443+
char b;
444+
int a;
445+
};
446+
447+
struct CompilerTrailingPadded {
448+
int a;
449+
char b;
450+
};
451+
452+
struct alignas(long long) ManuallyPadded {
453+
int b;
454+
char padding[sizeof(long long) - sizeof(int)];
455+
};
456+
457+
TEST(LlvmLibcTypeTraitsTest, has_unique_object_representations) {
458+
EXPECT_TRUE(has_unique_object_representations<int>::value);
459+
EXPECT_FALSE(has_unique_object_representations_v<CompilerLeadingPadded>);
460+
EXPECT_FALSE(has_unique_object_representations_v<CompilerTrailingPadded>);
461+
EXPECT_TRUE(has_unique_object_representations_v<ManuallyPadded>);
462+
}
463+
442464
// TODO type_identity
443465

444466
// TODO void_t

0 commit comments

Comments
 (0)