Skip to content

Commit 99bfd52

Browse files
MeinersburMeinersbur
authored
Merge pull request #25 from RWTH-HPC/archer-improvements
Various improvements of data race analysis in Archer / ThreadSanitizer
2 parents f8575ff + f8c968f commit 99bfd52

File tree

59 files changed

+2596
-150
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2596
-150
lines changed

‎clang/lib/CodeGen/CGOpenMPRuntime.cpp

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5019,13 +5019,16 @@ llvm::Function *CGOpenMPRuntime::emitReductionFunction(
50195019
Args.push_back(&RHSArg);
50205020
const auto &CGFI =
50215021
CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
5022+
CodeGenFunction CGF(CGM);
50225023
std::string Name = getReductionFuncName(ReducerName);
50235024
auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI),
50245025
llvm::GlobalValue::InternalLinkage, Name,
50255026
&CGM.getModule());
5027+
if (CGF.SanOpts.has(SanitizerKind::Thread)) {
5028+
return Fn;
5029+
}
50265030
CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI);
50275031
Fn->setDoesNotRecurse();
5028-
CodeGenFunction CGF(CGM);
50295032
CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc);
50305033

50315034
// Dst = (void*[n])(LHSArg);
@@ -5217,6 +5220,11 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
52175220
llvm::Function *ReductionFn = emitReductionFunction(
52185221
CGF.CurFn->getName(), Loc, CGF.ConvertTypeForMem(ReductionArrayTy),
52195222
Privates, LHSExprs, RHSExprs, ReductionOps);
5223+
llvm::Value *ReductionFnP = ReductionFn;
5224+
if (CGF.SanOpts.has(SanitizerKind::Thread)) {
5225+
ReductionFnP = llvm::ConstantPointerNull::get(
5226+
llvm::PointerType::get(ReductionFn->getFunctionType(), 0));
5227+
}
52205228

52215229
// 3. Create static kmp_critical_name lock = { 0 };
52225230
std::string Name = getName({"reduction"});
@@ -5235,8 +5243,8 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
52355243
CGF.Builder.getInt32(RHSExprs.size()), // i32 <n>
52365244
ReductionArrayTySize, // size_type sizeof(RedList)
52375245
RL, // void *RedList
5238-
ReductionFn, // void (*) (void *, void *) <reduce_func>
5239-
Lock // kmp_critical_name *&<lock>
5246+
ReductionFnP, // void (*) (void *, void *) <reduce_func>
5247+
Lock // kmp_critical_name *&<lock>
52405248
};
52415249
llvm::Value *Res = CGF.EmitRuntimeCall(
52425250
OMPBuilder.getOrCreateRuntimeFunction(

‎compiler-rt/cmake/config-ix.cmake

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ check_cxx_compiler_flag(-fno-profile-instr-use COMPILER_RT_HAS_FNO_PROFILE_INSTR
8989
check_cxx_compiler_flag(-fno-coverage-mapping COMPILER_RT_HAS_FNO_COVERAGE_MAPPING_FLAG)
9090
check_cxx_compiler_flag("-Werror -mcrc32" COMPILER_RT_HAS_MCRC32_FLAG)
9191
check_cxx_compiler_flag("-Werror -msse4.2" COMPILER_RT_HAS_MSSE4_2_FLAG)
92+
check_cxx_compiler_flag("-Werror -mavx2" COMPILER_RT_HAS_MAVX2_FLAG)
93+
check_cxx_compiler_flag("-Werror -mavx512f" COMPILER_RT_HAS_MAVX512F_FLAG)
9294
check_cxx_compiler_flag(--sysroot=. COMPILER_RT_HAS_SYSROOT_FLAG)
9395
check_cxx_compiler_flag("-Werror -mcrc" COMPILER_RT_HAS_MCRC_FLAG)
9496
check_cxx_compiler_flag(-fno-partial-inlining COMPILER_RT_HAS_FNO_PARTIAL_INLINING_FLAG)

‎compiler-rt/lib/tsan/rtl/CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,17 @@ else()
237237
else()
238238
set(TSAN_ASM_SOURCES)
239239
endif()
240+
add_compiler_rt_object_libraries(RTTSanAVX2
241+
ARCHS ${arch}
242+
SOURCES tsan_interface_avx2.cpp
243+
ADDITIONAL_HEADERS tsan_interface_avx2.h
244+
#CFLAGS ${TSAN_RTL_CFLAGS} $<IF:"$COMPILER_RT_HAS_MAVX2_FLAG","-mavx2","">)
245+
CFLAGS ${TSAN_RTL_CFLAGS} $<IF:$<BOOL:${COMPILER_RT_HAS_MAVX2_FLAG}>,-mavx2,"">)
246+
add_compiler_rt_object_libraries(RTTSanAVX512
247+
ARCHS ${arch}
248+
SOURCES tsan_interface_avx512.cpp
249+
ADDITIONAL_HEADERS tsan_interface_avx512.h
250+
CFLAGS ${TSAN_RTL_CFLAGS} $<IF:$<BOOL:${COMPILER_RT_HAS_MAVX512F_FLAG}>,-mavx512f,"">)
240251
add_compiler_rt_runtime(clang_rt.tsan
241252
STATIC
242253
ARCHS ${arch}
@@ -247,6 +258,8 @@ else()
247258
$<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
248259
$<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
249260
$<TARGET_OBJECTS:RTUbsan.${arch}>
261+
$<TARGET_OBJECTS:RTTSanAVX2.${arch}>
262+
$<TARGET_OBJECTS:RTTSanAVX512.${arch}>
250263
ADDITIONAL_HEADERS ${TSAN_HEADERS}
251264
CFLAGS ${TSAN_RTL_CFLAGS}
252265
PARENT_TARGET tsan)
@@ -270,6 +283,8 @@ else()
270283
$<TARGET_OBJECTS:RTSanitizerCommonCoverage.${arch}>
271284
$<TARGET_OBJECTS:RTSanitizerCommonSymbolizer.${arch}>
272285
$<TARGET_OBJECTS:RTUbsan.${arch}>
286+
$<TARGET_OBJECTS:RTTSanAVX2.${arch}>
287+
$<TARGET_OBJECTS:RTTSanAVX512.${arch}>
273288
ADDITIONAL_HEADERS ${TSAN_HEADERS}
274289
CFLAGS ${TSAN_RTL_DYNAMIC_CFLAGS}
275290
DEFS SANITIZER_SHARED

‎compiler-rt/lib/tsan/rtl/tsan_interface.cpp

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "tsan_interface.h"
1414
#include "tsan_interface_ann.h"
1515
#include "tsan_rtl.h"
16+
1617
#include "sanitizer_common/sanitizer_internal_defs.h"
1718
#include "sanitizer_common/sanitizer_ptrauth.h"
1819

@@ -42,18 +43,42 @@ void __tsan_write16_pc(void *addr, void *pc) {
4243

4344
// __tsan_unaligned_read/write calls are emitted by compiler.
4445

45-
void __tsan_unaligned_read16(const void *addr) {
46+
template <unsigned int N>
47+
void __tsan_unaligned_readx(const void *addr) {
4648
uptr pc = CALLERPC;
4749
ThreadState *thr = cur_thread();
48-
UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessRead);
49-
UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessRead);
50+
for (unsigned int i = 0; i < N / 8; i++)
51+
UnalignedMemoryAccess(thr, pc, (uptr)addr + (i * 8), 8, kAccessRead);
5052
}
5153

52-
void __tsan_unaligned_write16(void *addr) {
54+
template <unsigned int N>
55+
void __tsan_unaligned_writex(void *addr) {
5356
uptr pc = CALLERPC;
5457
ThreadState *thr = cur_thread();
55-
UnalignedMemoryAccess(thr, pc, (uptr)addr, 8, kAccessWrite);
56-
UnalignedMemoryAccess(thr, pc, (uptr)addr + 8, 8, kAccessWrite);
58+
for (unsigned int i = 0; i < N / 8; i++)
59+
UnalignedMemoryAccess(thr, pc, (uptr)addr + (i * 8), 8, kAccessWrite);
60+
}
61+
62+
void __tsan_unaligned_read16(const void *addr) {
63+
__tsan_unaligned_readx<16>(addr);
64+
}
65+
66+
void __tsan_unaligned_write16(void *addr) { __tsan_unaligned_writex<16>(addr); }
67+
68+
extern "C" void __tsan_unaligned_read32(const void *addr) {
69+
__tsan_unaligned_readx<32>(addr);
70+
}
71+
72+
extern "C" void __tsan_unaligned_write32(void *addr) {
73+
__tsan_unaligned_writex<32>(addr);
74+
}
75+
76+
extern "C" void __tsan_unaligned_read64(const void *addr) {
77+
__tsan_unaligned_readx<64>(addr);
78+
}
79+
80+
extern "C" void __tsan_unaligned_write64(void *addr) {
81+
__tsan_unaligned_writex<64>(addr);
5782
}
5883

5984
extern "C" {

‎compiler-rt/lib/tsan/rtl/tsan_interface.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,15 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read2(const void *addr);
5353
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read4(const void *addr);
5454
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read8(const void *addr);
5555
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read16(const void *addr);
56+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read32(const void *addr);
57+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read64(const void *addr);
5658

5759
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(void *addr);
5860
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(void *addr);
5961
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(void *addr);
6062
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write16(void *addr);
63+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write32(void *addr);
64+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write64(void *addr);
6165

6266
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read1_pc(void *addr, void *pc);
6367
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_read2_pc(void *addr, void *pc);

‎compiler-rt/lib/tsan/rtl/tsan_interface.inc

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,18 @@ void __tsan_read16(void *addr) {
3838
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
3939
}
4040

41+
extern "C" void __tsan_read32(void *addr) {
42+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
43+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessRead);
44+
}
45+
46+
extern "C" void __tsan_read64(void *addr) {
47+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessRead);
48+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessRead);
49+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 32, kAccessRead);
50+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 48, kAccessRead);
51+
}
52+
4153
void __tsan_write1(void *addr) {
4254
MemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 1, kAccessWrite);
4355
}
@@ -58,6 +70,21 @@ void __tsan_write16(void *addr) {
5870
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
5971
}
6072

73+
extern "C" void __tsan_write32(void *addr) {
74+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
75+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessWrite);
76+
}
77+
78+
extern "C" void __tsan_write64(void *addr) {
79+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr, kAccessWrite);
80+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 16, kAccessWrite);
81+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 32, kAccessWrite);
82+
MemoryAccess16(cur_thread(), CALLERPC, (uptr)addr + 48, kAccessWrite);
83+
}
84+
85+
// Our vector instructions
86+
// TODO
87+
6188
void __tsan_read1_pc(void *addr, void *pc) {
6289
MemoryAccess(cur_thread(), STRIP_PAC_PC(pc), (uptr)addr, 1, kAccessRead | kAccessExternalPC);
6390
}

‎compiler-rt/lib/tsan/rtl/tsan_interface_ann.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,16 @@ void INTERFACE_ATTRIBUTE AnnotateBenignRace(
266266
BenignRaceImpl(f, l, mem, 1, desc);
267267
}
268268

269+
void INTERFACE_ATTRIBUTE AnnotateAllAtomicBegin(char *f, int l) {
270+
SCOPED_ANNOTATION(AnnotateAllAtomicBegin);
271+
ThreadAtomicBegin(thr, pc);
272+
}
273+
274+
void INTERFACE_ATTRIBUTE AnnotateAllAtomicEnd(char *f, int l) {
275+
SCOPED_ANNOTATION(AnnotateAllAtomicEnd);
276+
ThreadAtomicEnd(thr);
277+
}
278+
269279
void INTERFACE_ATTRIBUTE AnnotateIgnoreReadsBegin(char *f, int l) {
270280
SCOPED_ANNOTATION(AnnotateIgnoreReadsBegin);
271281
ThreadIgnoreBegin(thr, pc);

‎compiler-rt/lib/tsan/rtl/tsan_interface_avx2.cpp

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#include "tsan_interface_avx2.h"
2+
3+
#include <immintrin.h>
4+
#include <inttypes.h>
5+
#include <stdint.h>
6+
#include <unistd.h>
7+
8+
#include "sanitizer_common/sanitizer_internal_defs.h"
9+
#include "sanitizer_common/sanitizer_ptrauth.h"
10+
#include "tsan_interface_ann.h"
11+
#include "tsan_rtl.h"
12+
13+
#define CALLERPC ((uptr)__builtin_return_address(0))
14+
15+
using namespace __tsan;
16+
17+
#ifdef __AVX__
18+
extern "C" void __tsan_scatter_vector4(__m256i vaddr, int size, uint8_t mask) {
19+
void *addr[4] = {};
20+
_mm256_store_si256((__m256i *)addr, vaddr);
21+
uptr pc = CALLERPC;
22+
ThreadState *thr = cur_thread();
23+
for (int i = 0; i < 4; i++)
24+
if ((mask >> i) & 1)
25+
UnalignedMemoryAccess(thr, pc, (uptr)addr[i], size, kAccessWrite);
26+
}
27+
28+
extern "C" void __tsan_gather_vector4(__m256i vaddr, int size, uint8_t mask) {
29+
void *addr[4] = {};
30+
_mm256_store_si256((__m256i *)addr, vaddr);
31+
uptr pc = CALLERPC;
32+
ThreadState *thr = cur_thread();
33+
for (int i = 0; i < 4; i++)
34+
if ((mask >> i) & 1)
35+
UnalignedMemoryAccess(thr, pc, (uptr)addr[i], size, kAccessRead);
36+
}
37+
#endif /*__AVX__*/

‎compiler-rt/lib/tsan/rtl/tsan_interface_avx2.h

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
//===-- tsan_interface_avx2.h ----------------------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// This file is a part of ThreadSanitizer (TSan), a race detector.
10+
//
11+
// The functions declared in this header will be inserted by the instrumentation
12+
// module.
13+
// This header can be included by the instrumented program or by TSan tests.
14+
//===----------------------------------------------------------------------===//
15+
#ifndef TSAN_INTERFACE_AVX2_H
16+
#define TSAN_INTERFACE_AVX2_H
17+
18+
#include <immintrin.h>
19+
#include <sanitizer_common/sanitizer_internal_defs.h>
20+
#include <stdint.h>
21+
using __sanitizer::tid_t;
22+
using __sanitizer::uptr;
23+
24+
// This header should NOT include any other headers.
25+
// All functions in this header are extern "C" and start with __tsan_.
26+
27+
#ifdef __cplusplus
28+
extern "C" {
29+
#endif
30+
31+
#if !SANITIZER_GO
32+
# ifdef __AVX__
33+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_scatter_vector4(__m256i vaddr,
34+
int width,
35+
uint8_t mask);
36+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_gather_vector4(__m256i vaddr,
37+
int width,
38+
uint8_t mask);
39+
# endif /*__AVX__*/
40+
#endif // SANITIZER_GO
41+
42+
#ifdef __cplusplus
43+
} // extern "C"
44+
#endif
45+
46+
#endif /*TSAN_INTERFACE_AVX2_H*/

‎compiler-rt/lib/tsan/rtl/tsan_interface_avx512.cpp

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#include "tsan_interface_avx512.h"
2+
3+
#include <immintrin.h>
4+
#include <inttypes.h>
5+
#include <stdint.h>
6+
#include <unistd.h>
7+
8+
#include "sanitizer_common/sanitizer_internal_defs.h"
9+
#include "sanitizer_common/sanitizer_ptrauth.h"
10+
#include "tsan_interface_ann.h"
11+
#include "tsan_rtl.h"
12+
13+
#define CALLERPC ((uptr)__builtin_return_address(0))
14+
15+
using namespace __tsan;
16+
17+
#ifdef __AVX512F__
18+
extern "C" void __tsan_scatter_vector8(__m512i vaddr, int size, uint8_t mask) {
19+
void *addr[8] = {};
20+
__m256i v256_1 = _mm512_extracti64x4_epi64(vaddr, 0);
21+
__m256i v256_2 = _mm512_extracti64x4_epi64(vaddr, 4);
22+
_mm256_store_si256((__m256i *)addr, v256_1);
23+
_mm256_store_si256((__m256i *)&(addr[4]), v256_2);
24+
uptr pc = CALLERPC;
25+
ThreadState *thr = cur_thread();
26+
for (int i = 0; i < 8; i++)
27+
if ((mask >> i) & 1)
28+
UnalignedMemoryAccess(thr, pc, (uptr)addr[i], size, kAccessWrite);
29+
}
30+
31+
extern "C" void __tsan_gather_vector8(__m512i vaddr, int size, uint8_t mask) {
32+
void *addr[8] = {};
33+
__m256i v256_1 = _mm512_extracti64x4_epi64(vaddr, 0);
34+
__m256i v256_2 = _mm512_extracti64x4_epi64(vaddr, 4);
35+
_mm256_store_si256((__m256i *)addr, v256_1);
36+
_mm256_store_si256((__m256i *)(&addr[4]), v256_2);
37+
uptr pc = CALLERPC;
38+
ThreadState *thr = cur_thread();
39+
for (int i = 0; i < 8; i++)
40+
if ((mask >> i) & 1)
41+
UnalignedMemoryAccess(thr, pc, (uptr)addr[i], size, kAccessRead);
42+
}
43+
#endif /*__AVX512F__*/

‎compiler-rt/lib/tsan/rtl/tsan_interface_avx512.h

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
//===-- tsan_interface_avx512.h ----------------------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
//
9+
// This file is a part of ThreadSanitizer (TSan), a race detector.
10+
//
11+
// The functions declared in this header will be inserted by the instrumentation
12+
// module.
13+
// This header can be included by the instrumented program or by TSan tests.
14+
//===----------------------------------------------------------------------===//
15+
#ifndef TSAN_INTERFACE_AVX512_H
16+
#define TSAN_INTERFACE_AVX512_H
17+
18+
#include <immintrin.h>
19+
#include <sanitizer_common/sanitizer_internal_defs.h>
20+
#include <stdint.h>
21+
using __sanitizer::tid_t;
22+
using __sanitizer::uptr;
23+
24+
// This header should NOT include any other headers.
25+
// All functions in this header are extern "C" and start with __tsan_.
26+
27+
#ifdef __cplusplus
28+
extern "C" {
29+
#endif
30+
31+
#if !SANITIZER_GO
32+
# ifdef __AVX512F__
33+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_scatter_vector8(__m512i vaddr,
34+
int width,
35+
uint8_t mask);
36+
SANITIZER_INTERFACE_ATTRIBUTE void __tsan_gather_vector8(__m512i vaddr,
37+
int width,
38+
uint8_t mask);
39+
# endif /*__AVX512F__*/
40+
#endif // SANITIZER_GO
41+
42+
#ifdef __cplusplus
43+
} // extern "C"
44+
#endif
45+
46+
#endif /*TSAN_INTERFACE_AVX512_H*/

0 commit comments

Comments
 (0)