Skip to content

Commit aa7fe1c

Browse files
KeenutsKeenuts
authored
[SPIR-V] Fixup storage class for global private (#116636)
Adds a new address spaces: `hlsl_private`. Variables with such address space will be emitted with a `Private` storage class. This is useful for variables global to a SPIR-V module, since up to now, they were still emitted with a `Function` storage class, which is wrong. --------- Signed-off-by: Nathan Gauër <[email protected]>
1 parent 40fb74a commit aa7fe1c

File tree

20 files changed

+147
-68
lines changed

20 files changed

+147
-68
lines changed

‎clang/include/clang/Basic/AddressSpaces.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ enum class LangAS : unsigned {
5858

5959
// HLSL specific address spaces.
6060
hlsl_groupshared,
61+
hlsl_private,
6162

6263
// Wasm specific address spaces.
6364
wasm_funcref,

‎clang/lib/AST/TypePrinter.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2553,6 +2553,8 @@ std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
25532553
return "__funcref";
25542554
case LangAS::hlsl_groupshared:
25552555
return "groupshared";
2556+
case LangAS::hlsl_private:
2557+
return "hlsl_private";
25562558
default:
25572559
return std::to_string(toTargetAddressSpace(AS));
25582560
}

‎clang/lib/Basic/TargetInfo.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ static const LangASMap FakeAddrSpaceMap = {
4747
11, // ptr32_uptr
4848
12, // ptr64
4949
13, // hlsl_groupshared
50+
14, // hlsl_private
5051
20, // wasm_funcref
5152
};
5253

‎clang/lib/Basic/Targets/AArch64.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ static const unsigned ARM64AddrSpaceMap[] = {
4444
static_cast<unsigned>(AArch64AddrSpace::ptr32_uptr),
4545
static_cast<unsigned>(AArch64AddrSpace::ptr64),
4646
0, // hlsl_groupshared
47+
0, // hlsl_private
4748
// Wasm address space values for this target are dummy values,
4849
// as it is only enabled for Wasm targets.
4950
20, // wasm_funcref

‎clang/lib/Basic/Targets/AMDGPU.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
5959
llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_uptr
6060
llvm::AMDGPUAS::FLAT_ADDRESS, // ptr64
6161
llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_groupshared
62+
llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_private
6263
};
6364

6465
const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
@@ -83,6 +84,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
8384
llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_uptr
8485
llvm::AMDGPUAS::FLAT_ADDRESS, // ptr64
8586
llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_groupshared
87+
llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_private
8688

8789
};
8890
} // namespace targets

‎clang/lib/Basic/Targets/DirectX.h

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,16 @@ static const unsigned DirectXAddrSpaceMap[] = {
3333
0, // cuda_constant
3434
0, // cuda_shared
3535
// SYCL address space values for this map are dummy
36-
0, // sycl_global
37-
0, // sycl_global_device
38-
0, // sycl_global_host
39-
0, // sycl_local
40-
0, // sycl_private
41-
0, // ptr32_sptr
42-
0, // ptr32_uptr
43-
0, // ptr64
44-
3, // hlsl_groupshared
36+
0, // sycl_global
37+
0, // sycl_global_device
38+
0, // sycl_global_host
39+
0, // sycl_local
40+
0, // sycl_private
41+
0, // ptr32_sptr
42+
0, // ptr32_uptr
43+
0, // ptr64
44+
3, // hlsl_groupshared
45+
10, // hlsl_private
4546
// Wasm address space values for this target are dummy values,
4647
// as it is only enabled for Wasm targets.
4748
20, // wasm_funcref

‎clang/lib/Basic/Targets/NVPTX.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ static const unsigned NVPTXAddrSpaceMap[] = {
4646
0, // ptr32_uptr
4747
0, // ptr64
4848
0, // hlsl_groupshared
49+
0, // hlsl_private
4950
// Wasm address space values for this target are dummy values,
5051
// as it is only enabled for Wasm targets.
5152
20, // wasm_funcref

‎clang/lib/Basic/Targets/SPIR.h

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,16 @@ static const unsigned SPIRDefIsPrivMap[] = {
3838
0, // cuda_constant
3939
0, // cuda_shared
4040
// SYCL address space values for this map are dummy
41-
0, // sycl_global
42-
0, // sycl_global_device
43-
0, // sycl_global_host
44-
0, // sycl_local
45-
0, // sycl_private
46-
0, // ptr32_sptr
47-
0, // ptr32_uptr
48-
0, // ptr64
49-
0, // hlsl_groupshared
41+
0, // sycl_global
42+
0, // sycl_global_device
43+
0, // sycl_global_host
44+
0, // sycl_local
45+
0, // sycl_private
46+
0, // ptr32_sptr
47+
0, // ptr32_uptr
48+
0, // ptr64
49+
0, // hlsl_groupshared
50+
10, // hlsl_private
5051
// Wasm address space values for this target are dummy values,
5152
// as it is only enabled for Wasm targets.
5253
20, // wasm_funcref
@@ -69,17 +70,18 @@ static const unsigned SPIRDefIsGenMap[] = {
6970
// cuda_constant pointer can be casted to default/"flat" pointer, but in
7071
// SPIR-V casts between constant and generic pointers are not allowed. For
7172
// this reason cuda_constant is mapped to SPIR-V CrossWorkgroup.
72-
1, // cuda_constant
73-
3, // cuda_shared
74-
1, // sycl_global
75-
5, // sycl_global_device
76-
6, // sycl_global_host
77-
3, // sycl_local
78-
0, // sycl_private
79-
0, // ptr32_sptr
80-
0, // ptr32_uptr
81-
0, // ptr64
82-
0, // hlsl_groupshared
73+
1, // cuda_constant
74+
3, // cuda_shared
75+
1, // sycl_global
76+
5, // sycl_global_device
77+
6, // sycl_global_host
78+
3, // sycl_local
79+
0, // sycl_private
80+
0, // ptr32_sptr
81+
0, // ptr32_uptr
82+
0, // ptr64
83+
0, // hlsl_groupshared
84+
10, // hlsl_private
8385
// Wasm address space values for this target are dummy values,
8486
// as it is only enabled for Wasm targets.
8587
20, // wasm_funcref

‎clang/lib/Basic/Targets/SystemZ.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ static const unsigned ZOSAddressMap[] = {
4242
1, // ptr32_uptr
4343
0, // ptr64
4444
0, // hlsl_groupshared
45+
0, // hlsl_private
4546
0 // wasm_funcref
4647
};
4748

‎clang/lib/Basic/Targets/TCE.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ static const unsigned TCEOpenCLAddrSpaceMap[] = {
5151
0, // ptr32_uptr
5252
0, // ptr64
5353
0, // hlsl_groupshared
54+
0, // hlsl_private
5455
// Wasm address space values for this target are dummy values,
5556
// as it is only enabled for Wasm targets.
5657
20, // wasm_funcref

‎clang/lib/Basic/Targets/WebAssembly.h

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,26 +22,27 @@ namespace clang {
2222
namespace targets {
2323

2424
static const unsigned WebAssemblyAddrSpaceMap[] = {
25-
0, // Default
26-
0, // opencl_global
27-
0, // opencl_local
28-
0, // opencl_constant
29-
0, // opencl_private
30-
0, // opencl_generic
31-
0, // opencl_global_device
32-
0, // opencl_global_host
33-
0, // cuda_device
34-
0, // cuda_constant
35-
0, // cuda_shared
36-
0, // sycl_global
37-
0, // sycl_global_device
38-
0, // sycl_global_host
39-
0, // sycl_local
40-
0, // sycl_private
41-
0, // ptr32_sptr
42-
0, // ptr32_uptr
43-
0, // ptr64
44-
0, // hlsl_groupshared
25+
0, // Default
26+
0, // opencl_global
27+
0, // opencl_local
28+
0, // opencl_constant
29+
0, // opencl_private
30+
0, // opencl_generic
31+
0, // opencl_global_device
32+
0, // opencl_global_host
33+
0, // cuda_device
34+
0, // cuda_constant
35+
0, // cuda_shared
36+
0, // sycl_global
37+
0, // sycl_global_device
38+
0, // sycl_global_host
39+
0, // sycl_local
40+
0, // sycl_private
41+
0, // ptr32_sptr
42+
0, // ptr32_uptr
43+
0, // ptr64
44+
0, // hlsl_groupshared
45+
0, // hlsl_private
4546
20, // wasm_funcref
4647
};
4748

‎clang/lib/Basic/Targets/X86.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ static const unsigned X86AddrSpaceMap[] = {
4646
271, // ptr32_uptr
4747
272, // ptr64
4848
0, // hlsl_groupshared
49+
0, // hlsl_private
4950
// Wasm address space values for this target are dummy values,
5051
// as it is only enabled for Wasm targets.
5152
20, // wasm_funcref

‎clang/test/SemaTemplate/address_space-dependent.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ void neg() {
4343

4444
template <long int I>
4545
void tooBig() {
46-
__attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388586)}}
46+
__attribute__((address_space(I))) int *bounds; // expected-error {{address space is larger than the maximum supported (8388585)}}
4747
}
4848

4949
template <long int I>
@@ -102,7 +102,7 @@ int main() {
102102
HasASTemplateFields<1> HASTF;
103103
neg<-1>(); // expected-note {{in instantiation of function template specialization 'neg<-1>' requested here}}
104104
correct<0x7FFFE9>();
105-
tooBig<8388650>(); // expected-note {{in instantiation of function template specialization 'tooBig<8388650L>' requested here}}
105+
tooBig<8388651>(); // expected-note {{in instantiation of function template specialization 'tooBig<8388651L>' requested here}}
106106

107107
__attribute__((address_space(1))) char *x;
108108
__attribute__((address_space(2))) char *y;

‎llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1460,6 +1460,16 @@ bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
14601460
.addUse(SrcPtr)
14611461
.constrainAllUses(TII, TRI, RBI);
14621462

1463+
if ((SrcSC == SPIRV::StorageClass::Function &&
1464+
DstSC == SPIRV::StorageClass::Private) ||
1465+
(DstSC == SPIRV::StorageClass::Function &&
1466+
SrcSC == SPIRV::StorageClass::Private)) {
1467+
return BuildMI(BB, I, DL, TII.get(TargetOpcode::COPY))
1468+
.addDef(ResVReg)
1469+
.addUse(SrcPtr)
1470+
.constrainAllUses(TII, TRI, RBI);
1471+
}
1472+
14631473
// Casting from an eligible pointer to Generic.
14641474
if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
14651475
return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
@@ -3461,11 +3471,7 @@ bool SPIRVInstructionSelector::selectGlobalValue(
34613471
if (HasInit && !Init)
34623472
return true;
34633473

3464-
unsigned AddrSpace = GV->getAddressSpace();
3465-
SPIRV::StorageClass::StorageClass Storage =
3466-
addressSpaceToStorageClass(AddrSpace, STI);
3467-
bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage &&
3468-
Storage != SPIRV::StorageClass::Function;
3474+
bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage;
34693475
SPIRV::LinkageType::LinkageType LnkType =
34703476
(GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
34713477
? SPIRV::LinkageType::Import
@@ -3474,12 +3480,14 @@ bool SPIRVInstructionSelector::selectGlobalValue(
34743480
? SPIRV::LinkageType::LinkOnceODR
34753481
: SPIRV::LinkageType::Export);
34763482

3477-
SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
3478-
PointerBaseType, I, TII,
3479-
addressSpaceToStorageClass(GV->getAddressSpace(), STI));
3480-
Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV,
3481-
Storage, Init, GlobalVar->isConstant(),
3482-
HasLnkTy, LnkType, MIRBuilder, true);
3483+
const unsigned AddrSpace = GV->getAddressSpace();
3484+
SPIRV::StorageClass::StorageClass StorageClass =
3485+
addressSpaceToStorageClass(AddrSpace, STI);
3486+
SPIRVType *ResType =
3487+
GR.getOrCreateSPIRVPointerType(PointerBaseType, I, TII, StorageClass);
3488+
Register Reg = GR.buildGlobalVariable(
3489+
ResVReg, ResType, GlobalIdent, GV, StorageClass, Init,
3490+
GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true);
34833491
return Reg.isValid();
34843492
}
34853493

‎llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
112112
const LLT p5 =
113113
LLT::pointer(5, PSize); // Input, SPV_INTEL_usm_storage_classes (Device)
114114
const LLT p6 = LLT::pointer(6, PSize); // SPV_INTEL_usm_storage_classes (Host)
115+
const LLT p7 = LLT::pointer(7, PSize); // Input
116+
const LLT p8 = LLT::pointer(8, PSize); // Output
117+
const LLT p10 = LLT::pointer(10, PSize); // Private
115118

116119
// TODO: remove copy-pasting here by using concatenation in some way.
117120
auto allPtrsScalarsAndVectors = {
@@ -148,7 +151,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
148151
auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
149152
p2, p3, p4, p5, p6};
150153

151-
auto allPtrs = {p0, p1, p2, p3, p4, p5, p6};
154+
auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p10};
152155

153156
bool IsExtendedInts =
154157
ST.canUseExtension(

‎llvm/lib/Target/SPIRV/SPIRVUtils.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,8 +207,12 @@ addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) {
207207
: SPIRV::StorageClass::CrossWorkgroup;
208208
case 7:
209209
return SPIRV::StorageClass::Input;
210+
case 8:
211+
return SPIRV::StorageClass::Output;
210212
case 9:
211213
return SPIRV::StorageClass::CodeSectionINTEL;
214+
case 10:
215+
return SPIRV::StorageClass::Private;
212216
default:
213217
report_fatal_error("Unknown address space");
214218
}

‎llvm/lib/Target/SPIRV/SPIRVUtils.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,12 @@ storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC) {
170170
return 6;
171171
case SPIRV::StorageClass::Input:
172172
return 7;
173+
case SPIRV::StorageClass::Output:
174+
return 8;
173175
case SPIRV::StorageClass::CodeSectionINTEL:
174176
return 9;
177+
case SPIRV::StorageClass::Private:
178+
return 10;
175179
default:
176180
report_fatal_error("Unable to get address space id");
177181
}

‎llvm/test/CodeGen/SPIRV/pointers/global-addrspacecast.ll

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
2+
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
3+
4+
@PrivInternal = internal addrspace(10) global i32 456
5+
; CHECK-DAG: %[[#type:]] = OpTypeInt 32 0
6+
; CHECK-DAG: %[[#ptrty:]] = OpTypePointer Private %[[#type]]
7+
; CHECK-DAG: %[[#value:]] = OpConstant %[[#type]] 456
8+
; CHECK-DAG: %[[#var:]] = OpVariable %[[#ptrty]] Private %[[#value]]
9+
10+
define spir_kernel void @Foo() {
11+
%p = addrspacecast ptr addrspace(10) @PrivInternal to ptr
12+
%v = load i32, ptr %p, align 4
13+
ret void
14+
; CHECK: OpLabel
15+
; CHECK-NEXT: OpLoad %[[#type]] %[[#var]] Aligned 4
16+
; CHECK-Next: OpReturn
17+
}

‎llvm/test/CodeGen/SPIRV/pointers/variables-storage-class-vk.ll

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - | FileCheck %s
2+
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-compute %s -o - -filetype=obj | spirv-val %}
3+
4+
; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
5+
6+
; CHECK-DAG: %[[#VAL:]] = OpConstant %[[#U32]] 456
7+
; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer Private %[[#U32]]
8+
; CHECK-DAG: %[[#VAR:]] = OpVariable %[[#VTYPE]] Private %[[#VAL]]
9+
; CHECK-NOT: OpDecorate %[[#VAR]] LinkageAttributes
10+
@PrivInternal = internal addrspace(10) global i32 456
11+
12+
define void @main() {
13+
%l = load i32, ptr addrspace(10) @PrivInternal
14+
ret void
15+
}

‎llvm/test/CodeGen/SPIRV/pointers/variables-storage-class.ll

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,31 @@
11
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
22
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
33

4+
; CHECK-DAG: %[[#U8:]] = OpTypeInt 8 0
5+
; CHECK-DAG: %[[#U32:]] = OpTypeInt 32 0
6+
7+
; CHECK-DAG: %[[#TYPE:]] = OpTypePointer CrossWorkgroup %[[#U8]]
8+
; CHECK-DAG: %[[#VAL:]] = OpConstantNull %[[#TYPE]]
9+
; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer CrossWorkgroup %[[#TYPE]]
10+
; CHECK-DAG: %[[#PTR:]] = OpVariable %[[#VTYPE]] CrossWorkgroup %[[#VAL]]
411
@Ptr = addrspace(1) global ptr addrspace(1) null
5-
@Init = private addrspace(2) constant i32 123
612

7-
; CHECK-DAG: %[[#PTR:]] = OpVariable %[[#]] UniformConstant %[[#]]
8-
; CHECK-DAG: %[[#INIT:]] = OpVariable %[[#]] CrossWorkgroup %[[#]]
13+
; CHECK-DAG: %[[#VAL:]] = OpConstant %[[#U32]] 123
14+
; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer UniformConstant %[[#U32]]
15+
; CHECK-DAG: %[[#INIT:]] = OpVariable %[[#VTYPE]] UniformConstant %[[#VAL]]
16+
@Init = private addrspace(2) constant i32 123
917

10-
; CHECK: %[[#]] = OpLoad %[[#]] %[[#INIT]] Aligned 8
11-
; CHECK: OpCopyMemorySized %[[#]] %[[#PTR]] %[[#]] Aligned 4
18+
; CHECK-DAG: %[[#VAL:]] = OpConstant %[[#U32]] 456
19+
; CHECK-DAG: %[[#VTYPE:]] = OpTypePointer Private %[[#U32]]
20+
; CHECK-DAG: %[[#]] = OpVariable %[[#VTYPE]] Private %[[#VAL]]
21+
@PrivInternal = internal addrspace(10) global i32 456
1222

1323
define spir_kernel void @Foo() {
24+
; CHECK: %[[#]] = OpLoad %[[#]] %[[#PTR]] Aligned 8
1425
%l = load ptr addrspace(1), ptr addrspace(1) @Ptr, align 8
26+
; CHECK: OpCopyMemorySized %[[#]] %[[#INIT]] %[[#]] Aligned 4
1527
call void @llvm.memcpy.p1.p2.i64(ptr addrspace(1) align 4 %l, ptr addrspace(2) align 1 @Init, i64 4, i1 false)
28+
1629
ret void
1730
}
1831

0 commit comments

Comments
 (0)