Skip to content

Commit e60c0ac

Browse files
mariusz-sikora-at-amdmariusz-sikora-at-amd
committed
Merged main:fcaefc2c19eb into origin/amd-gfx:25b40737e2c7
Local branch origin/amd-gfx 25b4073 Merged main:ac42b083f104 into origin/amd-gfx:160270860d83 Remote branch main fcaefc2 [AMDGPU][NPM] Port SIPreEmitPeephole to NPM (llvm#130065)
2 parents 25b4073 + fcaefc2 commit e60c0ac

File tree

174 files changed

+224721
-41441
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

174 files changed

+224721
-41441
lines changed

‎.ci/metrics/metrics.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,13 @@ def github_get_metrics(
282282
queued_count = collections.Counter()
283283
running_count = collections.Counter()
284284

285+
# Initialize all the counters to 0 so we report 0 when no job is queued
286+
# or running.
287+
for wf_name, wf_metric_name in GITHUB_WORKFLOW_TO_TRACK.items():
288+
for job_name, job_metric_name in GITHUB_JOB_TO_TRACK[wf_metric_name].items():
289+
queued_count[wf_metric_name + "_" + job_metric_name] = 0
290+
running_count[wf_metric_name + "_" + job_metric_name] = 0
291+
285292
# The list of workflows this iteration will process.
286293
# MaxSize = GITHUB_WORKFLOWS_MAX_PROCESS_COUNT
287294
workflow_seen_as_completed = set()

‎bolt/include/bolt/Core/MCPlusBuilder.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -577,12 +577,12 @@ class MCPlusBuilder {
577577
return getNoRegister();
578578
}
579579

580-
/// Returns the register used as call destination, or no-register, if not
581-
/// an indirect call. Sets IsAuthenticatedInternally if the instruction
582-
/// accepts a signed pointer as its operand and authenticates it internally.
580+
/// Returns the register used as the destination of an indirect branch or call
581+
/// instruction. Sets IsAuthenticatedInternally if the instruction accepts
582+
/// a signed pointer as its operand and authenticates it internally.
583583
virtual MCPhysReg
584-
getRegUsedAsCallDest(const MCInst &Inst,
585-
bool &IsAuthenticatedInternally) const {
584+
getRegUsedAsIndirectBranchDest(const MCInst &Inst,
585+
bool &IsAuthenticatedInternally) const {
586586
llvm_unreachable("not implemented");
587587
return getNoRegister();
588588
}

‎bolt/lib/Passes/PAuthGadgetScanner.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -498,14 +498,16 @@ static std::shared_ptr<Report>
498498
shouldReportCallGadget(const BinaryContext &BC, const MCInstReference &Inst,
499499
const State &S) {
500500
static const GadgetKind CallKind("non-protected call found");
501-
if (!BC.MIB->isCall(Inst) && !BC.MIB->isBranch(Inst))
501+
if (!BC.MIB->isIndirectCall(Inst) && !BC.MIB->isIndirectBranch(Inst))
502502
return nullptr;
503503

504504
bool IsAuthenticated = false;
505-
MCPhysReg DestReg = BC.MIB->getRegUsedAsCallDest(Inst, IsAuthenticated);
506-
if (IsAuthenticated || DestReg == BC.MIB->getNoRegister())
505+
MCPhysReg DestReg =
506+
BC.MIB->getRegUsedAsIndirectBranchDest(Inst, IsAuthenticated);
507+
if (IsAuthenticated)
507508
return nullptr;
508509

510+
assert(DestReg != BC.MIB->getNoRegister());
509511
LLVM_DEBUG({
510512
traceInst(BC, "Found call inst", Inst);
511513
traceReg(BC, "Call destination reg", DestReg);

‎bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
//
1111
//===----------------------------------------------------------------------===//
1212

13+
#include "AArch64InstrInfo.h"
1314
#include "AArch64MCSymbolizer.h"
1415
#include "MCTargetDesc/AArch64AddressingModes.h"
1516
#include "MCTargetDesc/AArch64FixupKinds.h"
@@ -277,15 +278,14 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
277278
}
278279
}
279280

280-
MCPhysReg
281-
getRegUsedAsCallDest(const MCInst &Inst,
282-
bool &IsAuthenticatedInternally) const override {
283-
assert(isCall(Inst) || isBranch(Inst));
284-
IsAuthenticatedInternally = false;
281+
MCPhysReg getRegUsedAsIndirectBranchDest(
282+
const MCInst &Inst, bool &IsAuthenticatedInternally) const override {
283+
assert(isIndirectCall(Inst) || isIndirectBranch(Inst));
285284

286285
switch (Inst.getOpcode()) {
287286
case AArch64::BR:
288287
case AArch64::BLR:
288+
IsAuthenticatedInternally = false;
289289
return Inst.getOperand(0).getReg();
290290
case AArch64::BRAA:
291291
case AArch64::BRAB:
@@ -298,9 +298,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
298298
IsAuthenticatedInternally = true;
299299
return Inst.getOperand(0).getReg();
300300
default:
301-
if (isIndirectCall(Inst) || isIndirectBranch(Inst))
302-
llvm_unreachable("Unhandled indirect branch");
303-
return getNoRegister();
301+
llvm_unreachable("Unhandled indirect branch or call");
304302
}
305303
}
306304

@@ -699,7 +697,7 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
699697
}
700698

701699
bool isIndirectCall(const MCInst &Inst) const override {
702-
return Inst.getOpcode() == AArch64::BLR;
700+
return isIndirectCallOpcode(Inst.getOpcode());
703701
}
704702

705703
MCPhysReg getSpRegister(int Size) const {

‎clang/include/clang/AST/Decl.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3048,6 +3048,8 @@ class FunctionDecl : public DeclaratorDecl,
30483048
static FunctionDecl *castFromDeclContext(const DeclContext *DC) {
30493049
return static_cast<FunctionDecl *>(const_cast<DeclContext*>(DC));
30503050
}
3051+
3052+
bool isReferenceableKernel() const;
30513053
};
30523054

30533055
/// Represents a member of a struct/union/class.

‎clang/include/clang/AST/GlobalDecl.h

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,15 @@ class GlobalDecl {
7070
GlobalDecl(const VarDecl *D) { Init(D);}
7171
GlobalDecl(const FunctionDecl *D, unsigned MVIndex = 0)
7272
: MultiVersionIndex(MVIndex) {
73-
if (!D->hasAttr<CUDAGlobalAttr>()) {
74-
Init(D);
73+
if (D->isReferenceableKernel()) {
74+
Value.setPointerAndInt(D, unsigned(getDefaultKernelReference(D)));
7575
return;
7676
}
77-
Value.setPointerAndInt(D, unsigned(getDefaultKernelReference(D)));
77+
Init(D);
7878
}
7979
GlobalDecl(const FunctionDecl *D, KernelReferenceKind Kind)
8080
: Value(D, unsigned(Kind)) {
81-
assert(D->hasAttr<CUDAGlobalAttr>() && "Decl is not a GPU kernel!");
81+
assert(D->isReferenceableKernel() && "Decl is not a GPU kernel!");
8282
}
8383
GlobalDecl(const NamedDecl *D) { Init(D); }
8484
GlobalDecl(const BlockDecl *D) { Init(D); }
@@ -131,12 +131,13 @@ class GlobalDecl {
131131

132132
KernelReferenceKind getKernelReferenceKind() const {
133133
assert(((isa<FunctionDecl>(getDecl()) &&
134-
cast<FunctionDecl>(getDecl())->hasAttr<CUDAGlobalAttr>()) ||
134+
cast<FunctionDecl>(getDecl())->isReferenceableKernel()) ||
135135
(isa<FunctionTemplateDecl>(getDecl()) &&
136136
cast<FunctionTemplateDecl>(getDecl())
137137
->getTemplatedDecl()
138138
->hasAttr<CUDAGlobalAttr>())) &&
139139
"Decl is not a GPU kernel!");
140+
140141
return static_cast<KernelReferenceKind>(Value.getInt());
141142
}
142143

@@ -160,8 +161,9 @@ class GlobalDecl {
160161
}
161162

162163
static KernelReferenceKind getDefaultKernelReference(const FunctionDecl *D) {
163-
return D->getLangOpts().CUDAIsDevice ? KernelReferenceKind::Kernel
164-
: KernelReferenceKind::Stub;
164+
return (D->hasAttr<OpenCLKernelAttr>() || D->getLangOpts().CUDAIsDevice)
165+
? KernelReferenceKind::Kernel
166+
: KernelReferenceKind::Stub;
165167
}
166168

167169
GlobalDecl getWithDecl(const Decl *D) {
@@ -197,7 +199,7 @@ class GlobalDecl {
197199

198200
GlobalDecl getWithKernelReferenceKind(KernelReferenceKind Kind) {
199201
assert(isa<FunctionDecl>(getDecl()) &&
200-
cast<FunctionDecl>(getDecl())->hasAttr<CUDAGlobalAttr>() &&
202+
cast<FunctionDecl>(getDecl())->isReferenceableKernel() &&
201203
"Decl is not a GPU kernel!");
202204
GlobalDecl Result(*this);
203205
Result.Value.setInt(unsigned(Kind));

‎clang/lib/AST/ByteCode/Compiler.cpp

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6148,7 +6148,8 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
61486148

61496149
if (!this->visit(SubExpr))
61506150
return false;
6151-
if (classifyPrim(SubExpr) == PT_Ptr && !E->getType()->isArrayType())
6151+
6152+
if (classifyPrim(SubExpr) == PT_Ptr)
61526153
return this->emitNarrowPtr(E);
61536154
return true;
61546155

@@ -6800,6 +6801,10 @@ bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc,
68006801
assert(!Desc->isPrimitive());
68016802
assert(!Desc->isPrimitiveArray());
68026803

6804+
// Can happen if the decl is invalid.
6805+
if (Desc->isDummy())
6806+
return true;
6807+
68036808
// Arrays.
68046809
if (Desc->isArray()) {
68056810
const Descriptor *ElemDesc = Desc->ElemDesc;
@@ -6818,15 +6823,17 @@ bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc,
68186823
return true;
68196824
}
68206825

6821-
for (ssize_t I = Desc->getNumElems() - 1; I >= 0; --I) {
6822-
if (!this->emitConstUint64(I, Loc))
6823-
return false;
6824-
if (!this->emitArrayElemPtrUint64(Loc))
6825-
return false;
6826-
if (!this->emitDestruction(ElemDesc, Loc))
6827-
return false;
6828-
if (!this->emitPopPtr(Loc))
6829-
return false;
6826+
if (size_t N = Desc->getNumElems()) {
6827+
for (ssize_t I = N - 1; I >= 0; --I) {
6828+
if (!this->emitConstUint64(I, Loc))
6829+
return false;
6830+
if (!this->emitArrayElemPtrUint64(Loc))
6831+
return false;
6832+
if (!this->emitDestruction(ElemDesc, Loc))
6833+
return false;
6834+
if (!this->emitPopPtr(Loc))
6835+
return false;
6836+
}
68306837
}
68316838
return true;
68326839
}

‎clang/lib/AST/ByteCode/Interp.h

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2059,8 +2059,11 @@ bool OffsetHelper(InterpState &S, CodePtr OpPC, const T &Offset,
20592059
// useful thing we can do. Any other index has been diagnosed before and
20602060
// we don't get here.
20612061
if (Result == 0 && Ptr.isOnePastEnd()) {
2062-
S.Stk.push<Pointer>(Ptr.asBlockPointer().Pointee,
2063-
Ptr.asBlockPointer().Base);
2062+
if (Ptr.getFieldDesc()->isArray())
2063+
S.Stk.push<Pointer>(Ptr.atIndex(0));
2064+
else
2065+
S.Stk.push<Pointer>(Ptr.asBlockPointer().Pointee,
2066+
Ptr.asBlockPointer().Base);
20642067
return true;
20652068
}
20662069

@@ -2677,8 +2680,16 @@ inline bool ArrayElemPtr(InterpState &S, CodePtr OpPC) {
26772680
return false;
26782681
}
26792682

2680-
if (!OffsetHelper<T, ArithOp::Add>(S, OpPC, Offset, Ptr))
2681-
return false;
2683+
if (Offset.isZero()) {
2684+
if (Ptr.getFieldDesc()->isArray() && Ptr.getIndex() == 0) {
2685+
S.Stk.push<Pointer>(Ptr.atIndex(0));
2686+
} else {
2687+
S.Stk.push<Pointer>(Ptr);
2688+
}
2689+
} else {
2690+
if (!OffsetHelper<T, ArithOp::Add>(S, OpPC, Offset, Ptr))
2691+
return false;
2692+
}
26822693

26832694
return NarrowPtr(S, OpPC);
26842695
}
@@ -2693,8 +2704,16 @@ inline bool ArrayElemPtrPop(InterpState &S, CodePtr OpPC) {
26932704
return false;
26942705
}
26952706

2696-
if (!OffsetHelper<T, ArithOp::Add>(S, OpPC, Offset, Ptr))
2697-
return false;
2707+
if (Offset.isZero()) {
2708+
if (Ptr.getFieldDesc()->isArray() && Ptr.getIndex() == 0) {
2709+
S.Stk.push<Pointer>(Ptr.atIndex(0));
2710+
} else {
2711+
S.Stk.push<Pointer>(Ptr);
2712+
}
2713+
} else {
2714+
if (!OffsetHelper<T, ArithOp::Add>(S, OpPC, Offset, Ptr))
2715+
return false;
2716+
}
26982717

26992718
return NarrowPtr(S, OpPC);
27002719
}

‎clang/lib/AST/ByteCode/Pointer.h

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -200,37 +200,28 @@ class Pointer {
200200
if (isZero() || isUnknownSizeArray())
201201
return *this;
202202

203+
unsigned Base = asBlockPointer().Base;
203204
// Pointer to an array of base types - enter block.
204-
if (asBlockPointer().Base == RootPtrMark)
205+
if (Base == RootPtrMark)
205206
return Pointer(asBlockPointer().Pointee, sizeof(InlineDescriptor),
206207
Offset == 0 ? Offset : PastEndMark);
207208

208209
// Pointer is one past end - magic offset marks that.
209210
if (isOnePastEnd())
210-
return Pointer(asBlockPointer().Pointee, asBlockPointer().Base,
211-
PastEndMark);
212-
213-
// Primitive arrays are a bit special since they do not have inline
214-
// descriptors. If Offset != Base, then the pointer already points to
215-
// an element and there is nothing to do. Otherwise, the pointer is
216-
// adjusted to the first element of the array.
217-
if (inPrimitiveArray()) {
218-
if (Offset != asBlockPointer().Base)
211+
return Pointer(asBlockPointer().Pointee, Base, PastEndMark);
212+
213+
if (Offset != Base) {
214+
// If we're pointing to a primitive array element, there's nothing to do.
215+
if (inPrimitiveArray())
219216
return *this;
220-
return Pointer(asBlockPointer().Pointee, asBlockPointer().Base,
221-
Offset + sizeof(InitMapPtr));
217+
// Pointer is to a composite array element - enter it.
218+
if (Offset != Base)
219+
return Pointer(asBlockPointer().Pointee, Offset, Offset);
222220
}
223221

224-
// Pointer is to a field or array element - enter it.
225-
if (Offset != asBlockPointer().Base)
226-
return Pointer(asBlockPointer().Pointee, Offset, Offset);
227-
228-
// Enter the first element of an array.
229-
if (!getFieldDesc()->isArray())
230-
return *this;
231-
232-
const unsigned NewBase = asBlockPointer().Base + sizeof(InlineDescriptor);
233-
return Pointer(asBlockPointer().Pointee, NewBase, NewBase);
222+
// Otherwise, we're pointing to a non-array element or
223+
// are already narrowed to a composite array element. Nothing to do.
224+
return *this;
234225
}
235226

236227
/// Expands a pointer to the containing array, undoing narrowing.

‎clang/lib/AST/Decl.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5468,6 +5468,10 @@ FunctionDecl *FunctionDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
54685468
/*TrailingRequiresClause=*/{});
54695469
}
54705470

5471+
bool FunctionDecl::isReferenceableKernel() const {
5472+
return hasAttr<CUDAGlobalAttr>() || hasAttr<OpenCLKernelAttr>();
5473+
}
5474+
54715475
BlockDecl *BlockDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L) {
54725476
return new (C, DC) BlockDecl(DC, L);
54735477
}

‎clang/lib/AST/Expr.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -695,9 +695,9 @@ std::string PredefinedExpr::ComputeName(PredefinedIdentKind IK,
695695
GD = GlobalDecl(CD, Ctor_Base);
696696
else if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(ND))
697697
GD = GlobalDecl(DD, Dtor_Base);
698-
else if (ND->hasAttr<CUDAGlobalAttr>())
699-
GD = GlobalDecl(cast<FunctionDecl>(ND));
700-
else
698+
else if (auto FD = dyn_cast<FunctionDecl>(ND)) {
699+
GD = FD->isReferenceableKernel() ? GlobalDecl(FD) : GlobalDecl(ND);
700+
} else
701701
GD = GlobalDecl(ND);
702702
MC->mangleName(GD, Out);
703703

‎clang/lib/AST/ItaniumMangle.cpp

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,6 +526,7 @@ class CXXNameMangler {
526526
void mangleSourceName(const IdentifierInfo *II);
527527
void mangleRegCallName(const IdentifierInfo *II);
528528
void mangleDeviceStubName(const IdentifierInfo *II);
529+
void mangleOCLDeviceStubName(const IdentifierInfo *II);
529530
void mangleSourceNameWithAbiTags(
530531
const NamedDecl *ND, const AbiTagList *AdditionalAbiTags = nullptr);
531532
void mangleLocalName(GlobalDecl GD,
@@ -1561,8 +1562,13 @@ void CXXNameMangler::mangleUnqualifiedName(
15611562
bool IsDeviceStub =
15621563
FD && FD->hasAttr<CUDAGlobalAttr>() &&
15631564
GD.getKernelReferenceKind() == KernelReferenceKind::Stub;
1565+
bool IsOCLDeviceStub =
1566+
FD && FD->hasAttr<OpenCLKernelAttr>() &&
1567+
GD.getKernelReferenceKind() == KernelReferenceKind::Stub;
15641568
if (IsDeviceStub)
15651569
mangleDeviceStubName(II);
1570+
else if (IsOCLDeviceStub)
1571+
mangleOCLDeviceStubName(II);
15661572
else if (IsRegCall)
15671573
mangleRegCallName(II);
15681574
else
@@ -1780,6 +1786,15 @@ void CXXNameMangler::mangleDeviceStubName(const IdentifierInfo *II) {
17801786
<< II->getName();
17811787
}
17821788

1789+
void CXXNameMangler::mangleOCLDeviceStubName(const IdentifierInfo *II) {
1790+
// <source-name> ::= <positive length number> __clang_ocl_kern_imp_
1791+
// <identifier> <number> ::= [n] <non-negative decimal integer> <identifier>
1792+
// ::= <unqualified source code identifier>
1793+
StringRef OCLDeviceStubNamePrefix = "__clang_ocl_kern_imp_";
1794+
Out << II->getLength() + OCLDeviceStubNamePrefix.size()
1795+
<< OCLDeviceStubNamePrefix << II->getName();
1796+
}
1797+
17831798
void CXXNameMangler::mangleSourceName(const IdentifierInfo *II) {
17841799
// <source-name> ::= <positive length number> <identifier>
17851800
// <number> ::= [n] <non-negative decimal integer>

‎clang/lib/AST/Mangle.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -540,9 +540,9 @@ class ASTNameGenerator::Implementation {
540540
GD = GlobalDecl(CtorD, Ctor_Complete);
541541
else if (const auto *DtorD = dyn_cast<CXXDestructorDecl>(D))
542542
GD = GlobalDecl(DtorD, Dtor_Complete);
543-
else if (D->hasAttr<CUDAGlobalAttr>())
544-
GD = GlobalDecl(cast<FunctionDecl>(D));
545-
else
543+
else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
544+
GD = FD->isReferenceableKernel() ? GlobalDecl(FD) : GlobalDecl(D);
545+
} else
546546
GD = GlobalDecl(D);
547547
MC->mangleName(GD, OS);
548548
return false;

0 commit comments

Comments
 (0)