Skip to content

Commit d5a9cfe

Browse files
harviniriawandavemgreen
harviniriawan
authored and
davemgreen
committed
[SelectionDAG] Update for scalable MemoryType in MMO
Remove getSizeOrUnknown call when MachineMemOperand is created. For Scalable TypeSize, the MemoryType created becomes a scalable_vector. 2 MMOs that have scalable memory access can then use the updated BasicAA that understands scalable LocationSize
1 parent 3e8b175 commit d5a9cfe

File tree

11 files changed

+198
-77
lines changed

11 files changed

+198
-77
lines changed

‎llvm/include/llvm/Analysis/MemoryLocation.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -291,8 +291,9 @@ class MemoryLocation {
291291
return MemoryLocation(Ptr, LocationSize::beforeOrAfterPointer(), AATags);
292292
}
293293

294-
// Return the exact size if the exact size is known at compiletime,
295-
// otherwise return MemoryLocation::UnknownSize.
294+
// TODO: Remove getSizeOrUnknown
295+
// interface once llvm/unittests/CodeGen/SelectionDAGAddressAnalysisTest is
296+
// updated
296297
static uint64_t getSizeOrUnknown(const TypeSize &T) {
297298
return T.isScalable() ? UnknownSize : T.getFixedValue();
298299
}
@@ -303,6 +304,10 @@ class MemoryLocation {
303304
const AAMDNodes &AATags = AAMDNodes())
304305
: Ptr(Ptr), Size(Size), AATags(AATags) {}
305306

307+
explicit MemoryLocation(const Value *Ptr, TypeSize Size,
308+
const AAMDNodes &AATags = AAMDNodes())
309+
: Ptr(Ptr), Size(LocationSize::precise(Size)), AATags(AATags) {}
310+
306311
MemoryLocation getWithNewPtr(const Value *NewPtr) const {
307312
MemoryLocation Copy(*this);
308313
Copy.Ptr = NewPtr;

‎llvm/include/llvm/CodeGen/MachineFunction.h

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,6 +1032,13 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
10321032
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
10331033
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
10341034

1035+
MachineMemOperand *getMachineMemOperand(
1036+
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
1037+
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
1038+
const MDNode *Ranges = nullptr, SyncScope::ID SSID = SyncScope::System,
1039+
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
1040+
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
1041+
10351042
MachineMemOperand *getMachineMemOperand(
10361043
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
10371044
Align base_alignment, const AAMDNodes &AAInfo = AAMDNodes(),
@@ -1051,13 +1058,27 @@ class LLVM_EXTERNAL_VISIBILITY MachineFunction {
10511058
MMO, Offset, Size == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * Size));
10521059
}
10531060

1061+
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
1062+
int64_t Offset, TypeSize ts) {
1063+
return getMachineMemOperand(
1064+
MMO, Offset,
1065+
ts.getKnownMinValue() == ~UINT64_C(0)
1066+
? LLT()
1067+
: ts.isScalable()
1068+
? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
1069+
: LLT::scalar(8 * ts.getKnownMinValue()));
1070+
}
1071+
10541072
/// getMachineMemOperand - Allocate a new MachineMemOperand by copying
10551073
/// an existing one, replacing only the MachinePointerInfo and size.
10561074
/// MachineMemOperands are owned by the MachineFunction and need not be
10571075
/// explicitly deallocated.
10581076
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
10591077
const MachinePointerInfo &PtrInfo,
10601078
uint64_t Size);
1079+
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
1080+
const MachinePointerInfo &PtrInfo,
1081+
TypeSize ts);
10611082
MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO,
10621083
const MachinePointerInfo &PtrInfo,
10631084
LLT Ty);

‎llvm/include/llvm/CodeGen/MachineMemOperand.h

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,12 @@ class MachineMemOperand {
192192
SyncScope::ID SSID = SyncScope::System,
193193
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
194194
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
195+
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, TypeSize ts,
196+
Align a, const AAMDNodes &AAInfo = AAMDNodes(),
197+
const MDNode *Ranges = nullptr,
198+
SyncScope::ID SSID = SyncScope::System,
199+
AtomicOrdering Ordering = AtomicOrdering::NotAtomic,
200+
AtomicOrdering FailureOrdering = AtomicOrdering::NotAtomic);
195201
MachineMemOperand(MachinePointerInfo PtrInfo, Flags flags, LLT type, Align a,
196202
const AAMDNodes &AAInfo = AAMDNodes(),
197203
const MDNode *Ranges = nullptr,
@@ -235,13 +241,15 @@ class MachineMemOperand {
235241
LLT getMemoryType() const { return MemoryType; }
236242

237243
/// Return the size in bytes of the memory reference.
238-
uint64_t getSize() const {
239-
return MemoryType.isValid() ? MemoryType.getSizeInBytes() : ~UINT64_C(0);
244+
TypeSize getSize() const {
245+
return MemoryType.isValid() ? MemoryType.getSizeInBytes()
246+
: TypeSize::Fixed(~UINT64_C(0));
240247
}
241248

242249
/// Return the size in bits of the memory reference.
243-
uint64_t getSizeInBits() const {
244-
return MemoryType.isValid() ? MemoryType.getSizeInBits() : ~UINT64_C(0);
250+
TypeSize getSizeInBits() const {
251+
return MemoryType.isValid() ? MemoryType.getSizeInBits()
252+
: TypeSize::Fixed(~UINT64_C(0));
245253
}
246254

247255
LLT getType() const {

‎llvm/lib/CodeGen/GlobalISel/LoadStoreOpt.cpp

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
196196
bool IsAtomic;
197197
Register BasePtr;
198198
int64_t Offset;
199-
uint64_t NumBytes;
199+
TypeSize NumBytes;
200200
MachineMemOperand *MMO;
201201
};
202202

@@ -212,16 +212,15 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
212212
Offset = 0;
213213
}
214214

215-
uint64_t Size = MemoryLocation::getSizeOrUnknown(
216-
LS->getMMO().getMemoryType().getSizeInBytes());
215+
TypeSize Size = LS->getMMO().getMemoryType().getSizeInBytes();
217216
return {LS->isVolatile(), LS->isAtomic(), BaseReg,
218217
Offset /*base offset*/, Size, &LS->getMMO()};
219218
}
220219
// FIXME: support recognizing lifetime instructions.
221220
// Default.
222221
return {false /*isvolatile*/,
223222
/*isAtomic*/ false, Register(),
224-
(int64_t)0 /*offset*/, 0 /*size*/,
223+
(int64_t)0 /*offset*/, TypeSize::getFixed(0) /*size*/,
225224
(MachineMemOperand *)nullptr};
226225
};
227226
MemUseCharacteristics MUC0 = getCharacteristics(&MI),
@@ -249,10 +248,20 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
249248
return false;
250249
}
251250

251+
// if NumBytes is scalable and offset is not 0, conservatively return may
252+
// alias
253+
if ((MUC0.NumBytes.isScalable() && (MUC0.Offset != 0)) ||
254+
(MUC1.NumBytes.isScalable() && (MUC1.Offset != 0)))
255+
return true;
256+
257+
const bool BothNotScalable =
258+
!(MUC0.NumBytes.isScalable() || MUC1.NumBytes.isScalable());
259+
252260
// Try to prove that there is aliasing, or that there is no aliasing. Either
253261
// way, we can return now. If nothing can be proved, proceed with more tests.
254262
bool IsAlias;
255-
if (GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
263+
if (BothNotScalable &&
264+
GISelAddressing::aliasIsKnownForLoadStore(MI, Other, IsAlias, MRI))
256265
return IsAlias;
257266

258267
// The following all rely on MMO0 and MMO1 being valid.
@@ -262,19 +271,23 @@ bool GISelAddressing::instMayAlias(const MachineInstr &MI,
262271
// FIXME: port the alignment based alias analysis from SDAG's isAlias().
263272
int64_t SrcValOffset0 = MUC0.MMO->getOffset();
264273
int64_t SrcValOffset1 = MUC1.MMO->getOffset();
265-
uint64_t Size0 = MUC0.NumBytes;
266-
uint64_t Size1 = MUC1.NumBytes;
274+
TypeSize Size0 = MUC0.NumBytes;
275+
TypeSize Size1 = MUC1.NumBytes;
267276
if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
268277
Size0 != MemoryLocation::UnknownSize &&
269278
Size1 != MemoryLocation::UnknownSize) {
270279
// Use alias analysis information.
271280
int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
272-
int64_t Overlap0 = Size0 + SrcValOffset0 - MinOffset;
273-
int64_t Overlap1 = Size1 + SrcValOffset1 - MinOffset;
274-
if (AA->isNoAlias(MemoryLocation(MUC0.MMO->getValue(), Overlap0,
275-
MUC0.MMO->getAAInfo()),
276-
MemoryLocation(MUC1.MMO->getValue(), Overlap1,
277-
MUC1.MMO->getAAInfo())))
281+
int64_t Overlap0 = Size0.getKnownMinValue() + SrcValOffset0 - MinOffset;
282+
int64_t Overlap1 = Size1.getKnownMinValue() + SrcValOffset1 - MinOffset;
283+
LocationSize Loc0 = Size0.isScalable() ? LocationSize::precise(Size0)
284+
: LocationSize::precise(Overlap0);
285+
LocationSize Loc1 = Size1.isScalable() ? LocationSize::precise(Size1)
286+
: LocationSize::precise(Overlap1);
287+
288+
if (AA->isNoAlias(
289+
MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
290+
MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
278291
return false;
279292
}
280293

‎llvm/lib/CodeGen/MachineFunction.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -492,6 +492,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
492492
SSID, Ordering, FailureOrdering);
493493
}
494494

495+
MachineMemOperand *MachineFunction::getMachineMemOperand(
496+
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, TypeSize ts,
497+
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
498+
SyncScope::ID SSID, AtomicOrdering Ordering,
499+
AtomicOrdering FailureOrdering) {
500+
return new (Allocator)
501+
MachineMemOperand(PtrInfo, f, ts, base_alignment, AAInfo, Ranges, SSID,
502+
Ordering, FailureOrdering);
503+
}
504+
495505
MachineMemOperand *MachineFunction::getMachineMemOperand(
496506
MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy,
497507
Align base_alignment, const AAMDNodes &AAInfo, const MDNode *Ranges,
@@ -510,6 +520,16 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
510520
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
511521
}
512522

523+
MachineMemOperand *
524+
MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
525+
const MachinePointerInfo &PtrInfo,
526+
TypeSize ts) {
527+
return new (Allocator)
528+
MachineMemOperand(PtrInfo, MMO->getFlags(), ts, MMO->getBaseAlign(),
529+
AAMDNodes(), nullptr, MMO->getSyncScopeID(),
530+
MMO->getSuccessOrdering(), MMO->getFailureOrdering());
531+
}
532+
513533
MachineMemOperand *MachineFunction::getMachineMemOperand(
514534
const MachineMemOperand *MMO, const MachinePointerInfo &PtrInfo, LLT Ty) {
515535
return new (Allocator)

‎llvm/lib/CodeGen/MachineInstr.cpp

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1302,10 +1302,11 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
13021302
int64_t OffsetB = MMOb->getOffset();
13031303
int64_t MinOffset = std::min(OffsetA, OffsetB);
13041304

1305-
uint64_t WidthA = MMOa->getSize();
1306-
uint64_t WidthB = MMOb->getSize();
1307-
bool KnownWidthA = WidthA != MemoryLocation::UnknownSize;
1308-
bool KnownWidthB = WidthB != MemoryLocation::UnknownSize;
1305+
TypeSize WidthA = MMOa->getSize();
1306+
TypeSize WidthB = MMOb->getSize();
1307+
bool KnownWidthA = WidthA.getKnownMinValue() != MemoryLocation::UnknownSize;
1308+
bool KnownWidthB = WidthB.getKnownMinValue() != MemoryLocation::UnknownSize;
1309+
bool BothMMONonScalable = !WidthA.isScalable() && !WidthB.isScalable();
13091310

13101311
const Value *ValA = MMOa->getValue();
13111312
const Value *ValB = MMOb->getValue();
@@ -1321,11 +1322,12 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
13211322
SameVal = true;
13221323
}
13231324

1324-
if (SameVal) {
1325+
if (SameVal && BothMMONonScalable) {
13251326
if (!KnownWidthA || !KnownWidthB)
13261327
return true;
13271328
int64_t MaxOffset = std::max(OffsetA, OffsetB);
1328-
int64_t LowWidth = (MinOffset == OffsetA) ? WidthA : WidthB;
1329+
int64_t LowWidth = (MinOffset == OffsetA) ? WidthA.getKnownMinValue()
1330+
: WidthB.getKnownMinValue();
13291331
return (MinOffset + LowWidth > MaxOffset);
13301332
}
13311333

@@ -1338,15 +1340,30 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA,
13381340
assert((OffsetA >= 0) && "Negative MachineMemOperand offset");
13391341
assert((OffsetB >= 0) && "Negative MachineMemOperand offset");
13401342

1341-
int64_t OverlapA =
1342-
KnownWidthA ? WidthA + OffsetA - MinOffset : MemoryLocation::UnknownSize;
1343-
int64_t OverlapB =
1344-
KnownWidthB ? WidthB + OffsetB - MinOffset : MemoryLocation::UnknownSize;
1343+
// If Scalable Location Size has non-zero offset,
1344+
// Width + Offset does not work at the moment
1345+
if ((WidthA.isScalable() && (OffsetA > 0)) ||
1346+
(WidthB.isScalable() && (OffsetB > 0))) {
1347+
return true;
1348+
}
1349+
1350+
int64_t OverlapA = KnownWidthA
1351+
? WidthA.getKnownMinValue() + OffsetA - MinOffset
1352+
: MemoryLocation::UnknownSize;
1353+
int64_t OverlapB = KnownWidthB
1354+
? WidthB.getKnownMinValue() + OffsetB - MinOffset
1355+
: MemoryLocation::UnknownSize;
1356+
1357+
LocationSize LocA = WidthA.isScalable() && KnownWidthA
1358+
? LocationSize::precise(WidthA)
1359+
: LocationSize(OverlapA);
1360+
LocationSize LocB = WidthB.isScalable() && KnownWidthB
1361+
? LocationSize::precise(WidthB)
1362+
: LocationSize(OverlapB);
13451363

13461364
return !AA->isNoAlias(
1347-
MemoryLocation(ValA, OverlapA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
1348-
MemoryLocation(ValB, OverlapB,
1349-
UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
1365+
MemoryLocation(ValA, LocA, UseTBAA ? MMOa->getAAInfo() : AAMDNodes()),
1366+
MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes()));
13501367
}
13511368

13521369
bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other,

‎llvm/lib/CodeGen/MachineOperand.cpp

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1098,14 +1098,29 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
10981098
s == ~UINT64_C(0) ? LLT() : LLT::scalar(8 * s), a,
10991099
AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
11001100

1101+
MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
1102+
TypeSize ts, Align a,
1103+
const AAMDNodes &AAInfo,
1104+
const MDNode *Ranges, SyncScope::ID SSID,
1105+
AtomicOrdering Ordering,
1106+
AtomicOrdering FailureOrdering)
1107+
: MachineMemOperand(
1108+
ptrinfo, f,
1109+
ts.getKnownMinValue() == ~UINT64_C(0)
1110+
? LLT()
1111+
: ts.isScalable()
1112+
? LLT::scalable_vector(1, 8 * ts.getKnownMinValue())
1113+
: LLT::scalar(8 * ts.getKnownMinValue()),
1114+
a, AAInfo, Ranges, SSID, Ordering, FailureOrdering) {}
1115+
11011116
void MachineMemOperand::refineAlignment(const MachineMemOperand *MMO) {
11021117
// The Value and Offset may differ due to CSE. But the flags and size
11031118
// should be the same.
11041119
assert(MMO->getFlags() == getFlags() && "Flags mismatch!");
1105-
assert((MMO->getSize() == ~UINT64_C(0) || getSize() == ~UINT64_C(0) ||
1120+
assert((MMO->getSize().getKnownMinValue() == ~UINT64_C(0) ||
1121+
getSize().getKnownMinValue() == ~UINT64_C(0) ||
11061122
MMO->getSize() == getSize()) &&
11071123
"Size mismatch!");
1108-
11091124
if (MMO->getBaseAlign() >= getBaseAlign()) {
11101125
// Update the alignment value.
11111126
BaseAlign = MMO->getBaseAlign();
@@ -1227,7 +1242,10 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
12271242
<< "unknown-address";
12281243
}
12291244
MachineOperand::printOperandOffset(OS, getOffset());
1230-
if (getSize() > 0 && getAlign() != getSize())
1245+
if (getSize().isScalable())
1246+
OS << ", vscale ";
1247+
if (getSize().getKnownMinValue() > 0 &&
1248+
getAlign() != getSize().getKnownMinValue())
12311249
OS << ", align " << getAlign().value();
12321250
if (getAlign() != getBaseAlign())
12331251
OS << ", basealign " << getBaseAlign().value();

0 commit comments

Comments
 (0)