Skip to content

[X86][GlobalISel] Support addr matching in SDAG patterns #130445

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Apr 19, 2025
Merged

[X86][GlobalISel] Support addr matching in SDAG patterns #130445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
766c7cd
[X86][GlobalISel] Support addr matching in SDAG pattern
e-kud Feb 6, 2025
74a7f95
Merge remote-tracking branch 'origin/main' into global-load-2
e-kud Mar 18, 2025
71586ad
Merge remote-tracking branch 'origin/main' into global-load-2
e-kud Mar 23, 2025
933ccb6
Minor fixes
e-kud Mar 23, 2025
4843059
Addressing comments: assert message and style
e-kud Mar 25, 2025
00e6f86
Merge remote-tracking branch 'origin/main' into global-load-2
e-kud Apr 1, 2025
0e33f15
Update sqrt.mir
e-kud Apr 1, 2025
8ffbda9
Format
e-kud Apr 1, 2025
f12a77f
Merge branch 'main' into global-load-2
e-kud Apr 2, 2025
b6ea8a9
Merge branch 'main' into global-load-2
e-kud Apr 8, 2025
5ca0f05
Refactor getPtrLoadStoreOp
e-kud Apr 8, 2025
970fcb0
Merge branch 'main' into global-load-2
e-kud Apr 15, 2025
3ed2ae0
Merge branch 'main' into global-load-2
e-kud Apr 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ class GMemOperation : public GenericMachineInstr {
/// memory operations can't be reordered.
bool isUnordered() const { return getMMO().isUnordered(); }

/// Return the minimum known alignment in bytes of the actual memory
/// reference.
Align getAlign() const { return getMMO().getAlign(); }
/// Returns the size in bytes of the memory access.
LocationSize getMemSize() const { return getMMO().getSize(); }
/// Returns the size in bits of the memory access.
Expand Down
185 changes: 127 additions & 58 deletions llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ class X86InstructionSelector : public InstructionSelector {
// TODO: remove after supported by Tablegen-erated instruction selection.
unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
Align Alignment) const;
// TODO: remove once p0<->i32/i64 matching is available
unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
unsigned Opc) const;

bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
MachineFunction &MF) const;
Expand Down Expand Up @@ -119,6 +122,8 @@ class X86InstructionSelector : public InstructionSelector {
bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
MachineFunction &MF) const;

ComplexRendererFns selectAddr(MachineOperand &Root) const;

// emit insert subreg instruction and insert it before MachineInstr &I
bool emitInsertSubreg(Register DstReg, Register SrcReg, MachineInstr &I,
MachineRegisterInfo &MRI, MachineFunction &MF) const;
Expand Down Expand Up @@ -445,6 +450,25 @@ bool X86InstructionSelector::select(MachineInstr &I) {
return false;
}

unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
const RegisterBank &RB,
unsigned Opc) const {
assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
"Only G_STORE and G_LOAD are expected for selection");
if (Ty.isPointer() && X86::GPRRegBankID == RB.getID()) {
bool IsLoad = (Opc == TargetOpcode::G_LOAD);
switch (Ty.getSizeInBits()) {
default:
break;
case 32:
return IsLoad ? X86::MOV32rm : X86::MOV32mr;
case 64:
return IsLoad ? X86::MOV64rm : X86::MOV64mr;
}
}
return Opc;
}

unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
const RegisterBank &RB,
unsigned Opc,
Expand All @@ -460,7 +484,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
} else if (Ty == LLT::scalar(16)) {
if (X86::GPRRegBankID == RB.getID())
return Isload ? X86::MOV16rm : X86::MOV16mr;
} else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
} else if (Ty == LLT::scalar(32)) {
if (X86::GPRRegBankID == RB.getID())
return Isload ? X86::MOV32rm : X86::MOV32mr;
if (X86::VECRRegBankID == RB.getID())
Expand All @@ -472,7 +496,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
X86::MOVSSmr);
if (X86::PSRRegBankID == RB.getID())
return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
} else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
} else if (Ty == LLT::scalar(64)) {
if (X86::GPRRegBankID == RB.getID())
return Isload ? X86::MOV64rm : X86::MOV64mr;
if (X86::VECRRegBankID == RB.getID())
Expand Down Expand Up @@ -530,30 +554,76 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
}

// Fill in an address from the given instruction.
static void X86SelectAddress(const MachineInstr &I,
static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
const MachineRegisterInfo &MRI,
X86AddressMode &AM) {
assert(I.getOperand(0).isReg() && "unsupported opperand.");
const X86Subtarget &STI, X86AddressMode &AM) {
assert(I.getOperand(0).isReg() && "unsupported operand.");
assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
"unsupported type.");

if (I.getOpcode() == TargetOpcode::G_PTR_ADD) {
switch (I.getOpcode()) {
default:
break;
case TargetOpcode::G_FRAME_INDEX:
AM.Base.FrameIndex = I.getOperand(1).getIndex();
AM.BaseType = X86AddressMode::FrameIndexBase;
return true;
case TargetOpcode::G_PTR_ADD: {
if (auto COff = getIConstantVRegSExtVal(I.getOperand(2).getReg(), MRI)) {
int64_t Imm = *COff;
if (isInt<32>(Imm)) { // Check for displacement overflow.
AM.Disp = static_cast<int32_t>(Imm);
AM.Base.Reg = I.getOperand(1).getReg();
return;
return true;
}
}
} else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
AM.Base.FrameIndex = I.getOperand(1).getIndex();
AM.BaseType = X86AddressMode::FrameIndexBase;
return;
break;
}
case TargetOpcode::G_GLOBAL_VALUE: {
auto GV = I.getOperand(1).getGlobal();
if (GV->isThreadLocal()) {
return false; // TODO: we don't support TLS yet.
}
// Can't handle alternate code models yet.
if (TM.getCodeModel() != CodeModel::Small)
return false;
AM.GV = GV;
AM.GVOpFlags = STI.classifyGlobalReference(GV);

// TODO: The ABI requires an extra load. not supported yet.
if (isGlobalStubReference(AM.GVOpFlags))
return false;

// TODO: This reference is relative to the pic base. not supported yet.
if (isGlobalRelativeToPICBase(AM.GVOpFlags))
return false;

if (STI.isPICStyleRIPRel()) {
// Use rip-relative addressing.
assert(AM.Base.Reg == 0 && AM.IndexReg == 0 &&
"RIP-relative addresses can't have additional register operands");
AM.Base.Reg = X86::RIP;
}
return true;
}
case TargetOpcode::G_CONSTANT_POOL: {
// TODO: Need a separate move for Large model
if (TM.getCodeModel() == CodeModel::Large)
return false;

AM.GVOpFlags = STI.classifyLocalReference(nullptr);
if (AM.GVOpFlags == X86II::MO_GOTOFF)
AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(I.getMF());
else if (STI.is64Bit())
AM.Base.Reg = X86::RIP;
AM.CP = true;
AM.Disp = I.getOperand(1).getIndex();
return true;
}
}
// Default behavior.
AM.Base.Reg = I.getOperand(0).getReg();
return true;
}

bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
Expand Down Expand Up @@ -586,36 +656,18 @@ bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
}
}

unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
if (NewOpc == Opc)
return false;

I.setDesc(TII.get(NewOpc));
MachineInstrBuilder MIB(MF, I);
const MachineInstr *Ptr = MRI.getVRegDef(I.getOperand(1).getReg());

if (Ptr->getOpcode() == TargetOpcode::G_CONSTANT_POOL) {
assert(Opc == TargetOpcode::G_LOAD &&
"Only G_LOAD from constant pool is expected");
// TODO: Need a separate move for Large model
if (TM.getCodeModel() == CodeModel::Large)
return false;

unsigned char OpFlag = STI.classifyLocalReference(nullptr);
Register PICBase;
if (OpFlag == X86II::MO_GOTOFF)
PICBase = TII.getGlobalBaseReg(&MF);
else if (STI.is64Bit())
PICBase = X86::RIP;

I.removeOperand(1);
addConstantPoolReference(MIB, Ptr->getOperand(1).getIndex(), PICBase,
OpFlag);
return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
}
MachineInstr *Ptr = MRI.getVRegDef(I.getOperand(1).getReg());

X86AddressMode AM;
X86SelectAddress(*Ptr, MRI, AM);
if (!X86SelectAddress(*Ptr, TM, MRI, STI, AM))
return false;

if (Opc == TargetOpcode::G_LOAD) {
I.removeOperand(1);
addFullAddress(MIB, AM);
Expand Down Expand Up @@ -673,33 +725,10 @@ bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
"unexpected instruction");

auto GV = I.getOperand(1).getGlobal();
if (GV->isThreadLocal()) {
return false; // TODO: we don't support TLS yet.
}

// Can't handle alternate code models yet.
if (TM.getCodeModel() != CodeModel::Small)
return false;

X86AddressMode AM;
AM.GV = GV;
AM.GVOpFlags = STI.classifyGlobalReference(GV);

// TODO: The ABI requires an extra load. not supported yet.
if (isGlobalStubReference(AM.GVOpFlags))
if (!X86SelectAddress(I, TM, MRI, STI, AM))
return false;

// TODO: This reference is relative to the pic base. not supported yet.
if (isGlobalRelativeToPICBase(AM.GVOpFlags))
return false;

if (STI.isPICStyleRIPRel()) {
// Use rip-relative addressing.
assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
AM.Base.Reg = X86::RIP;
}

const Register DefReg = I.getOperand(0).getReg();
LLT Ty = MRI.getType(DefReg);
unsigned NewOpc = getLeaOP(Ty, STI);
Expand Down Expand Up @@ -1880,6 +1909,46 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I,
return true;
}

InstructionSelector::ComplexRendererFns
X86InstructionSelector::selectAddr(MachineOperand &Root) const {
MachineInstr *MI = Root.getParent();
MachineIRBuilder MIRBuilder(*MI);

MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
MachineInstr *Ptr = MRI.getVRegDef(Root.getReg());
X86AddressMode AM;
X86SelectAddress(*Ptr, TM, MRI, STI, AM);

if (AM.IndexReg)
return std::nullopt;

return {// Base
{[=](MachineInstrBuilder &MIB) {
if (AM.BaseType == X86AddressMode::RegBase)
MIB.addUse(AM.Base.Reg);
else {
assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
"Unknown type of address base");
MIB.addFrameIndex(AM.Base.FrameIndex);
}
},
// Scale
[=](MachineInstrBuilder &MIB) { MIB.addImm(AM.Scale); },
// Index
[=](MachineInstrBuilder &MIB) { MIB.addUse(0); },
// Disp
[=](MachineInstrBuilder &MIB) {
if (AM.GV)
MIB.addGlobalAddress(AM.GV, AM.Disp, AM.GVOpFlags);
else if (AM.CP)
MIB.addConstantPoolIndex(AM.Disp, 0, AM.GVOpFlags);
else
MIB.addImm(AM.Disp);
},
// Segment
[=](MachineInstrBuilder &MIB) { MIB.addUse(0); }}};
}

InstructionSelector *
llvm::createX86InstructionSelector(const X86TargetMachine &TM,
const X86Subtarget &Subtarget,
Expand Down
47 changes: 37 additions & 10 deletions llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,22 +373,16 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
// load/store: add more corner cases
for (unsigned Op : {G_LOAD, G_STORE}) {
auto &Action = getActionDefinitionsBuilder(Op);
Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},
{s8, p0, s8, 1},
{s16, p0, s8, 1},
Action.legalForTypesWithMemDesc({{s8, p0, s8, 1},
{s16, p0, s16, 1},
{s32, p0, s8, 1},
{s32, p0, s16, 1},
{s32, p0, s32, 1},
{s80, p0, s80, 1},
{p0, p0, p0, 1},
{v4s8, p0, v4s8, 1}});
if (Is64Bit)
Action.legalForTypesWithMemDesc({{s64, p0, s8, 1},
{s64, p0, s16, 1},
{s64, p0, s32, 1},
{s64, p0, s64, 1},
{v2s32, p0, v2s32, 1}});
Action.legalForTypesWithMemDesc(
{{s64, p0, s64, 1}, {v2s32, p0, v2s32, 1}});

if (HasSSE1)
Action.legalForTypesWithMemDesc({{v4s32, p0, v4s32, 1}});
if (HasSSE2)
Expand All @@ -407,6 +401,21 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
{v32s16, p0, v32s16, 1},
{v16s32, p0, v16s32, 1},
{v8s64, p0, v8s64, 1}});

// X86 supports extending loads but not stores for GPRs
if (Op == G_LOAD) {
Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},
{s16, p0, s8, 1},
{s32, p0, s8, 1},
{s32, p0, s16, 1}});
if (Is64Bit)
Action.legalForTypesWithMemDesc(
{{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});
} else {
Action.customIf([=](const LegalityQuery &Query) {
return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
});
}
Action.widenScalarToNextPow2(0, /*Min=*/8)
.clampScalar(0, s8, sMaxScalar)
.scalarize(0);
Expand Down Expand Up @@ -660,6 +669,8 @@ bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
return legalizeFPTOUI(MI, MRI, Helper);
case TargetOpcode::G_UITOFP:
return legalizeUITOFP(MI, MRI, Helper);
case TargetOpcode::G_STORE:
return legalizeNarrowingStore(MI, MRI, Helper);
}
llvm_unreachable("expected switch to return");
}
Expand Down Expand Up @@ -754,6 +765,22 @@ bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,
return false;
}

bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,
MachineRegisterInfo &MRI,
LegalizerHelper &Helper) const {
auto &Store = cast<GStore>(MI);
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
MachineMemOperand &MMO = **Store.memoperands_begin();
MachineFunction &MF = MIRBuilder.getMF();
LLT ValTy = MRI.getType(Store.getValueReg());
auto *NewMMO = MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), ValTy);

Helper.Observer.changingInstr(Store);
Store.setMemRefs(MF, {NewMMO});
Helper.Observer.changedInstr(Store);
return true;
}

bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MachineInstr &MI) const {
return true;
Expand Down
3 changes: 3 additions & 0 deletions llvm/lib/Target/X86/GISel/X86LegalizerInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ class X86LegalizerInfo : public LegalizerInfo {

bool legalizeUITOFP(MachineInstr &MI, MachineRegisterInfo &MRI,
LegalizerHelper &Helper) const;

bool legalizeNarrowingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
LegalizerHelper &Helper) const;
};
} // namespace llvm
#endif
1 change: 1 addition & 0 deletions llvm/lib/Target/X86/X86InstrBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ struct X86AddressMode {
int Disp = 0;
const GlobalValue *GV = nullptr;
unsigned GVOpFlags = 0;
bool CP = false;

void getFullAddress(SmallVectorImpl<MachineOperand> &MO) {
assert(Scale == 1 || Scale == 2 || Scale == 4 || Scale == 8);
Expand Down
Loading
Loading