Skip to content

[RISCV] Use vsetvli instead of vlenb in Prologue/Epilogue #113756

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 21, 2025
Merged

[RISCV] Use vsetvli instead of vlenb in Prologue/Epilogue #113756

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7149518
[RISCV] Use vsetvli instead of vlenb in Prologue/Epilogue
kito-cheng Oct 26, 2024
1573c2b
Apply Craig's suggestion
kito-cheng Mar 14, 2025
0b1e453
Address Luke's comment
kito-cheng Mar 14, 2025
bbd5b17
Apply Craig's comment
kito-cheng Mar 17, 2025
091fe30
fixup Rename PseudoReadMulVLENB to PseudoReadVLENBViaVSETVLIX0
kito-cheng Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ class RISCVExpandPseudo : public MachineFunctionPass {
MachineBasicBlock::iterator MBBI);
bool expandRV32ZdinxLoad(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
bool expandPseudoReadMulVLENB(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI);
#ifndef NDEBUG
unsigned getInstSizeInBytes(const MachineFunction &MF) const {
unsigned Size = 0;
Expand Down Expand Up @@ -164,6 +166,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
case RISCV::PseudoVMSET_M_B64:
// vmset.m vd => vmxnor.mm vd, vd, vd
return expandVMSET_VMCLR(MBB, MBBI, RISCV::VMXNOR_MM);
case RISCV::PseudoReadMulVLENB:
return expandPseudoReadMulVLENB(MBB, MBBI);
}

return false;
Expand Down Expand Up @@ -415,6 +419,39 @@ bool RISCVExpandPseudo::expandRV32ZdinxLoad(MachineBasicBlock &MBB,
return true;
}

bool RISCVExpandPseudo::expandPseudoReadMulVLENB(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
DebugLoc DL = MBBI->getDebugLoc();
Register Dst = MBBI->getOperand(0).getReg();
unsigned Mul = MBBI->getOperand(1).getImm();
RISCVVType::VLMUL VLMUL = RISCVVType::VLMUL::LMUL_1;
switch (Mul) {
case 1:
VLMUL = RISCVVType::VLMUL::LMUL_1;
break;
case 2:
VLMUL = RISCVVType::VLMUL::LMUL_2;
break;
case 4:
VLMUL = RISCVVType::VLMUL::LMUL_4;
break;
case 8:
VLMUL = RISCVVType::VLMUL::LMUL_8;
break;
default:
llvm_unreachable("Unexpected VLENB value");
}
unsigned VTypeImm = RISCVVType::encodeVTYPE(
VLMUL, /*SEW*/ 8, /*TailAgnostic*/ true, /*MaskAgnostic*/ true);

BuildMI(MBB, MBBI, DL, TII->get(RISCV::VSETVLI), Dst)
.addReg(RISCV::X0)
.addImm(VTypeImm);

MBBI->eraseFromParent();
return true;
}

class RISCVPreRAExpandPseudo : public MachineFunctionPass {
public:
const RISCVSubtarget *STI;
Expand Down
6 changes: 6 additions & 0 deletions llvm/lib/Target/RISCV/RISCVFeatures.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1534,6 +1534,12 @@ def TuneVentanaVeyron : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "V
def TuneVXRMPipelineFlush : SubtargetFeature<"vxrm-pipeline-flush", "HasVXRMPipelineFlush",
"true", "VXRM writes causes pipeline flush">;

def TunePreferVsetvliOverReadVLENB
: SubtargetFeature<"prefer-vsetvli-over-read-vlenb",
"PreferVsetvliOverReadVLENB",
"true",
"Prefer vsetvli over read vlenb CSR when calculate VLEN">;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when calculate -> to calculate


// Assume that lock-free native-width atomics are available, even if the target
// and operating system combination would not usually provide them. The user
// is responsible for providing any necessary __sync implementations. Code
Expand Down
43 changes: 31 additions & 12 deletions llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,7 +669,8 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
// Simply allocate the stack if it's not big enough to require a probe.
if (!NeedProbe || Offset <= ProbeSize) {
RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-Offset),
MachineInstr::FrameSetup, getStackAlign());
MachineInstr::FrameSetup, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);

if (EmitCFI) {
// Emit ".cfi_def_cfa_offset RealStackSize"
Expand Down Expand Up @@ -698,7 +699,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
while (CurrentOffset + ProbeSize <= Offset) {
RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
StackOffset::getFixed(-ProbeSize), MachineInstr::FrameSetup,
getStackAlign());
getStackAlign(), /*IsPrologueOrEpilogue*/ true);
// s[d|w] zero, 0(sp)
BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
.addReg(RISCV::X0)
Expand All @@ -721,7 +722,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
if (Residual) {
RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
StackOffset::getFixed(-Residual), MachineInstr::FrameSetup,
getStackAlign());
getStackAlign(), /*IsPrologueOrEpilogue*/ true);
if (EmitCFI) {
// Emit ".cfi_def_cfa_offset Offset"
unsigned CFIIndex =
Expand Down Expand Up @@ -752,7 +753,7 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,
// SUB TargetReg, SP, RoundedSize
RI->adjustReg(MBB, MBBI, DL, TargetReg, SPReg,
StackOffset::getFixed(-RoundedSize), MachineInstr::FrameSetup,
getStackAlign());
getStackAlign(), /*IsPrologueOrEpilogue*/ true);

if (EmitCFI) {
// Set the CFA register to TargetReg.
Expand Down Expand Up @@ -781,7 +782,8 @@ void RISCVFrameLowering::allocateStack(MachineBasicBlock &MBB,

if (Residual) {
RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-Residual),
MachineInstr::FrameSetup, getStackAlign());
MachineInstr::FrameSetup, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);
if (DynAllocation) {
// s[d|w] zero, 0(sp)
BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
Expand Down Expand Up @@ -1014,7 +1016,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
RI->adjustReg(
MBB, MBBI, DL, FPReg, SPReg,
StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
MachineInstr::FrameSetup, getStackAlign());
MachineInstr::FrameSetup, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);
}

// Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
Expand Down Expand Up @@ -1047,7 +1050,8 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
// updates.
RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
StackOffset::getScalable(-RVVStackSize),
MachineInstr::FrameSetup, getStackAlign());
MachineInstr::FrameSetup, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);
}

if (!hasFP(MF)) {
Expand Down Expand Up @@ -1125,7 +1129,8 @@ void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
const RISCVInstrInfo *TII = STI.getInstrInfo();

RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
MachineInstr::FrameDestroy, getStackAlign());
MachineInstr::FrameDestroy, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);
StackSize = 0;

unsigned CFIIndex =
Expand Down Expand Up @@ -1189,7 +1194,8 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
if (!RestoreSPFromFP)
RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
StackOffset::getScalable(RVVStackSize),
MachineInstr::FrameDestroy, getStackAlign());
MachineInstr::FrameDestroy, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);

if (!hasFP(MF)) {
unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
Expand All @@ -1214,7 +1220,8 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
if (!RestoreSPFromFP)
RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
StackOffset::getFixed(SecondSPAdjustAmount),
MachineInstr::FrameDestroy, getStackAlign());
MachineInstr::FrameDestroy, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);

if (!hasFP(MF)) {
unsigned CFIIndex = MF.addFrameInst(
Expand All @@ -1240,7 +1247,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
assert(hasFP(MF) && "frame pointer should not have been eliminated");
RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, FPReg,
StackOffset::getFixed(-FPOffset), MachineInstr::FrameDestroy,
getStackAlign());
getStackAlign(), /*IsPrologueOrEpilogue*/ true);
}

if (hasFP(MF)) {
Expand Down Expand Up @@ -1771,7 +1778,8 @@ MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr(

const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
RI.adjustReg(MBB, MI, DL, SPReg, SPReg, StackOffset::getFixed(Amount),
MachineInstr::NoFlags, getStackAlign());
MachineInstr::NoFlags, getStackAlign(),
/*IsPrologueOrEpilogue*/ true);
}
}

Expand Down Expand Up @@ -2195,6 +2203,17 @@ bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const {
const MachineFunction *MF = MBB.getParent();
const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();

// Make sure VTYPE and VL are not live-in since we will use vsetvli in the
// prologue to get the VLEN, and that will clobber these registers.
//
// We may do also check the stack has contain for the object with the
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"has contain for the object with the " -> "contains objects with"

// scalable vector type, but this will require iterating over all the stack
// objects, but this may not worth since the situation is rare, we could do
// further check in future if we find it is necessary.
if (STI.preferVsetvliOverReadVLENB() &&
(MBB.isLiveIn(RISCV::VTYPE) || MBB.isLiveIn(RISCV::VL)))
return false;

if (!RVFI->useSaveRestoreLibCalls(*MF))
return true;

Expand Down
5 changes: 5 additions & 0 deletions llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6049,6 +6049,11 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1 in {
[(set GPR:$rd, (riscv_read_vlenb))]>,
PseudoInstExpansion<(CSRRS GPR:$rd, SysRegVLENB.Encoding, X0)>,
Sched<[WriteRdVLENB]>;
let Defs = [VL, VTYPE] in {
def PseudoReadMulVLENB : Pseudo<(outs GPR:$rd), (ins uimm5:$shamt),
[]>,
Sched<[WriteVSETVLI, ReadVSETVLI]>;
}
}

let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1,
Expand Down
53 changes: 39 additions & 14 deletions llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ void RISCVRegisterInfo::adjustReg(MachineBasicBlock &MBB,
const DebugLoc &DL, Register DestReg,
Register SrcReg, StackOffset Offset,
MachineInstr::MIFlag Flag,
MaybeAlign RequiredAlign) const {
MaybeAlign RequiredAlign,
bool IsPrologueOrEpilogue) const {

if (DestReg == SrcReg && !Offset.getFixed() && !Offset.getScalable())
return;
Expand Down Expand Up @@ -226,21 +227,44 @@ void RISCVRegisterInfo::adjustReg(MachineBasicBlock &MBB,
assert(isInt<32>(ScalableValue / (RISCV::RVVBitsPerBlock / 8)) &&
"Expect the number of vector registers within 32-bits.");
uint32_t NumOfVReg = ScalableValue / (RISCV::RVVBitsPerBlock / 8);
BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), ScratchReg)
.setMIFlag(Flag);

if (ScalableAdjOpc == RISCV::ADD && ST.hasStdExtZba() &&
(NumOfVReg == 2 || NumOfVReg == 4 || NumOfVReg == 8)) {
unsigned Opc = NumOfVReg == 2 ? RISCV::SH1ADD :
(NumOfVReg == 4 ? RISCV::SH2ADD : RISCV::SH3ADD);
BuildMI(MBB, II, DL, TII->get(Opc), DestReg)
.addReg(ScratchReg, RegState::Kill).addReg(SrcReg)
// Only use vsetvli rather than vlenb if adjusting in the prologue or
// epilogue, otherwise it may distrube the VTYPE and VL status.
bool UseVsetvliRatherThanVlenb =
IsPrologueOrEpilogue && ST.preferVsetvliOverReadVLENB();
if (UseVsetvliRatherThanVlenb && (NumOfVReg == 1 || NumOfVReg == 2 ||
NumOfVReg == 4 || NumOfVReg == 8)) {
BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadMulVLENB), ScratchReg)
.addImm(NumOfVReg)
.setMIFlag(Flag);
} else {
TII->mulImm(MF, MBB, II, DL, ScratchReg, NumOfVReg, Flag);
BuildMI(MBB, II, DL, TII->get(ScalableAdjOpc), DestReg)
.addReg(SrcReg).addReg(ScratchReg, RegState::Kill)
.addReg(SrcReg)
.addReg(ScratchReg, RegState::Kill)
.setMIFlag(Flag);
} else {
if (UseVsetvliRatherThanVlenb)
BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadMulVLENB), ScratchReg)
.addImm(1)
.setMIFlag(Flag);
else
BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), ScratchReg)
.setMIFlag(Flag);

if (ScalableAdjOpc == RISCV::ADD && ST.hasStdExtZba() &&
(NumOfVReg == 2 || NumOfVReg == 4 || NumOfVReg == 8)) {
unsigned Opc = NumOfVReg == 2
? RISCV::SH1ADD
: (NumOfVReg == 4 ? RISCV::SH2ADD : RISCV::SH3ADD);
BuildMI(MBB, II, DL, TII->get(Opc), DestReg)
.addReg(ScratchReg, RegState::Kill)
.addReg(SrcReg)
.setMIFlag(Flag);
} else {
TII->mulImm(MF, MBB, II, DL, ScratchReg, NumOfVReg, Flag);
BuildMI(MBB, II, DL, TII->get(ScalableAdjOpc), DestReg)
.addReg(SrcReg)
.addReg(ScratchReg, RegState::Kill)
.setMIFlag(Flag);
}
}
SrcReg = DestReg;
KillSrcReg = true;
Expand Down Expand Up @@ -533,7 +557,8 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
else
DestReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
adjustReg(*II->getParent(), II, DL, DestReg, FrameReg, Offset,
MachineInstr::NoFlags, std::nullopt);
MachineInstr::NoFlags, std::nullopt,
/*IsPrologueOrEpilogue*/ false);
MI.getOperand(FIOperandNum).ChangeToRegister(DestReg, /*IsDef*/false,
/*IsImp*/false,
/*IsKill*/true);
Expand Down
4 changes: 3 additions & 1 deletion llvm/lib/Target/RISCV/RISCVRegisterInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,12 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo {
// used during frame layout, and we may need to ensure that if we
// split the offset internally that the DestReg is always aligned,
// assuming that source reg was.
// If IsPrologueOrEpilogue is true, the function is called during prologue
// or epilogue generation.
void adjustReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator II,
const DebugLoc &DL, Register DestReg, Register SrcReg,
StackOffset Offset, MachineInstr::MIFlag Flag,
MaybeAlign RequiredAlign) const;
MaybeAlign RequiredAlign, bool IsPrologueOrEpilogue) const;

bool eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj,
unsigned FIOperandNum,
Expand Down
Loading