-
Notifications
You must be signed in to change notification settings - Fork 13.7k
[X86][MC] Compress APX Promoted instrs from evex to legacy encoding to save code size. #77065
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,7 +8,7 @@ tablegen(LLVM X86GenAsmWriter1.inc -gen-asm-writer -asmwriternum=1) | |
tablegen(LLVM X86GenCallingConv.inc -gen-callingconv) | ||
tablegen(LLVM X86GenDAGISel.inc -gen-dag-isel) | ||
tablegen(LLVM X86GenDisassemblerTables.inc -gen-disassembler) | ||
tablegen(LLVM X86GenEVEX2VEXTables.inc -gen-x86-EVEX2VEX-tables) | ||
tablegen(LLVM X86GenEVEX2NonEVEXTables.inc -gen-x86-EVEX2NonEVEX-tables) | ||
tablegen(LLVM X86GenExegesis.inc -gen-exegesis) | ||
tablegen(LLVM X86GenFastISel.inc -gen-fast-isel) | ||
tablegen(LLVM X86GenGlobalISel.inc -gen-global-isel) | ||
|
@@ -61,7 +61,7 @@ set(sources | |
X86InstrFMA3Info.cpp | ||
X86InstrFoldTables.cpp | ||
X86InstrInfo.cpp | ||
X86EvexToVex.cpp | ||
X86EvexToNonEvex.cpp | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. CompressEVEX |
||
X86LoadValueInjectionLoadHardening.cpp | ||
X86LoadValueInjectionRetHardening.cpp | ||
X86MCInstLower.cpp | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,6 @@ | ||
//===- X86EvexToVex.cpp ---------------------------------------------------===// | ||
// Compress EVEX instructions to VEX encoding when possible to reduce code size | ||
//===- X86EvexToNonEvex.cpp -----------------------------------------------===// | ||
// Compress EVEX instructions to Non-EVEX encoding when possible to reduce code | ||
// size. | ||
// | ||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
// See https://llvm.org/LICENSE.txt for license information. | ||
|
@@ -16,7 +17,11 @@ | |
/// accessed by instruction is less than 512 bits and when it does not use | ||
// the xmm or the mask registers or xmm/ymm registers with indexes higher | ||
// than 15. | ||
/// The pass applies code reduction on the generated code for AVX-512 instrs. | ||
// APX promoted instrs use evex encoding which could let them use r16-r31, if | ||
// they don't use egpr, we could compress them back to legacy encoding to save | ||
// code size. | ||
/// The pass applies code reduction on the generated code for AVX-512 instrs and | ||
/// APX promoted instrs. | ||
// | ||
//===----------------------------------------------------------------------===// | ||
|
||
|
@@ -38,34 +43,35 @@ | |
|
||
using namespace llvm; | ||
|
||
// Including the generated EVEX2VEX tables. | ||
struct X86EvexToVexCompressTableEntry { | ||
// Including the generated EVEX2NonEVEX tables. | ||
struct X86EvexToNonEvexCompressTableEntry { | ||
uint16_t EvexOpc; | ||
uint16_t VexOpc; | ||
uint16_t NonEvexOpc; | ||
|
||
bool operator<(const X86EvexToVexCompressTableEntry &RHS) const { | ||
bool operator<(const X86EvexToNonEvexCompressTableEntry &RHS) const { | ||
return EvexOpc < RHS.EvexOpc; | ||
} | ||
|
||
friend bool operator<(const X86EvexToVexCompressTableEntry &TE, | ||
friend bool operator<(const X86EvexToNonEvexCompressTableEntry &TE, | ||
unsigned Opc) { | ||
return TE.EvexOpc < Opc; | ||
} | ||
}; | ||
#include "X86GenEVEX2VEXTables.inc" | ||
#include "X86GenEVEX2NonEVEXTables.inc" | ||
|
||
#define EVEX2VEX_DESC "Compressing EVEX instrs to VEX encoding when possible" | ||
#define EVEX2VEX_NAME "x86-evex-to-vex-compress" | ||
#define EVEX2NONEVEX_DESC \ | ||
"Compressing EVEX instrs to Non-EVEX encoding when possible" | ||
#define EVEX2NONEVEX_NAME "x86-evex-to-non-evex-compress" | ||
|
||
#define DEBUG_TYPE EVEX2VEX_NAME | ||
#define DEBUG_TYPE EVEX2NONEVEX_NAME | ||
|
||
namespace { | ||
|
||
class EvexToVexInstPass : public MachineFunctionPass { | ||
class EvexToNonEvexInstPass : public MachineFunctionPass { | ||
public: | ||
static char ID; | ||
EvexToVexInstPass() : MachineFunctionPass(ID) {} | ||
StringRef getPassName() const override { return EVEX2VEX_DESC; } | ||
EvexToNonEvexInstPass() : MachineFunctionPass(ID) {} | ||
StringRef getPassName() const override { return EVEX2NONEVEX_DESC; } | ||
|
||
/// Loop over all of the basic blocks, replacing EVEX instructions | ||
/// by equivalent VEX instructions when possible for reducing code size. | ||
|
@@ -80,7 +86,7 @@ class EvexToVexInstPass : public MachineFunctionPass { | |
|
||
} // end anonymous namespace | ||
|
||
char EvexToVexInstPass::ID = 0; | ||
char EvexToNonEvexInstPass::ID = 0; | ||
|
||
static bool usesExtendedRegister(const MachineInstr &MI) { | ||
auto isHiRegIdx = [](unsigned Reg) { | ||
|
@@ -200,7 +206,7 @@ static bool performCustomAdjustments(MachineInstr &MI, unsigned VexOpc) { | |
case X86::VRNDSCALESDZm_Int: | ||
case X86::VRNDSCALESSZr_Int: | ||
case X86::VRNDSCALESSZm_Int: | ||
const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1); | ||
const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1); | ||
int64_t ImmVal = Imm.getImm(); | ||
// Ensure that only bits 3:0 of the immediate are used. | ||
if ((ImmVal & 0xf) != ImmVal) | ||
|
@@ -214,6 +220,8 @@ static bool performCustomAdjustments(MachineInstr &MI, unsigned VexOpc) { | |
// For EVEX instructions that can be encoded using VEX encoding | ||
// replace them by the VEX encoding in order to reduce size. | ||
static bool CompressEvexToVexImpl(MachineInstr &MI, const X86Subtarget &ST) { | ||
if (!ST.hasAVX512()) | ||
return false; | ||
// VEX format. | ||
// # of bytes: 0,2,3 1 1 0,1 0,1,2,4 0,1 | ||
// [Prefixes] [VEX] OPCODE ModR/M [SIB] [DISP] [IMM] | ||
|
@@ -239,7 +247,7 @@ static bool CompressEvexToVexImpl(MachineInstr &MI, const X86Subtarget &ST) { | |
return false; | ||
|
||
// Use the VEX.L bit to select the 128 or 256-bit table. | ||
ArrayRef<X86EvexToVexCompressTableEntry> Table = | ||
ArrayRef<X86EvexToNonEvexCompressTableEntry> Table = | ||
(Desc.TSFlags & X86II::VEX_L) ? ArrayRef(X86EvexToVex256CompressTable) | ||
: ArrayRef(X86EvexToVex128CompressTable); | ||
|
||
|
@@ -252,15 +260,37 @@ static bool CompressEvexToVexImpl(MachineInstr &MI, const X86Subtarget &ST) { | |
return false; | ||
if (!checkVEXInstPredicate(EvexOpc, ST)) | ||
return false; | ||
if (!performCustomAdjustments(MI, I->VexOpc)) | ||
if (!performCustomAdjustments(MI, I->NonEvexOpc)) | ||
return false; | ||
|
||
MI.setDesc(ST.getInstrInfo()->get(I->VexOpc)); | ||
MI.setDesc(ST.getInstrInfo()->get(I->NonEvexOpc)); | ||
MI.setAsmPrinterFlag(X86::AC_EVEX_2_VEX); | ||
return true; | ||
} | ||
|
||
bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) { | ||
// For apx promoted instructions, if they don't use egpr, we could try to use | ||
// legacy encoding to save code size. | ||
static bool CompressEVEX2LegacyImpl(MachineInstr &MI, const X86Subtarget &ST) { | ||
if (!ST.hasEGPR()) | ||
return false; | ||
ArrayRef<X86EvexToNonEvexCompressTableEntry> Table = | ||
X86EvexToLegacyCompressTable; | ||
unsigned EvexOpc = MI.getOpcode(); | ||
const auto *I = llvm::lower_bound(Table, EvexOpc); | ||
if (I == Table.end() || I->EvexOpc != EvexOpc) | ||
return false; | ||
unsigned NewOpc = I->NonEvexOpc; | ||
for (unsigned Index = 0, Size = MI.getNumOperands(); Index < Size; Index++) { | ||
const MachineOperand &Op = MI.getOperand(Index); | ||
if (Op.isReg() && X86II::isApxExtendedReg(Op.getReg())) | ||
return false; | ||
} | ||
MI.setDesc(ST.getInstrInfo()->get(NewOpc)); | ||
MI.setAsmPrinterFlag(X86::AC_EVEX_2_LEGACY); | ||
return true; | ||
} | ||
|
||
bool EvexToNonEvexInstPass::runOnMachineFunction(MachineFunction &MF) { | ||
#ifndef NDEBUG | ||
// Make sure the tables are sorted. | ||
static std::atomic<bool> TableChecked(false); | ||
|
@@ -269,28 +299,33 @@ bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) { | |
"X86EvexToVex128CompressTable is not sorted!"); | ||
assert(llvm::is_sorted(X86EvexToVex256CompressTable) && | ||
"X86EvexToVex256CompressTable is not sorted!"); | ||
assert(llvm::is_sorted(X86EvexToLegacyCompressTable) && | ||
"X86EvexToLegacyCompressTable is not sorted!"); | ||
TableChecked.store(true, std::memory_order_relaxed); | ||
} | ||
#endif | ||
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>(); | ||
if (!ST.hasAVX512()) | ||
if (!ST.hasAVX512() && !ST.hasEGPR()) | ||
return false; | ||
|
||
bool Changed = false; | ||
|
||
/// Go over all basic blocks in function and replace | ||
/// EVEX encoded instrs by VEX encoding when possible. | ||
/// EVEX encoded instrs by VEX/Legacy encoding when possible. | ||
for (MachineBasicBlock &MBB : MF) { | ||
// Traverse the basic block. | ||
for (MachineInstr &MI : MBB) | ||
for (MachineInstr &MI : MBB) { | ||
Changed |= CompressEvexToVexImpl(MI, ST); | ||
Changed |= CompressEVEX2LegacyImpl(MI, ST); | ||
Comment on lines
318
to
+319
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This can simplify to
Because if one instruction can be compressed to VEX, it cannot be compressed to legacy anymore. |
||
} | ||
} | ||
|
||
return Changed; | ||
} | ||
|
||
INITIALIZE_PASS(EvexToVexInstPass, EVEX2VEX_NAME, EVEX2VEX_DESC, false, false) | ||
INITIALIZE_PASS(EvexToNonEvexInstPass, EVEX2NONEVEX_NAME, EVEX2NONEVEX_DESC, | ||
false, false) | ||
|
||
FunctionPass *llvm::createX86EvexToVexInsts() { | ||
return new EvexToVexInstPass(); | ||
FunctionPass *llvm::createX86EvexToNonEvexInsts() { | ||
return new EvexToNonEvexInstPass(); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,7 +30,9 @@ namespace X86 { | |
|
||
enum AsmComments { | ||
// For instr that was compressed from EVEX to VEX. | ||
AC_EVEX_2_VEX = MachineInstr::TAsmComments | ||
AC_EVEX_2_VEX = MachineInstr::TAsmComments, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No need new comment, rename it to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Never mind this suggestion |
||
// For instrs that was compressed from EVEX to Legacy. | ||
AC_EVEX_2_LEGACY = AC_EVEX_2_VEX << 1 | ||
}; | ||
|
||
/// Return a pair of condition code for the given predicate and whether | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
EVEX2NonEVEX -> CompressEVEX