Skip to content

Commit ffec343

Browse files
dpaoliellokostasalv
dpaoliello
authored and
kostasalv
committed
[x64][win] Add compiler support for x64 import call optimization (equivalent to MSVC /d2guardretpoline) (llvm#126631)
This is the x64 equivalent of llvm#121516 Since import call optimization was originally [added to x64 Windows to implement a more efficient retpoline mitigation](https://techcommunity.microsoft.com/blog/windowsosplatform/mitigating-spectre-variant-2-with-retpoline-on-windows/295618) the section and constant names relating to this all mention "retpoline" and we need to mark indirect calls, control-flow guard calls and jumps for jump tables in the section alongside calls to imported functions. As with the AArch64 feature, this emits a new section into the obj which is used by the MSVC linker to generate the Dynamic Value Relocation Table and the section itself does not appear in the final binary. The Windows Loader requires a specific sequence of instructions be emitted when this feature is enabled: * Indirect calls/jumps must have the function pointer to jump to in `rax`. * Calls to imported functions must use the `rex` prefix and be followed by a 5-byte nop. * Indirect calls must be followed by a 3-byte nop.
1 parent 7655f3e commit ffec343

24 files changed

+588
-31
lines changed

‎llvm/include/llvm/Transforms/CFGuard.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
namespace llvm {
1717

1818
class FunctionPass;
19+
class GlobalValue;
1920

2021
class CFGuardPass : public PassInfoMixin<CFGuardPass> {
2122
public:
@@ -34,6 +35,8 @@ FunctionPass *createCFGuardCheckPass();
3435
/// Insert Control FLow Guard dispatches on indirect function calls.
3536
FunctionPass *createCFGuardDispatchPass();
3637

38+
bool isCFGuardFunction(const GlobalValue *GV);
39+
3740
} // namespace llvm
3841

3942
#endif

‎llvm/lib/MC/MCObjectFileInfo.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,11 @@ void MCObjectFileInfo::initCOFFMCObjectFileInfo(const Triple &T) {
599599
if (T.getArch() == Triple::aarch64) {
600600
ImportCallSection =
601601
Ctx->getCOFFSection(".impcall", COFF::IMAGE_SCN_LNK_INFO);
602+
} else if (T.getArch() == Triple::x86_64) {
603+
// Import Call Optimization on x64 leverages the same metadata as the
604+
// retpoline mitigation, hence the unusual section name.
605+
ImportCallSection =
606+
Ctx->getCOFFSection(".retplne", COFF::IMAGE_SCN_LNK_INFO);
602607
}
603608

604609
// Debug info.

‎llvm/lib/Target/X86/X86AsmPrinter.cpp

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,8 @@ static bool isIndirectBranchOrTailCall(const MachineInstr &MI) {
464464
Opc == X86::TAILJMPr64 || Opc == X86::TAILJMPm64 ||
465465
Opc == X86::TCRETURNri || Opc == X86::TCRETURNmi ||
466466
Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNmi64 ||
467-
Opc == X86::TAILJMPr64_REX || Opc == X86::TAILJMPm64_REX;
467+
Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TAILJMPr64_REX ||
468+
Opc == X86::TAILJMPm64_REX;
468469
}
469470

470471
void X86AsmPrinter::emitBasicBlockEnd(const MachineBasicBlock &MBB) {
@@ -912,6 +913,9 @@ void X86AsmPrinter::emitStartOfAsmFile(Module &M) {
912913
if (TT.isOSBinFormatCOFF()) {
913914
emitCOFFFeatureSymbol(M);
914915
emitCOFFReplaceableFunctionData(M);
916+
917+
if (M.getModuleFlag("import-call-optimization"))
918+
EnableImportCallOptimization = true;
915919
}
916920
OutStreamer->emitSyntaxDirective();
917921

@@ -1016,6 +1020,35 @@ void X86AsmPrinter::emitEndOfAsmFile(Module &M) {
10161020
// safe to set.
10171021
OutStreamer->emitSubsectionsViaSymbols();
10181022
} else if (TT.isOSBinFormatCOFF()) {
1023+
// If import call optimization is enabled, emit the appropriate section.
1024+
// We do this whether or not we recorded any items.
1025+
if (EnableImportCallOptimization) {
1026+
OutStreamer->switchSection(getObjFileLowering().getImportCallSection());
1027+
1028+
// Section always starts with some magic.
1029+
constexpr char ImpCallMagic[12] = "RetpolineV1";
1030+
OutStreamer->emitBytes(StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1031+
1032+
// Layout of this section is:
1033+
// Per section that contains an item to record:
1034+
// uint32_t SectionSize: Size in bytes for information in this section.
1035+
// uint32_t Section Number
1036+
// Per call to imported function in section:
1037+
// uint32_t Kind: the kind of item.
1038+
// uint32_t InstOffset: the offset of the instr in its parent section.
1039+
for (auto &[Section, CallsToImportedFuncs] :
1040+
SectionToImportedFunctionCalls) {
1041+
unsigned SectionSize =
1042+
sizeof(uint32_t) * (2 + 2 * CallsToImportedFuncs.size());
1043+
OutStreamer->emitInt32(SectionSize);
1044+
OutStreamer->emitCOFFSecNumber(Section->getBeginSymbol());
1045+
for (auto &[CallsiteSymbol, Kind] : CallsToImportedFuncs) {
1046+
OutStreamer->emitInt32(Kind);
1047+
OutStreamer->emitCOFFSecOffset(CallsiteSymbol);
1048+
}
1049+
}
1050+
}
1051+
10191052
if (usesMSVCFloatingPoint(TT, M)) {
10201053
// In Windows' libcmt.lib, there is a file which is linked in only if the
10211054
// symbol _fltused is referenced. Linking this in causes some

‎llvm/lib/Target/X86/X86AsmPrinter.h

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,26 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
3535
bool EmitFPOData = false;
3636
bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
3737
bool IndCSPrefix = false;
38+
bool EnableImportCallOptimization = false;
39+
40+
enum ImportCallKind : unsigned {
41+
IMAGE_RETPOLINE_AMD64_IMPORT_BR = 0x02,
42+
IMAGE_RETPOLINE_AMD64_IMPORT_CALL = 0x03,
43+
IMAGE_RETPOLINE_AMD64_INDIR_BR = 0x04,
44+
IMAGE_RETPOLINE_AMD64_INDIR_CALL = 0x05,
45+
IMAGE_RETPOLINE_AMD64_INDIR_BR_REX = 0x06,
46+
IMAGE_RETPOLINE_AMD64_CFG_BR = 0x08,
47+
IMAGE_RETPOLINE_AMD64_CFG_CALL = 0x09,
48+
IMAGE_RETPOLINE_AMD64_CFG_BR_REX = 0x0A,
49+
IMAGE_RETPOLINE_AMD64_SWITCHTABLE_FIRST = 0x010,
50+
IMAGE_RETPOLINE_AMD64_SWITCHTABLE_LAST = 0x01F,
51+
};
52+
struct ImportCallInfo {
53+
MCSymbol *CalleeSymbol;
54+
ImportCallKind Kind;
55+
};
56+
DenseMap<MCSection *, std::vector<ImportCallInfo>>
57+
SectionToImportedFunctionCalls;
3858

3959
// This utility class tracks the length of a stackmap instruction's 'shadow'.
4060
// It is used by the X86AsmPrinter to ensure that the stackmap shadow
@@ -49,7 +69,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
4969
void startFunction(MachineFunction &MF) {
5070
this->MF = &MF;
5171
}
52-
void count(MCInst &Inst, const MCSubtargetInfo &STI,
72+
void count(const MCInst &Inst, const MCSubtargetInfo &STI,
5373
MCCodeEmitter *CodeEmitter);
5474

5575
// Called to signal the start of a shadow of RequiredSize bytes.
@@ -130,6 +150,12 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
130150
void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
131151
MCSymbol *LazyPointer) override;
132152

153+
void emitCallInstruction(const llvm::MCInst &MCI);
154+
155+
// Emits a label to mark the next instruction as being relevant to Import Call
156+
// Optimization.
157+
void emitLabelAndRecordForImportCallOptimization(ImportCallKind Kind);
158+
133159
public:
134160
X86AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer);
135161

‎llvm/lib/Target/X86/X86ExpandPseudo.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
274274
case X86::TCRETURNdi64:
275275
case X86::TCRETURNdi64cc:
276276
case X86::TCRETURNri64:
277+
case X86::TCRETURNri64_ImpCall:
277278
case X86::TCRETURNmi64: {
278279
bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64;
279280
MachineOperand &JumpTarget = MBBI->getOperand(0);
@@ -345,12 +346,14 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
345346
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
346347
for (unsigned i = 0; i != X86::AddrNumOperands; ++i)
347348
MIB.add(MBBI->getOperand(i));
348-
} else if (Opcode == X86::TCRETURNri64) {
349+
} else if ((Opcode == X86::TCRETURNri64) ||
350+
(Opcode == X86::TCRETURNri64_ImpCall)) {
349351
JumpTarget.setIsKill();
350352
BuildMI(MBB, MBBI, DL,
351353
TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
352354
.add(JumpTarget);
353355
} else {
356+
assert(!IsWin64 && "Win64 requires REX for indirect jumps.");
354357
JumpTarget.setIsKill();
355358
BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr))
356359
.add(JumpTarget);
@@ -875,6 +878,9 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
875878
case X86::CALL64m_RVMARKER:
876879
expandCALL_RVMARKER(MBB, MBBI);
877880
return true;
881+
case X86::CALL64r_ImpCall:
882+
MI.setDesc(TII->get(X86::CALL64r));
883+
return true;
878884
case X86::ADD32mi_ND:
879885
case X86::ADD64mi32_ND:
880886
case X86::SUB32mi_ND:

‎llvm/lib/Target/X86/X86FastISel.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "llvm/IR/Instructions.h"
3535
#include "llvm/IR/IntrinsicInst.h"
3636
#include "llvm/IR/IntrinsicsX86.h"
37+
#include "llvm/IR/Module.h"
3738
#include "llvm/IR/Operator.h"
3839
#include "llvm/MC/MCAsmInfo.h"
3940
#include "llvm/MC/MCSymbol.h"
@@ -3316,6 +3317,11 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
33163317
if (Flag.isSwiftError() || Flag.isPreallocated())
33173318
return false;
33183319

3320+
// Can't handle import call optimization.
3321+
if (Is64Bit &&
3322+
MF->getFunction().getParent()->getModuleFlag("import-call-optimization"))
3323+
return false;
3324+
33193325
SmallVector<MVT, 16> OutVTs;
33203326
SmallVector<Register, 16> ArgRegs;
33213327

‎llvm/lib/Target/X86/X86FrameLowering.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2399,7 +2399,8 @@ X86FrameLowering::getWinEHFuncletFrameSize(const MachineFunction &MF) const {
23992399
static bool isTailCallOpcode(unsigned Opc) {
24002400
return Opc == X86::TCRETURNri || Opc == X86::TCRETURNdi ||
24012401
Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
2402-
Opc == X86::TCRETURNdi64 || Opc == X86::TCRETURNmi64;
2402+
Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TCRETURNdi64 ||
2403+
Opc == X86::TCRETURNmi64;
24032404
}
24042405

24052406
void X86FrameLowering::emitEpilogue(MachineFunction &MF,

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19179,7 +19179,7 @@ SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
1917919179

1918019180
SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
1918119181
SelectionDAG &DAG) const {
19182-
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
19182+
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false, nullptr);
1918319183
}
1918419184

1918519185
SDValue
@@ -19207,7 +19207,8 @@ X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
1920719207
/// Creates target global address or external symbol nodes for calls or
1920819208
/// other uses.
1920919209
SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
19210-
bool ForCall) const {
19210+
bool ForCall,
19211+
bool *IsImpCall) const {
1921119212
// Unpack the global address or external symbol.
1921219213
SDLoc dl(Op);
1921319214
const GlobalValue *GV = nullptr;
@@ -19257,6 +19258,16 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1925719258
if (ForCall && !NeedsLoad && !HasPICReg && Offset == 0)
1925819259
return Result;
1925919260

19261+
// If Import Call Optimization is enabled and this is an imported function
19262+
// then make a note of it and return the global address without wrapping.
19263+
if (IsImpCall && (OpFlags == X86II::MO_DLLIMPORT) &&
19264+
Mod.getModuleFlag("import-call-optimization")) {
19265+
assert(ForCall && "Should only enable import call optimization if we are "
19266+
"lowering a call");
19267+
*IsImpCall = true;
19268+
return Result;
19269+
}
19270+
1926019271
Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result);
1926119272

1926219273
// With PIC, the address is actually $g + Offset.
@@ -19282,7 +19293,7 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1928219293

1928319294
SDValue
1928419295
X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
19285-
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
19296+
return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false, nullptr);
1928619297
}
1928719298

1928819299
static SDValue GetTLSADDR(SelectionDAG &DAG, GlobalAddressSDNode *GA,
@@ -34821,6 +34832,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
3482134832
NODE_NAME_CASE(FST)
3482234833
NODE_NAME_CASE(CALL)
3482334834
NODE_NAME_CASE(CALL_RVMARKER)
34835+
NODE_NAME_CASE(IMP_CALL)
3482434836
NODE_NAME_CASE(BT)
3482534837
NODE_NAME_CASE(CMP)
3482634838
NODE_NAME_CASE(FCMP)
@@ -62092,6 +62104,7 @@ X86TargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
6209262104
Register TargetReg;
6209362105
switch (MBBI->getOpcode()) {
6209462106
case X86::CALL64r:
62107+
case X86::CALL64r_ImpCall:
6209562108
case X86::CALL64r_NT:
6209662109
case X86::TAILJMPr64:
6209762110
case X86::TAILJMPr64_REX:

‎llvm/lib/Target/X86/X86ISelLowering.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ namespace llvm {
9090
/// POP_FROM_X87_REG (which may remove a required FPU stack pop).
9191
POP_FROM_X87_REG,
9292

93+
// Pseudo for a call to an imported function to ensure the correct machine
94+
// instruction is emitted for Import Call Optimization.
95+
IMP_CALL,
96+
9397
/// X86 compare and logical compare instructions.
9498
CMP,
9599
FCMP,
@@ -1746,8 +1750,8 @@ namespace llvm {
17461750

17471751
/// Creates target global address or external symbol nodes for calls or
17481752
/// other uses.
1749-
SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1750-
bool ForCall) const;
1753+
SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, bool ForCall,
1754+
bool *IsImpCall) const;
17511755

17521756
SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
17531757
SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;

‎llvm/lib/Target/X86/X86ISelLoweringCall.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2050,6 +2050,12 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
20502050
if (CallConv == CallingConv::X86_INTR)
20512051
report_fatal_error("X86 interrupts may not be called directly");
20522052

2053+
if (IsIndirectCall && !IsWin64 &&
2054+
M->getModuleFlag("import-call-optimization"))
2055+
errorUnsupported(DAG, dl,
2056+
"Indirect calls must have a normal calling convention if "
2057+
"Import Call Optimization is enabled");
2058+
20532059
// Analyze operands of the call, assigning locations to each operand.
20542060
SmallVector<CCValAssign, 16> ArgLocs;
20552061
CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
@@ -2421,6 +2427,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
24212427
InGlue = Chain.getValue(1);
24222428
}
24232429

2430+
bool IsImpCall = false;
24242431
if (DAG.getTarget().getCodeModel() == CodeModel::Large) {
24252432
assert(Is64Bit && "Large code model is only legal in 64-bit mode.");
24262433
// In the 64-bit large code model, we have to make all calls
@@ -2433,7 +2440,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
24332440
// ForCall to true here has the effect of removing WrapperRIP when possible
24342441
// to allow direct calls to be selected without first materializing the
24352442
// address into a register.
2436-
Callee = LowerGlobalOrExternal(Callee, DAG, /*ForCall=*/true);
2443+
Callee = LowerGlobalOrExternal(Callee, DAG, /*ForCall=*/true, &IsImpCall);
24372444
} else if (Subtarget.isTarget64BitILP32() &&
24382445
Callee.getValueType() == MVT::i32) {
24392446
// Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI
@@ -2555,7 +2562,9 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
25552562

25562563
// Returns a chain & a glue for retval copy to use.
25572564
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2558-
if (IsNoTrackIndirectCall) {
2565+
if (IsImpCall) {
2566+
Chain = DAG.getNode(X86ISD::IMP_CALL, dl, NodeTys, Ops);
2567+
} else if (IsNoTrackIndirectCall) {
25592568
Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops);
25602569
} else if (CLI.CB && objcarc::hasAttachedCallOpBundle(CLI.CB)) {
25612570
// Calls with a "clang.arc.attachedcall" bundle are special. They should be

‎llvm/lib/Target/X86/X86InstrCompiler.td

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1313,6 +1313,8 @@ def : Pat<(X86call_rvmarker (i64 tglobaladdr:$rvfunc), (i64 texternalsym:$dst)),
13131313
def : Pat<(X86call_rvmarker (i64 tglobaladdr:$rvfunc), (i64 tglobaladdr:$dst)),
13141314
(CALL64pcrel32_RVMARKER tglobaladdr:$rvfunc, tglobaladdr:$dst)>;
13151315

1316+
def : Pat<(X86imp_call (i64 tglobaladdr:$dst)),
1317+
(CALL64pcrel32 tglobaladdr:$dst)>;
13161318

13171319
// Tailcall stuff. The TCRETURN instructions execute after the epilog, so they
13181320
// can never use callee-saved registers. That is the purpose of the GR64_TC
@@ -1344,7 +1346,11 @@ def : Pat<(X86tcret (i32 texternalsym:$dst), timm:$off),
13441346

13451347
def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off),
13461348
(TCRETURNri64 ptr_rc_tailcall:$dst, timm:$off)>,
1347-
Requires<[In64BitMode, NotUseIndirectThunkCalls]>;
1349+
Requires<[In64BitMode, NotUseIndirectThunkCalls, ImportCallOptimizationDisabled]>;
1350+
1351+
def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off),
1352+
(TCRETURNri64_ImpCall ptr_rc_tailcall:$dst, timm:$off)>,
1353+
Requires<[In64BitMode, NotUseIndirectThunkCalls, ImportCallOptimizationEnabled]>;
13481354

13491355
// Don't fold loads into X86tcret requiring more than 6 regs.
13501356
// There wouldn't be enough scratch registers for base+index.

‎llvm/lib/Target/X86/X86InstrControl.td

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,7 @@ let isCall = 1, Uses = [RSP, SSP], SchedRW = [WriteJump] in {
327327
Requires<[In64BitMode]>;
328328
def CALL64r : I<0xFF, MRM2r, (outs), (ins GR64:$dst),
329329
"call{q}\t{*}$dst", [(X86call GR64:$dst)]>,
330-
Requires<[In64BitMode,NotUseIndirectThunkCalls]>;
330+
Requires<[In64BitMode,NotUseIndirectThunkCalls,ImportCallOptimizationDisabled]>;
331331
def CALL64m : I<0xFF, MRM2m, (outs), (ins i64mem:$dst),
332332
"call{q}\t{*}$dst", [(X86call (loadi64 addr:$dst))]>,
333333
Requires<[In64BitMode,FavorMemIndirectCall,
@@ -357,6 +357,10 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
357357
def TCRETURNri64 : PseudoI<(outs),
358358
(ins ptr_rc_tailcall:$dst, i32imm:$offset),
359359
[]>, Sched<[WriteJump]>;
360+
def TCRETURNri64_ImpCall : PseudoI<(outs),
361+
(ins GR64_A:$dst, i32imm:$offset),
362+
[]>, Sched<[WriteJump]>;
363+
360364
let mayLoad = 1 in
361365
def TCRETURNmi64 : PseudoI<(outs),
362366
(ins i64mem_TC:$dst, i32imm:$offset),
@@ -418,6 +422,10 @@ let isPseudo = 1, isCall = 1, isCodeGenOnly = 1,
418422
def CALL64pcrel32_RVMARKER :
419423
PseudoI<(outs), (ins i64imm:$rvfunc, i64i32imm_brtarget:$dst), []>,
420424
Requires<[In64BitMode]>;
425+
426+
def CALL64r_ImpCall :
427+
PseudoI<(outs), (ins GR64_A:$dst), [(X86call GR64_A:$dst)]>,
428+
Requires<[In64BitMode,NotUseIndirectThunkCalls,ImportCallOptimizationEnabled]>;
421429
}
422430

423431
// Conditional tail calls are similar to the above, but they are branches

‎llvm/lib/Target/X86/X86InstrFragments.td

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,9 @@ def X86call_rvmarker : SDNode<"X86ISD::CALL_RVMARKER", SDT_X86Call,
210210
[SDNPHasChain, SDNPOutGlue, SDNPOptInGlue,
211211
SDNPVariadic]>;
212212

213+
def X86imp_call : SDNode<"X86ISD::IMP_CALL", SDT_X86Call,
214+
[SDNPHasChain, SDNPOutGlue, SDNPOptInGlue,
215+
SDNPVariadic]>;
213216

214217
def X86NoTrackCall : SDNode<"X86ISD::NT_CALL", SDT_X86Call,
215218
[SDNPHasChain, SDNPOutGlue, SDNPOptInGlue,

0 commit comments

Comments
 (0)