Skip to content

Commit d7e8a74

Browse files
ahmedbougachaahmedbougacha
authored
[AArch64][PAC] Lower auth/resign into checked sequence. (llvm#79024)
This introduces 3 hardening modes in the authentication step of auth/resign lowering: - unchecked, which uses the AUT instructions as-is - poison, which detects authentication failure (using an XPAC+CMP sequence), explicitly yielding the XPAC result rather than the AUT result, to avoid leaking - trap, which additionally traps on authentication failure, using BRK #0xC470 + key (IA C470, IB C471, DA C472, DB C473.) Not all modes are necessarily useful in all contexts, and there are more performant alternative lowerings in specific contexts (e.g., when I/D TBI enablement is a target ABI guarantee.) These will be implemented separately. This is controlled by the `ptrauth-auth-traps` function attributes, and can be overridden using `-aarch64-ptrauth-auth-checks=`. This also adds the FPAC extension, which we haven't needed before, to improve isel when we can rely on HW checking.
1 parent 67937a3 commit d7e8a74

8 files changed

+1696
-1
lines changed

‎llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp

+234
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,15 @@
6868

6969
using namespace llvm;
7070

71+
enum PtrauthCheckMode { Default, Unchecked, Poison, Trap };
72+
static cl::opt<PtrauthCheckMode> PtrauthAuthChecks(
73+
"aarch64-ptrauth-auth-checks", cl::Hidden,
74+
cl::values(clEnumValN(Unchecked, "none", "don't test for failure"),
75+
clEnumValN(Poison, "poison", "poison on failure"),
76+
clEnumValN(Trap, "trap", "trap on failure")),
77+
cl::desc("Check pointer authentication auth/resign failures"),
78+
cl::init(Default));
79+
7180
#define DEBUG_TYPE "asm-printer"
7281

7382
namespace {
@@ -134,6 +143,10 @@ class AArch64AsmPrinter : public AsmPrinter {
134143

135144
// Emit the sequence for BRA/BLRA (authenticate + branch/call).
136145
void emitPtrauthBranch(const MachineInstr *MI);
146+
147+
// Emit the sequence for AUT or AUTPAC.
148+
void emitPtrauthAuthResign(const MachineInstr *MI);
149+
137150
// Emit the sequence to compute a discriminator into x17, or reuse AddrDisc.
138151
unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc,
139152
unsigned &InstsEmitted);
@@ -1760,6 +1773,222 @@ unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
17601773
return AArch64::X17;
17611774
}
17621775

1776+
void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
1777+
unsigned InstsEmitted = 0;
1778+
const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
1779+
1780+
// We can expand AUT/AUTPAC into 3 possible sequences:
1781+
// - unchecked:
1782+
// autia x16, x0
1783+
// pacib x16, x1 ; if AUTPAC
1784+
//
1785+
// - checked and clearing:
1786+
// mov x17, x0
1787+
// movk x17, #disc, lsl #48
1788+
// autia x16, x17
1789+
// mov x17, x16
1790+
// xpaci x17
1791+
// cmp x16, x17
1792+
// b.eq Lsuccess
1793+
// mov x16, x17
1794+
// b Lend
1795+
// Lsuccess:
1796+
// mov x17, x1
1797+
// movk x17, #disc, lsl #48
1798+
// pacib x16, x17
1799+
// Lend:
1800+
// Where we only emit the AUT if we started with an AUT.
1801+
//
1802+
// - checked and trapping:
1803+
// mov x17, x0
1804+
// movk x17, #disc, lsl #48
1805+
// autia x16, x0
1806+
// mov x17, x16
1807+
// xpaci x17
1808+
// cmp x16, x17
1809+
// b.eq Lsuccess
1810+
// brk #<0xc470 + aut key>
1811+
// Lsuccess:
1812+
// mov x17, x1
1813+
// movk x17, #disc, lsl #48
1814+
// pacib x16, x17 ; if AUTPAC
1815+
// Where the b.eq skips over the trap if the PAC is valid.
1816+
//
1817+
// This sequence is expensive, but we need more information to be able to
1818+
// do better.
1819+
//
1820+
// We can't TBZ the poison bit because EnhancedPAC2 XORs the PAC bits
1821+
// on failure.
1822+
// We can't TST the PAC bits because we don't always know how the address
1823+
// space is setup for the target environment (and the bottom PAC bit is
1824+
// based on that).
1825+
// Either way, we also don't always know whether TBI is enabled or not for
1826+
// the specific target environment.
1827+
1828+
// By default, auth/resign sequences check for auth failures.
1829+
bool ShouldCheck = true;
1830+
// In the checked sequence, we only trap if explicitly requested.
1831+
bool ShouldTrap = MF->getFunction().hasFnAttribute("ptrauth-auth-traps");
1832+
1833+
// On an FPAC CPU, you get traps whether you want them or not: there's
1834+
// no point in emitting checks or traps.
1835+
if (STI->hasFPAC())
1836+
ShouldCheck = ShouldTrap = false;
1837+
1838+
// However, command-line flags can override this, for experimentation.
1839+
switch (PtrauthAuthChecks) {
1840+
case PtrauthCheckMode::Default:
1841+
break;
1842+
case PtrauthCheckMode::Unchecked:
1843+
ShouldCheck = ShouldTrap = false;
1844+
break;
1845+
case PtrauthCheckMode::Poison:
1846+
ShouldCheck = true;
1847+
ShouldTrap = false;
1848+
break;
1849+
case PtrauthCheckMode::Trap:
1850+
ShouldCheck = ShouldTrap = true;
1851+
break;
1852+
}
1853+
1854+
auto AUTKey = (AArch64PACKey::ID)MI->getOperand(0).getImm();
1855+
uint64_t AUTDisc = MI->getOperand(1).getImm();
1856+
unsigned AUTAddrDisc = MI->getOperand(2).getReg();
1857+
1858+
unsigned XPACOpc = getXPACOpcodeForKey(AUTKey);
1859+
1860+
// Compute aut discriminator into x17
1861+
assert(isUInt<16>(AUTDisc));
1862+
unsigned AUTDiscReg =
1863+
emitPtrauthDiscriminator(AUTDisc, AUTAddrDisc, InstsEmitted);
1864+
bool AUTZero = AUTDiscReg == AArch64::XZR;
1865+
unsigned AUTOpc = getAUTOpcodeForKey(AUTKey, AUTZero);
1866+
1867+
// autiza x16 ; if AUTZero
1868+
// autia x16, x17 ; if !AUTZero
1869+
MCInst AUTInst;
1870+
AUTInst.setOpcode(AUTOpc);
1871+
AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
1872+
AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
1873+
if (!AUTZero)
1874+
AUTInst.addOperand(MCOperand::createReg(AUTDiscReg));
1875+
EmitToStreamer(*OutStreamer, AUTInst);
1876+
++InstsEmitted;
1877+
1878+
// Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done.
1879+
if (!IsAUTPAC && (!ShouldCheck || !ShouldTrap)) {
1880+
assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1881+
return;
1882+
}
1883+
1884+
MCSymbol *EndSym = nullptr;
1885+
1886+
// Checked sequences do an additional strip-and-compare.
1887+
if (ShouldCheck) {
1888+
MCSymbol *SuccessSym = createTempSymbol("auth_success_");
1889+
1890+
// XPAC has tied src/dst: use x17 as a temporary copy.
1891+
// mov x17, x16
1892+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
1893+
.addReg(AArch64::X17)
1894+
.addReg(AArch64::XZR)
1895+
.addReg(AArch64::X16)
1896+
.addImm(0));
1897+
++InstsEmitted;
1898+
1899+
// xpaci x17
1900+
EmitToStreamer(
1901+
*OutStreamer,
1902+
MCInstBuilder(XPACOpc).addReg(AArch64::X17).addReg(AArch64::X17));
1903+
++InstsEmitted;
1904+
1905+
// cmp x16, x17
1906+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXrs)
1907+
.addReg(AArch64::XZR)
1908+
.addReg(AArch64::X16)
1909+
.addReg(AArch64::X17)
1910+
.addImm(0));
1911+
++InstsEmitted;
1912+
1913+
// b.eq Lsuccess
1914+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::Bcc)
1915+
.addImm(AArch64CC::EQ)
1916+
.addExpr(MCSymbolRefExpr::create(
1917+
SuccessSym, OutContext)));
1918+
++InstsEmitted;
1919+
1920+
if (ShouldTrap) {
1921+
// Trapping sequences do a 'brk'.
1922+
// brk #<0xc470 + aut key>
1923+
EmitToStreamer(*OutStreamer,
1924+
MCInstBuilder(AArch64::BRK).addImm(0xc470 | AUTKey));
1925+
++InstsEmitted;
1926+
} else {
1927+
// Non-trapping checked sequences return the stripped result in x16,
1928+
// skipping over the PAC if there is one.
1929+
1930+
// FIXME: can we simply return the AUT result, already in x16? without..
1931+
// ..traps this is usable as an oracle anyway, based on high bits
1932+
// mov x17, x16
1933+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
1934+
.addReg(AArch64::X16)
1935+
.addReg(AArch64::XZR)
1936+
.addReg(AArch64::X17)
1937+
.addImm(0));
1938+
++InstsEmitted;
1939+
1940+
if (IsAUTPAC) {
1941+
EndSym = createTempSymbol("resign_end_");
1942+
1943+
// b Lend
1944+
EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B)
1945+
.addExpr(MCSymbolRefExpr::create(
1946+
EndSym, OutContext)));
1947+
++InstsEmitted;
1948+
}
1949+
}
1950+
1951+
// If the auth check succeeds, we can continue.
1952+
// Lsuccess:
1953+
OutStreamer->emitLabel(SuccessSym);
1954+
}
1955+
1956+
// We already emitted unchecked and checked-but-non-trapping AUTs.
1957+
// That left us with trapping AUTs, and AUTPACs.
1958+
// Trapping AUTs don't need PAC: we're done.
1959+
if (!IsAUTPAC) {
1960+
assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1961+
return;
1962+
}
1963+
1964+
auto PACKey = (AArch64PACKey::ID)MI->getOperand(3).getImm();
1965+
uint64_t PACDisc = MI->getOperand(4).getImm();
1966+
unsigned PACAddrDisc = MI->getOperand(5).getReg();
1967+
1968+
// Compute pac discriminator into x17
1969+
assert(isUInt<16>(PACDisc));
1970+
unsigned PACDiscReg =
1971+
emitPtrauthDiscriminator(PACDisc, PACAddrDisc, InstsEmitted);
1972+
bool PACZero = PACDiscReg == AArch64::XZR;
1973+
unsigned PACOpc = getPACOpcodeForKey(PACKey, PACZero);
1974+
1975+
// pacizb x16 ; if PACZero
1976+
// pacib x16, x17 ; if !PACZero
1977+
MCInst PACInst;
1978+
PACInst.setOpcode(PACOpc);
1979+
PACInst.addOperand(MCOperand::createReg(AArch64::X16));
1980+
PACInst.addOperand(MCOperand::createReg(AArch64::X16));
1981+
if (!PACZero)
1982+
PACInst.addOperand(MCOperand::createReg(PACDiscReg));
1983+
EmitToStreamer(*OutStreamer, PACInst);
1984+
++InstsEmitted;
1985+
1986+
assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1987+
// Lend:
1988+
if (EndSym)
1989+
OutStreamer->emitLabel(EndSym);
1990+
}
1991+
17631992
void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
17641993
unsigned InstsEmitted = 0;
17651994
bool IsCall = MI->getOpcode() == AArch64::BLRA;
@@ -2214,6 +2443,11 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
22142443
return;
22152444
}
22162445

2446+
case AArch64::AUT:
2447+
case AArch64::AUTPAC:
2448+
emitPtrauthAuthResign(MI);
2449+
return;
2450+
22172451
case AArch64::LOADauthptrstatic:
22182452
LowerLOADauthptrstatic(*MI);
22192453
return;

‎llvm/lib/Target/AArch64/AArch64Features.td

+3
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,9 @@ def FeatureJS : ExtensionWithMArch<"jsconv", "JS", "FEAT_JSCVT",
185185
"Enable Armv8.3-A JavaScript FP conversion instructions",
186186
[FeatureFPARMv8]>;
187187

188+
def FeatureFPAC : Extension<"fpac", "FPAC", "FEAT_FPAC",
189+
"Enable v8.3-A Pointer Authentication Faulting enhancement">;
190+
188191
def FeatureCCIDX : Extension<"ccidx", "CCIDX", "FEAT_CCIDX",
189192
"Enable Armv8.3-A Extend of the CCSIDR number of sets">;
190193

‎llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

+102
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,9 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
366366

367367
bool tryIndexedLoad(SDNode *N);
368368

369+
void SelectPtrauthAuth(SDNode *N);
370+
void SelectPtrauthResign(SDNode *N);
371+
369372
bool trySelectStackSlotTagP(SDNode *N);
370373
void SelectTagP(SDNode *N);
371374

@@ -1481,6 +1484,96 @@ void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
14811484
ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
14821485
}
14831486

1487+
static std::tuple<SDValue, SDValue>
1488+
extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
1489+
SDLoc DL(Disc);
1490+
SDValue AddrDisc;
1491+
SDValue ConstDisc;
1492+
1493+
// If this is a blend, remember the constant and address discriminators.
1494+
// Otherwise, it's either a constant discriminator, or a non-blended
1495+
// address discriminator.
1496+
if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1497+
Disc->getConstantOperandVal(0) == Intrinsic::ptrauth_blend) {
1498+
AddrDisc = Disc->getOperand(1);
1499+
ConstDisc = Disc->getOperand(2);
1500+
} else {
1501+
ConstDisc = Disc;
1502+
}
1503+
1504+
// If the constant discriminator (either the blend RHS, or the entire
1505+
// discriminator value) isn't a 16-bit constant, bail out, and let the
1506+
// discriminator be computed separately.
1507+
auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
1508+
if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
1509+
return std::make_tuple(DAG->getTargetConstant(0, DL, MVT::i64), Disc);
1510+
1511+
// If there's no address discriminator, use XZR directly.
1512+
if (!AddrDisc)
1513+
AddrDisc = DAG->getRegister(AArch64::XZR, MVT::i64);
1514+
1515+
return std::make_tuple(
1516+
DAG->getTargetConstant(ConstDiscN->getZExtValue(), DL, MVT::i64),
1517+
AddrDisc);
1518+
}
1519+
1520+
void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
1521+
SDLoc DL(N);
1522+
// IntrinsicID is operand #0
1523+
SDValue Val = N->getOperand(1);
1524+
SDValue AUTKey = N->getOperand(2);
1525+
SDValue AUTDisc = N->getOperand(3);
1526+
1527+
unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1528+
AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64);
1529+
1530+
SDValue AUTAddrDisc, AUTConstDisc;
1531+
std::tie(AUTConstDisc, AUTAddrDisc) =
1532+
extractPtrauthBlendDiscriminators(AUTDisc, CurDAG);
1533+
1534+
SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
1535+
AArch64::X16, Val, SDValue());
1536+
SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(1)};
1537+
1538+
SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT, DL, MVT::i64, Ops);
1539+
ReplaceNode(N, AUT);
1540+
return;
1541+
}
1542+
1543+
void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
1544+
SDLoc DL(N);
1545+
// IntrinsicID is operand #0
1546+
SDValue Val = N->getOperand(1);
1547+
SDValue AUTKey = N->getOperand(2);
1548+
SDValue AUTDisc = N->getOperand(3);
1549+
SDValue PACKey = N->getOperand(4);
1550+
SDValue PACDisc = N->getOperand(5);
1551+
1552+
unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1553+
unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
1554+
1555+
AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64);
1556+
PACKey = CurDAG->getTargetConstant(PACKeyC, DL, MVT::i64);
1557+
1558+
SDValue AUTAddrDisc, AUTConstDisc;
1559+
std::tie(AUTConstDisc, AUTAddrDisc) =
1560+
extractPtrauthBlendDiscriminators(AUTDisc, CurDAG);
1561+
1562+
SDValue PACAddrDisc, PACConstDisc;
1563+
std::tie(PACConstDisc, PACAddrDisc) =
1564+
extractPtrauthBlendDiscriminators(PACDisc, CurDAG);
1565+
1566+
SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
1567+
AArch64::X16, Val, SDValue());
1568+
1569+
SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1570+
PACConstDisc, PACAddrDisc, X16Copy.getValue(1)};
1571+
1572+
SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC, DL, MVT::i64, Ops);
1573+
ReplaceNode(N, AUTPAC);
1574+
return;
1575+
}
1576+
14841577
bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
14851578
LoadSDNode *LD = cast<LoadSDNode>(N);
14861579
if (LD->isUnindexed())
@@ -5437,6 +5530,15 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
54375530
case Intrinsic::aarch64_tagp:
54385531
SelectTagP(Node);
54395532
return;
5533+
5534+
case Intrinsic::ptrauth_auth:
5535+
SelectPtrauthAuth(Node);
5536+
return;
5537+
5538+
case Intrinsic::ptrauth_resign:
5539+
SelectPtrauthResign(Node);
5540+
return;
5541+
54405542
case Intrinsic::aarch64_neon_tbl2:
54415543
SelectTable(Node, 2,
54425544
VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,

0 commit comments

Comments
 (0)