Skip to content

Commit 286daaf

Browse files
yonghong-songyonghong-song
committed
[BPF] support atomic instructions
Implement fetch_<op>/fetch_and_<op>/exchange/compare-and-exchange instructions for BPF. Specially, the following gcc intrinsics are implemented. __sync_fetch_and_add (32, 64) __sync_fetch_and_sub (32, 64) __sync_fetch_and_and (32, 64) __sync_fetch_and_or (32, 64) __sync_fetch_and_xor (32, 64) __sync_lock_test_and_set (32, 64) __sync_val_compare_and_swap (32, 64) For __sync_fetch_and_sub, internally, it is implemented as a negation followed by __sync_fetch_and_add. For __sync_lock_test_and_set, despite its name, it actually does an atomic exchange and return the old content. https://gcc.gnu.org/onlinedocs/gcc-4.1.1/gcc/Atomic-Builtins.html For intrinsics like __sync_{add,sub}_and_fetch and __sync_bool_compare_and_swap, the compiler is able to generate codes using __sync_fetch_and_{add,sub} and __sync_val_compare_and_swap. Similar to xadd, atomic xadd, xor and xxor (atomic_<op>) instructions are added for atomic operations which do not have return values. LLVM will check the return value for __sync_fetch_and_{add,and,or,xor}. If the return value is used, instructions atomic_fetch_<op> will be used. Otherwise, atomic_<op> instructions will be used. All new instructions only support 64bit and 32bit with alu32 mode. old xadd instruction still supports 32bit without alu32 mode. For encoding, please take a look at test atomics_2.ll. Differential Revision: https://reviews.llvm.org/D72184
1 parent 517828a commit 286daaf

File tree

8 files changed

+528
-20
lines changed

8 files changed

+528
-20
lines changed

‎llvm/lib/Target/BPF/BPFInstrFormats.td

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ def BPF_MOV : BPFArithOp<0xb>;
4444
def BPF_ARSH : BPFArithOp<0xc>;
4545
def BPF_END : BPFArithOp<0xd>;
4646

47+
def BPF_XCHG : BPFArithOp<0xe>;
48+
def BPF_CMPXCHG : BPFArithOp<0xf>;
49+
4750
class BPFEndDir<bits<1> val> {
4851
bits<1> Value = val;
4952
}
@@ -86,7 +89,13 @@ def BPF_IMM : BPFModeModifer<0x0>;
8689
def BPF_ABS : BPFModeModifer<0x1>;
8790
def BPF_IND : BPFModeModifer<0x2>;
8891
def BPF_MEM : BPFModeModifer<0x3>;
89-
def BPF_XADD : BPFModeModifer<0x6>;
92+
def BPF_ATOMIC : BPFModeModifer<0x6>;
93+
94+
class BPFAtomicFlag<bits<4> val> {
95+
bits<4> Value = val;
96+
}
97+
98+
def BPF_FETCH : BPFAtomicFlag<0x1>;
9099

91100
class InstBPF<dag outs, dag ins, string asmstr, list<dag> pattern>
92101
: Instruction {

‎llvm/lib/Target/BPF/BPFInstrInfo.td

Lines changed: 181 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -617,9 +617,9 @@ let Predicates = [BPFNoALU32] in {
617617
def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>;
618618
}
619619

620-
// Atomics
620+
// Atomic XADD for BPFNoALU32
621621
class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
622-
: TYPE_LD_ST<BPF_XADD.Value, SizeOp.Value,
622+
: TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
623623
(outs GPR:$dst),
624624
(ins MEMri:$addr, GPR:$val),
625625
"lock *("#OpcodeStr#" *)($addr) += $val",
@@ -630,34 +630,206 @@ class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
630630
let Inst{51-48} = addr{19-16}; // base reg
631631
let Inst{55-52} = dst;
632632
let Inst{47-32} = addr{15-0}; // offset
633+
let Inst{7-4} = BPF_ADD.Value;
633634
let BPFClass = BPF_STX;
634635
}
635636

636-
class XADD32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
637-
: TYPE_LD_ST<BPF_XADD.Value, SizeOp.Value,
637+
let Constraints = "$dst = $val" in {
638+
let Predicates = [BPFNoALU32] in {
639+
def XADDW : XADD<BPF_W, "u32", atomic_load_add_32>;
640+
}
641+
}
642+
643+
// Atomic add, and, or, xor
644+
class ATOMIC_NOFETCH<BPFArithOp Opc, string Opstr>
645+
: TYPE_LD_ST<BPF_ATOMIC.Value, BPF_DW.Value,
646+
(outs GPR:$dst),
647+
(ins MEMri:$addr, GPR:$val),
648+
"lock *(u64 *)($addr) " #Opstr# "= $val",
649+
[]> {
650+
bits<4> dst;
651+
bits<20> addr;
652+
653+
let Inst{51-48} = addr{19-16}; // base reg
654+
let Inst{55-52} = dst;
655+
let Inst{47-32} = addr{15-0}; // offset
656+
let Inst{7-4} = Opc.Value;
657+
let BPFClass = BPF_STX;
658+
}
659+
660+
class ATOMIC32_NOFETCH<BPFArithOp Opc, string Opstr>
661+
: TYPE_LD_ST<BPF_ATOMIC.Value, BPF_W.Value,
638662
(outs GPR32:$dst),
639663
(ins MEMri:$addr, GPR32:$val),
640-
"lock *("#OpcodeStr#" *)($addr) += $val",
664+
"lock *(u32 *)($addr) " #Opstr# "= $val",
665+
[]> {
666+
bits<4> dst;
667+
bits<20> addr;
668+
669+
let Inst{51-48} = addr{19-16}; // base reg
670+
let Inst{55-52} = dst;
671+
let Inst{47-32} = addr{15-0}; // offset
672+
let Inst{7-4} = Opc.Value;
673+
let BPFClass = BPF_STX;
674+
}
675+
676+
let Constraints = "$dst = $val" in {
677+
let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
678+
def XADDW32 : ATOMIC32_NOFETCH<BPF_ADD, "+">;
679+
def XANDW32 : ATOMIC32_NOFETCH<BPF_AND, "&">;
680+
def XORW32 : ATOMIC32_NOFETCH<BPF_OR, "|">;
681+
def XXORW32 : ATOMIC32_NOFETCH<BPF_XOR, "^">;
682+
}
683+
684+
def XADDD : ATOMIC_NOFETCH<BPF_ADD, "+">;
685+
def XANDD : ATOMIC_NOFETCH<BPF_AND, "&">;
686+
def XORD : ATOMIC_NOFETCH<BPF_OR, "|">;
687+
def XXORD : ATOMIC_NOFETCH<BPF_XOR, "^">;
688+
}
689+
690+
// Atomic Fetch-and-<add, and, or, xor> operations
691+
class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr,
692+
string OpcStr, PatFrag OpNode>
693+
: TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
694+
(outs GPR:$dst),
695+
(ins MEMri:$addr, GPR:$val),
696+
"$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
697+
[(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> {
698+
bits<4> dst;
699+
bits<20> addr;
700+
701+
let Inst{51-48} = addr{19-16}; // base reg
702+
let Inst{55-52} = dst;
703+
let Inst{47-32} = addr{15-0}; // offset
704+
let Inst{7-4} = Opc.Value;
705+
let Inst{3-0} = BPF_FETCH.Value;
706+
let BPFClass = BPF_STX;
707+
}
708+
709+
class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr,
710+
string OpcStr, PatFrag OpNode>
711+
: TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
712+
(outs GPR32:$dst),
713+
(ins MEMri:$addr, GPR32:$val),
714+
"$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
641715
[(set GPR32:$dst, (OpNode ADDRri:$addr, GPR32:$val))]> {
642716
bits<4> dst;
643717
bits<20> addr;
644718

645719
let Inst{51-48} = addr{19-16}; // base reg
646720
let Inst{55-52} = dst;
647721
let Inst{47-32} = addr{15-0}; // offset
722+
let Inst{7-4} = Opc.Value;
723+
let Inst{3-0} = BPF_FETCH.Value;
648724
let BPFClass = BPF_STX;
649725
}
650726

651727
let Constraints = "$dst = $val" in {
652-
let Predicates = [BPFNoALU32] in {
653-
def XADDW : XADD<BPF_W, "u32", atomic_load_add_32>;
728+
let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
729+
def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add", atomic_load_add_32>;
730+
def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and", atomic_load_and_32>;
731+
def XFORW32 : XFALU32<BPF_W, BPF_OR, "u32", "or", atomic_load_or_32>;
732+
def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor", atomic_load_xor_32>;
654733
}
655734

735+
def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add", atomic_load_add_64>;
736+
def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and", atomic_load_and_64>;
737+
def XFORD : XFALU64<BPF_DW, BPF_OR, "u64", "or", atomic_load_or_64>;
738+
def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor", atomic_load_xor_64>;
739+
}
740+
741+
// atomic_load_sub can be represented as a neg followed
742+
// by an atomic_load_add.
743+
def : Pat<(atomic_load_sub_32 ADDRri:$addr, GPR32:$val),
744+
(XFADDW32 ADDRri:$addr, (NEG_32 GPR32:$val))>;
745+
def : Pat<(atomic_load_sub_64 ADDRri:$addr, GPR:$val),
746+
(XFADDD ADDRri:$addr, (NEG_64 GPR:$val))>;
747+
748+
// Atomic Exchange
749+
class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
750+
: TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
751+
(outs GPR:$dst),
752+
(ins MEMri:$addr, GPR:$val),
753+
"$dst = xchg_"#OpcodeStr#"($addr, $val)",
754+
[(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> {
755+
bits<4> dst;
756+
bits<20> addr;
757+
758+
let Inst{51-48} = addr{19-16}; // base reg
759+
let Inst{55-52} = dst;
760+
let Inst{47-32} = addr{15-0}; // offset
761+
let Inst{7-4} = BPF_XCHG.Value;
762+
let Inst{3-0} = BPF_FETCH.Value;
763+
let BPFClass = BPF_STX;
764+
}
765+
766+
class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
767+
: TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
768+
(outs GPR32:$dst),
769+
(ins MEMri:$addr, GPR32:$val),
770+
"$dst = xchg32_"#OpcodeStr#"($addr, $val)",
771+
[(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> {
772+
bits<4> dst;
773+
bits<20> addr;
774+
775+
let Inst{51-48} = addr{19-16}; // base reg
776+
let Inst{55-52} = dst;
777+
let Inst{47-32} = addr{15-0}; // offset
778+
let Inst{7-4} = BPF_XCHG.Value;
779+
let Inst{3-0} = BPF_FETCH.Value;
780+
let BPFClass = BPF_STX;
781+
}
782+
783+
let Constraints = "$dst = $val" in {
656784
let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
657-
def XADDW32 : XADD32<BPF_W, "u32", atomic_load_add_32>;
785+
def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_32>;
658786
}
659787

660-
def XADDD : XADD<BPF_DW, "u64", atomic_load_add_64>;
788+
def XCHGD : XCHG<BPF_DW, "64", atomic_swap_64>;
789+
}
790+
791+
// Compare-And-Exchange
792+
class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
793+
: TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
794+
(outs),
795+
(ins MEMri:$addr, GPR:$new),
796+
"r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)",
797+
[(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> {
798+
bits<4> new;
799+
bits<20> addr;
800+
801+
let Inst{51-48} = addr{19-16}; // base reg
802+
let Inst{55-52} = new;
803+
let Inst{47-32} = addr{15-0}; // offset
804+
let Inst{7-4} = BPF_CMPXCHG.Value;
805+
let Inst{3-0} = BPF_FETCH.Value;
806+
let BPFClass = BPF_STX;
807+
}
808+
809+
class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
810+
: TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
811+
(outs),
812+
(ins MEMri:$addr, GPR32:$new),
813+
"w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)",
814+
[(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> {
815+
bits<4> new;
816+
bits<20> addr;
817+
818+
let Inst{51-48} = addr{19-16}; // base reg
819+
let Inst{55-52} = new;
820+
let Inst{47-32} = addr{15-0}; // offset
821+
let Inst{7-4} = BPF_CMPXCHG.Value;
822+
let Inst{3-0} = BPF_FETCH.Value;
823+
let BPFClass = BPF_STX;
824+
}
825+
826+
let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0],
827+
DecoderNamespace = "BPFALU32" in {
828+
def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_32>;
829+
}
830+
831+
let Defs = [R0], Uses = [R0] in {
832+
def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_64>;
661833
}
662834

663835
// bswap16, bswap32, bswap64

‎llvm/lib/Target/BPF/BPFMIChecking.cpp

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,15 @@ struct BPFMIPreEmitChecking : public MachineFunctionPass {
4141
// Initialize class variables.
4242
void initialize(MachineFunction &MFParm);
4343

44-
void checkingIllegalXADD(void);
44+
bool processAtomicInsts(void);
4545

4646
public:
4747

4848
// Main entry point for this pass.
4949
bool runOnMachineFunction(MachineFunction &MF) override {
5050
if (!skipFunction(MF.getFunction())) {
5151
initialize(MF);
52-
checkingIllegalXADD();
52+
return processAtomicInsts();
5353
}
5454
return false;
5555
}
@@ -151,7 +151,7 @@ static bool hasLiveDefs(const MachineInstr &MI, const TargetRegisterInfo *TRI) {
151151
return false;
152152
}
153153

154-
void BPFMIPreEmitChecking::checkingIllegalXADD(void) {
154+
bool BPFMIPreEmitChecking::processAtomicInsts(void) {
155155
for (MachineBasicBlock &MBB : *MF) {
156156
for (MachineInstr &MI : MBB) {
157157
if (MI.getOpcode() != BPF::XADDW &&
@@ -172,7 +172,71 @@ void BPFMIPreEmitChecking::checkingIllegalXADD(void) {
172172
}
173173
}
174174

175-
return;
175+
// Check return values of atomic_fetch_and_{add,and,or,xor}.
176+
// If the return is not used, the atomic_fetch_and_<op> instruction
177+
// is replaced with atomic_<op> instruction.
178+
MachineInstr *ToErase = nullptr;
179+
bool Changed = false;
180+
const BPFInstrInfo *TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
181+
for (MachineBasicBlock &MBB : *MF) {
182+
for (MachineInstr &MI : MBB) {
183+
if (ToErase) {
184+
ToErase->eraseFromParent();
185+
ToErase = nullptr;
186+
}
187+
188+
if (MI.getOpcode() != BPF::XFADDW32 && MI.getOpcode() != BPF::XFADDD &&
189+
MI.getOpcode() != BPF::XFANDW32 && MI.getOpcode() != BPF::XFANDD &&
190+
MI.getOpcode() != BPF::XFXORW32 && MI.getOpcode() != BPF::XFXORD &&
191+
MI.getOpcode() != BPF::XFORW32 && MI.getOpcode() != BPF::XFORD)
192+
continue;
193+
194+
if (hasLiveDefs(MI, TRI))
195+
continue;
196+
197+
LLVM_DEBUG(dbgs() << "Transforming "; MI.dump());
198+
unsigned newOpcode;
199+
switch (MI.getOpcode()) {
200+
case BPF::XFADDW32:
201+
newOpcode = BPF::XADDW32;
202+
break;
203+
case BPF::XFADDD:
204+
newOpcode = BPF::XADDD;
205+
break;
206+
case BPF::XFANDW32:
207+
newOpcode = BPF::XANDW32;
208+
break;
209+
case BPF::XFANDD:
210+
newOpcode = BPF::XANDD;
211+
break;
212+
case BPF::XFXORW32:
213+
newOpcode = BPF::XXORW32;
214+
break;
215+
case BPF::XFXORD:
216+
newOpcode = BPF::XXORD;
217+
break;
218+
case BPF::XFORW32:
219+
newOpcode = BPF::XORW32;
220+
break;
221+
case BPF::XFORD:
222+
newOpcode = BPF::XORD;
223+
break;
224+
default:
225+
llvm_unreachable("Incorrect Atomic Instruction Opcode");
226+
}
227+
228+
BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(newOpcode))
229+
.add(MI.getOperand(0))
230+
.add(MI.getOperand(1))
231+
.add(MI.getOperand(2))
232+
.add(MI.getOperand(3));
233+
234+
ToErase = &MI;
235+
Changed = true;
236+
}
237+
}
238+
239+
return Changed;
176240
}
177241

178242
} // end default namespace

‎llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ class BPFDisassembler : public MCDisassembler {
5858
BPF_MEM = 0x3,
5959
BPF_LEN = 0x4,
6060
BPF_MSH = 0x5,
61-
BPF_XADD = 0x6
61+
BPF_ATOMIC = 0x6
6262
};
6363

6464
BPFDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx)
@@ -176,7 +176,7 @@ DecodeStatus BPFDisassembler::getInstruction(MCInst &Instr, uint64_t &Size,
176176
uint8_t InstMode = getInstMode(Insn);
177177
if ((InstClass == BPF_LDX || InstClass == BPF_STX) &&
178178
getInstSize(Insn) != BPF_DW &&
179-
(InstMode == BPF_MEM || InstMode == BPF_XADD) &&
179+
(InstMode == BPF_MEM || InstMode == BPF_ATOMIC) &&
180180
STI.getFeatureBits()[BPF::ALU32])
181181
Result = decodeInstruction(DecoderTableBPFALU3264, Instr, Insn, Address,
182182
this, STI);

‎llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,12 +159,18 @@ void BPFMCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
159159
uint64_t BPFMCCodeEmitter::getMemoryOpValue(const MCInst &MI, unsigned Op,
160160
SmallVectorImpl<MCFixup> &Fixups,
161161
const MCSubtargetInfo &STI) const {
162+
// For CMPXCHG instructions, output is implicitly in R0/W0,
163+
// so memory operand starts from operand 0.
164+
int MemOpStartIndex = 1, Opcode = MI.getOpcode();
165+
if (Opcode == BPF::CMPXCHGW32 || Opcode == BPF::CMPXCHGD)
166+
MemOpStartIndex = 0;
167+
162168
uint64_t Encoding;
163-
const MCOperand Op1 = MI.getOperand(1);
169+
const MCOperand Op1 = MI.getOperand(MemOpStartIndex);
164170
assert(Op1.isReg() && "First operand is not register.");
165171
Encoding = MRI.getEncodingValue(Op1.getReg());
166172
Encoding <<= 16;
167-
MCOperand Op2 = MI.getOperand(2);
173+
MCOperand Op2 = MI.getOperand(MemOpStartIndex + 1);
168174
assert(Op2.isImm() && "Second operand is not immediate.");
169175
Encoding |= Op2.getImm() & 0xffff;
170176
return Encoding;

0 commit comments

Comments
 (0)