Skip to content

Commit 5f88bea

Browse files
borsbors
committed
Auto merge of #75014 - Amanieu:fix_asm_reg, r=nagisa
Work around LLVM issues with explicit register in inline asm Fixes #74658
2 parents 4bb4b96 + 9198e8a commit 5f88bea

File tree

7 files changed

+959
-91
lines changed

7 files changed

+959
-91
lines changed

‎src/librustc_codegen_llvm/asm.rs

+126-29
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,9 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
130130
for (idx, op) in operands.iter().enumerate() {
131131
match *op {
132132
InlineAsmOperandRef::Out { reg, late, place } => {
133-
let ty = if let Some(place) = place {
133+
let mut layout = None;
134+
let ty = if let Some(ref place) = place {
135+
layout = Some(&place.layout);
134136
llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
135137
} else {
136138
// If the output is discarded, we don't really care what
@@ -141,20 +143,21 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
141143
output_types.push(ty);
142144
op_idx.insert(idx, constraints.len());
143145
let prefix = if late { "=" } else { "=&" };
144-
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
146+
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
145147
}
146148
InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
147-
let ty = if let Some(ref out_place) = out_place {
148-
llvm_fixup_output_type(self.cx, reg.reg_class(), &out_place.layout)
149+
let layout = if let Some(ref out_place) = out_place {
150+
&out_place.layout
149151
} else {
150152
// LLVM required tied operands to have the same type,
151153
// so we just use the type of the input.
152-
llvm_fixup_output_type(self.cx, reg.reg_class(), &in_value.layout)
154+
&in_value.layout
153155
};
156+
let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout);
154157
output_types.push(ty);
155158
op_idx.insert(idx, constraints.len());
156159
let prefix = if late { "=" } else { "=&" };
157-
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
160+
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
158161
}
159162
_ => {}
160163
}
@@ -165,11 +168,11 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
165168
for (idx, op) in operands.iter().enumerate() {
166169
match *op {
167170
InlineAsmOperandRef::In { reg, value } => {
168-
let value =
171+
let llval =
169172
llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
170-
inputs.push(value);
173+
inputs.push(llval);
171174
op_idx.insert(idx, constraints.len());
172-
constraints.push(reg_to_llvm(reg));
175+
constraints.push(reg_to_llvm(reg, Some(&value.layout)));
173176
}
174177
InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => {
175178
let value = llvm_fixup_input(
@@ -410,10 +413,80 @@ fn inline_asm_call(
410413
}
411414
}
412415

416+
/// If the register is an xmm/ymm/zmm register then return its index.
417+
fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
418+
match reg {
419+
InlineAsmReg::X86(reg)
420+
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
421+
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
422+
{
423+
Some(reg as u32 - X86InlineAsmReg::xmm0 as u32)
424+
}
425+
InlineAsmReg::X86(reg)
426+
if reg as u32 >= X86InlineAsmReg::ymm0 as u32
427+
&& reg as u32 <= X86InlineAsmReg::ymm15 as u32 =>
428+
{
429+
Some(reg as u32 - X86InlineAsmReg::ymm0 as u32)
430+
}
431+
InlineAsmReg::X86(reg)
432+
if reg as u32 >= X86InlineAsmReg::zmm0 as u32
433+
&& reg as u32 <= X86InlineAsmReg::zmm31 as u32 =>
434+
{
435+
Some(reg as u32 - X86InlineAsmReg::zmm0 as u32)
436+
}
437+
_ => None,
438+
}
439+
}
440+
441+
/// If the register is an AArch64 vector register then return its index.
442+
fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
443+
match reg {
444+
InlineAsmReg::AArch64(reg)
445+
if reg as u32 >= AArch64InlineAsmReg::v0 as u32
446+
&& reg as u32 <= AArch64InlineAsmReg::v31 as u32 =>
447+
{
448+
Some(reg as u32 - AArch64InlineAsmReg::v0 as u32)
449+
}
450+
_ => None,
451+
}
452+
}
453+
413454
/// Converts a register class to an LLVM constraint code.
414-
fn reg_to_llvm(reg: InlineAsmRegOrRegClass) -> String {
455+
fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'tcx>>) -> String {
415456
match reg {
416-
InlineAsmRegOrRegClass::Reg(reg) => format!("{{{}}}", reg.name()),
457+
// For vector registers LLVM wants the register name to match the type size.
458+
InlineAsmRegOrRegClass::Reg(reg) => {
459+
if let Some(idx) = xmm_reg_index(reg) {
460+
let class = if let Some(layout) = layout {
461+
match layout.size.bytes() {
462+
64 => 'z',
463+
32 => 'y',
464+
_ => 'x',
465+
}
466+
} else {
467+
// We use f32 as the type for discarded outputs
468+
'x'
469+
};
470+
format!("{{{}mm{}}}", class, idx)
471+
} else if let Some(idx) = a64_vreg_index(reg) {
472+
let class = if let Some(layout) = layout {
473+
match layout.size.bytes() {
474+
16 => 'q',
475+
8 => 'd',
476+
4 => 's',
477+
2 => 'h',
478+
1 => 'd', // We fixup i8 to i8x8
479+
_ => unreachable!(),
480+
}
481+
} else {
482+
// We use i32 as the type for discarded outputs
483+
's'
484+
};
485+
format!("{{{}{}}}", class, idx)
486+
} else {
487+
format!("{{{}}}", reg.name())
488+
}
489+
}
417490
InlineAsmRegOrRegClass::RegClass(reg) => match reg {
418491
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
419492
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
@@ -600,18 +673,26 @@ fn llvm_fixup_input(
600673
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
601674
Abi::Vector { .. },
602675
) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
676+
(
677+
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
678+
Abi::Scalar(s),
679+
) => {
680+
if let Primitive::Int(Integer::I32, _) = s.value {
681+
bx.bitcast(value, bx.cx.type_f32())
682+
} else {
683+
value
684+
}
685+
}
603686
(
604687
InlineAsmRegClass::Arm(
605-
ArmInlineAsmRegClass::sreg_low16
688+
ArmInlineAsmRegClass::dreg
606689
| ArmInlineAsmRegClass::dreg_low8
607-
| ArmInlineAsmRegClass::qreg_low4
608-
| ArmInlineAsmRegClass::dreg
609-
| ArmInlineAsmRegClass::qreg,
690+
| ArmInlineAsmRegClass::dreg_low16,
610691
),
611692
Abi::Scalar(s),
612693
) => {
613-
if let Primitive::Int(Integer::I32, _) = s.value {
614-
bx.bitcast(value, bx.cx.type_f32())
694+
if let Primitive::Int(Integer::I64, _) = s.value {
695+
bx.bitcast(value, bx.cx.type_f64())
615696
} else {
616697
value
617698
}
@@ -660,18 +741,26 @@ fn llvm_fixup_output(
660741
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
661742
Abi::Vector { .. },
662743
) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
744+
(
745+
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
746+
Abi::Scalar(s),
747+
) => {
748+
if let Primitive::Int(Integer::I32, _) = s.value {
749+
bx.bitcast(value, bx.cx.type_i32())
750+
} else {
751+
value
752+
}
753+
}
663754
(
664755
InlineAsmRegClass::Arm(
665-
ArmInlineAsmRegClass::sreg_low16
756+
ArmInlineAsmRegClass::dreg
666757
| ArmInlineAsmRegClass::dreg_low8
667-
| ArmInlineAsmRegClass::qreg_low4
668-
| ArmInlineAsmRegClass::dreg
669-
| ArmInlineAsmRegClass::qreg,
758+
| ArmInlineAsmRegClass::dreg_low16,
670759
),
671760
Abi::Scalar(s),
672761
) => {
673-
if let Primitive::Int(Integer::I32, _) = s.value {
674-
bx.bitcast(value, bx.cx.type_i32())
762+
if let Primitive::Int(Integer::I64, _) = s.value {
763+
bx.bitcast(value, bx.cx.type_i64())
675764
} else {
676765
value
677766
}
@@ -715,18 +804,26 @@ fn llvm_fixup_output_type(
715804
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
716805
Abi::Vector { .. },
717806
) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
807+
(
808+
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
809+
Abi::Scalar(s),
810+
) => {
811+
if let Primitive::Int(Integer::I32, _) = s.value {
812+
cx.type_f32()
813+
} else {
814+
layout.llvm_type(cx)
815+
}
816+
}
718817
(
719818
InlineAsmRegClass::Arm(
720-
ArmInlineAsmRegClass::sreg_low16
819+
ArmInlineAsmRegClass::dreg
721820
| ArmInlineAsmRegClass::dreg_low8
722-
| ArmInlineAsmRegClass::qreg_low4
723-
| ArmInlineAsmRegClass::dreg
724-
| ArmInlineAsmRegClass::qreg,
821+
| ArmInlineAsmRegClass::dreg_low16,
725822
),
726823
Abi::Scalar(s),
727824
) => {
728-
if let Primitive::Int(Integer::I32, _) = s.value {
729-
cx.type_f32()
825+
if let Primitive::Int(Integer::I64, _) = s.value {
826+
cx.type_f64()
730827
} else {
731828
layout.llvm_type(cx)
732829
}

0 commit comments

Comments
 (0)