Skip to content

Commit 9aa9a84

Browse files
committed
Allow transmutes to produce OperandValues instead of always using allocas
LLVM can usually optimize these away, but especially for things like transmutes of newtypes it's silly to generate the `alloc`+`store`+`load` at all when it's actually a nop at LLVM level.
1 parent 480068c commit 9aa9a84

File tree

6 files changed

+381
-74
lines changed

6 files changed

+381
-74
lines changed

compiler/rustc_codegen_ssa/src/mir/operand.rs

+17-1
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,26 @@ pub enum OperandValue<V> {
2323
/// to be valid for the operand's lifetime.
2424
/// The second value, if any, is the extra data (vtable or length)
2525
/// which indicates that it refers to an unsized rvalue.
26+
///
27+
/// An `OperandValue` has this variant for types which are neither
28+
/// `Immediate` nor `Pair`s. The backend value in this variant must be a
29+
/// pointer to the *non*-immediate backend type. That pointee type is the
30+
/// one returned by [`LayoutTypeMethods::backend_type`].
2631
Ref(V, Option<V>, Align),
27-
/// A single LLVM value.
32+
/// A single LLVM immediate value.
33+
///
34+
/// An `OperandValue` *must* be this variant for any type for which
35+
/// [`LayoutTypeMethods::is_backend_immediate`] returns `true`.
36+
/// The backend value in this variant must be the *immediate* backend type,
37+
/// as returned by [`LayoutTypeMethods::immediate_backend_type`].
2838
Immediate(V),
2939
/// A pair of immediate LLVM values. Used by fat pointers too.
40+
///
41+
/// An `OperandValue` *must* be this variant for any type for which
42+
/// [`LayoutTypeMethods::is_backend_scalar_pair`] returns `true`.
43+
/// The backend values in this variant must be the *immediate* backend types,
44+
/// as returned by [`LayoutTypeMethods::scalar_pair_element_backend_type`]
45+
/// with `immediate: true`.
3046
Pair(V, V),
3147
}
3248

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

+159-19
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use crate::MemFlags;
1010
use rustc_middle::mir;
1111
use rustc_middle::mir::Operand;
1212
use rustc_middle::ty::cast::{CastTy, IntTy};
13-
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
13+
use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
1414
use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
1515
use rustc_span::source_map::{Span, DUMMY_SP};
1616
use rustc_target::abi::{self, FIRST_VARIANT};
@@ -159,8 +159,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
159159
debug_assert!(dst.layout.is_sized());
160160

161161
if src.layout.size != dst.layout.size
162-
|| src.layout.abi == abi::Abi::Uninhabited
163-
|| dst.layout.abi == abi::Abi::Uninhabited
162+
|| src.layout.abi.is_uninhabited()
163+
|| dst.layout.abi.is_uninhabited()
164164
{
165165
// In all of these cases it's UB to run this transmute, but that's
166166
// known statically so might as well trap for it, rather than just
@@ -169,22 +169,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
169169
return;
170170
}
171171

172-
let size_in_bytes = src.layout.size.bytes();
173-
if size_in_bytes == 0 {
174-
// Nothing to write
172+
if let Some(val) = self.codegen_transmute_operand(bx, src, dst.layout) {
173+
val.store(bx, dst);
175174
return;
176175
}
177176

178177
match src.val {
179-
OperandValue::Ref(src_llval, meta, src_align) => {
180-
debug_assert_eq!(meta, None);
181-
// For a place-to-place transmute, call `memcpy` directly so that
182-
// both arguments get the best-available alignment information.
183-
let bytes = bx.cx().const_usize(size_in_bytes);
184-
let flags = MemFlags::empty();
185-
bx.memcpy(dst.llval, dst.align, src_llval, src_align, bytes, flags);
178+
OperandValue::Ref(..) => {
179+
span_bug!(
180+
self.mir.span,
181+
"Operand path should have handled transmute \
182+
from `Ref` {src:?} to place {dst:?}"
183+
);
186184
}
187-
OperandValue::Immediate(_) | OperandValue::Pair(_, _) => {
185+
OperandValue::Immediate(..) | OperandValue::Pair(..) => {
188186
// When we have immediate(s), the alignment of the source is irrelevant,
189187
// so we can store them using the destination's alignment.
190188
let llty = bx.backend_type(src.layout);
@@ -194,6 +192,94 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
194192
}
195193
}
196194

195+
/// Attempts to transmute an `OperandValue` to another `OperandValue`.
196+
///
197+
/// Returns `None` for cases that can't work in that framework, such as for
198+
/// `Immediate`->`Ref` that needs an `alloc` to get the location.
199+
fn codegen_transmute_operand(
200+
&mut self,
201+
bx: &mut Bx,
202+
operand: OperandRef<'tcx, Bx::Value>,
203+
cast: TyAndLayout<'tcx>,
204+
) -> Option<OperandValue<Bx::Value>> {
205+
// Callers already checked that the layout sizes match
206+
debug_assert_eq!(operand.layout.size, cast.size);
207+
208+
let operand_kind = self.value_kind(operand.layout);
209+
let cast_kind = self.value_kind(cast);
210+
211+
match operand.val {
212+
OperandValue::Ref(ptr, meta, align) => {
213+
debug_assert_eq!(meta, None);
214+
debug_assert!(matches!(operand_kind, OperandValueKind::Ref));
215+
let cast_bty = bx.backend_type(cast);
216+
let cast_ptr = bx.pointercast(ptr, bx.type_ptr_to(cast_bty));
217+
let fake_place = PlaceRef::new_sized_aligned(cast_ptr, cast, align);
218+
Some(bx.load_operand(fake_place).val)
219+
}
220+
OperandValue::Immediate(imm) => {
221+
let OperandValueKind::Immediate(in_scalar) = operand_kind else {
222+
bug!("Found {operand_kind:?} for operand {operand:?}");
223+
};
224+
if let OperandValueKind::Immediate(out_scalar) = cast_kind {
225+
let cast_bty = bx.backend_type(cast);
226+
Some(OperandValue::Immediate(Self::transmute_immediate(
227+
bx, imm, in_scalar, out_scalar, cast_bty,
228+
)))
229+
} else {
230+
None
231+
}
232+
}
233+
OperandValue::Pair(imm_a, imm_b) => {
234+
let OperandValueKind::Pair(in_a, in_b) = operand_kind else {
235+
bug!("Found {operand_kind:?} for operand {operand:?}");
236+
};
237+
if let OperandValueKind::Pair(out_a, out_b) = cast_kind {
238+
let out_a_ibty = bx.scalar_pair_element_backend_type(cast, 0, false);
239+
let out_b_ibty = bx.scalar_pair_element_backend_type(cast, 1, false);
240+
Some(OperandValue::Pair(
241+
Self::transmute_immediate(bx, imm_a, in_a, out_a, out_a_ibty),
242+
Self::transmute_immediate(bx, imm_b, in_b, out_b, out_b_ibty),
243+
))
244+
} else {
245+
None
246+
}
247+
}
248+
}
249+
}
250+
251+
/// Transmutes one of the immediates from an [`OperandValue::Immediate`]
252+
/// or an [`OperandValue::Pair`] to an immediate of the target type.
253+
///
254+
/// `to_backend_ty` must be the *non*-immediate backend type (so it will be
255+
/// `i8`, not `i1`, for `bool`-like types.)
256+
fn transmute_immediate(
257+
bx: &mut Bx,
258+
mut imm: Bx::Value,
259+
from_scalar: abi::Scalar,
260+
to_scalar: abi::Scalar,
261+
to_backend_ty: Bx::Type,
262+
) -> Bx::Value {
263+
use abi::Primitive::*;
264+
imm = bx.from_immediate(imm);
265+
imm = match (from_scalar.primitive(), to_scalar.primitive()) {
266+
(Int(..) | F32 | F64, Int(..) | F32 | F64) => bx.bitcast(imm, to_backend_ty),
267+
(Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty),
268+
(Int(..), Pointer(..)) => bx.inttoptr(imm, to_backend_ty),
269+
(Pointer(..), Int(..)) => bx.ptrtoint(imm, to_backend_ty),
270+
(F32 | F64, Pointer(..)) => {
271+
let int_imm = bx.bitcast(imm, bx.cx().type_isize());
272+
bx.inttoptr(int_imm, to_backend_ty)
273+
}
274+
(Pointer(..), F32 | F64) => {
275+
let int_imm = bx.ptrtoint(imm, bx.cx().type_isize());
276+
bx.bitcast(int_imm, to_backend_ty)
277+
}
278+
};
279+
imm = bx.to_immediate_scalar(imm, to_scalar);
280+
imm
281+
}
282+
197283
pub fn codegen_rvalue_unsized(
198284
&mut self,
199285
bx: &mut Bx,
@@ -396,7 +482,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
396482
OperandValue::Immediate(newval)
397483
}
398484
mir::CastKind::Transmute => {
399-
bug!("Transmute operand {:?} in `codegen_rvalue_operand`", operand);
485+
self.codegen_transmute_operand(bx, operand, cast).unwrap_or_else(|| {
486+
bug!("Unsupported transmute-as-operand of {operand:?} to {cast:?}");
487+
})
400488
}
401489
};
402490
OperandRef { val, layout: cast }
@@ -739,10 +827,36 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
739827
impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
740828
pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool {
741829
match *rvalue {
742-
mir::Rvalue::Cast(mir::CastKind::Transmute, ..) =>
743-
// FIXME: Now that transmute is an Rvalue, it would be nice if
744-
// it could create `Immediate`s for scalars, where possible.
745-
false,
830+
mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, cast_ty) => {
831+
let operand_ty = operand.ty(self.mir, self.cx.tcx());
832+
let cast_layout = self.cx.layout_of(self.monomorphize(cast_ty));
833+
let operand_layout = self.cx.layout_of(self.monomorphize(operand_ty));
834+
if operand_layout.size != cast_layout.size
835+
|| operand_layout.abi.is_uninhabited()
836+
|| cast_layout.abi.is_uninhabited()
837+
{
838+
// Send UB cases to the full form so the operand version can
839+
// `bitcast` without worrying about malformed IR.
840+
return false;
841+
}
842+
843+
match (self.value_kind(operand_layout), self.value_kind(cast_layout)) {
844+
// Can always load from a pointer as needed
845+
(OperandValueKind::Ref, _) => true,
846+
847+
// Need to generate an `alloc` to get a pointer from an immediate
848+
(OperandValueKind::Immediate(..) | OperandValueKind::Pair(..), OperandValueKind::Ref) => false,
849+
850+
// When we have scalar immediates, we can convert them as needed
851+
(OperandValueKind::Immediate(..), OperandValueKind::Immediate(..)) |
852+
(OperandValueKind::Pair(..), OperandValueKind::Pair(..)) => true,
853+
854+
// Send mixings between scalars and pairs through the memory route
855+
// FIXME: Maybe this could use insertvalue/extractvalue instead?
856+
(OperandValueKind::Immediate(..), OperandValueKind::Pair(..)) |
857+
(OperandValueKind::Pair(..), OperandValueKind::Immediate(..)) => false,
858+
}
859+
}
746860
mir::Rvalue::Ref(..) |
747861
mir::Rvalue::CopyForDeref(..) |
748862
mir::Rvalue::AddressOf(..) |
@@ -767,4 +881,30 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
767881

768882
// (*) this is only true if the type is suitable
769883
}
884+
885+
/// Gets which variant of [`OperandValue`] is expected for a particular type.
886+
fn value_kind(&self, layout: TyAndLayout<'tcx>) -> OperandValueKind {
887+
if self.cx.is_backend_immediate(layout) {
888+
debug_assert!(!self.cx.is_backend_scalar_pair(layout));
889+
OperandValueKind::Immediate(match layout.abi {
890+
abi::Abi::Scalar(s) => s,
891+
abi::Abi::Vector { element, .. } => element,
892+
x => bug!("Couldn't translate {x:?} as backend immediate"),
893+
})
894+
} else if self.cx.is_backend_scalar_pair(layout) {
895+
let abi::Abi::ScalarPair(s1, s2) = layout.abi else {
896+
bug!("Couldn't translate {:?} as backend scalar pair", layout.abi)
897+
};
898+
OperandValueKind::Pair(s1, s2)
899+
} else {
900+
OperandValueKind::Ref
901+
}
902+
}
903+
}
904+
905+
#[derive(Debug, Copy, Clone)]
906+
enum OperandValueKind {
907+
Ref,
908+
Immediate(abi::Scalar),
909+
Pair(abi::Scalar, abi::Scalar),
770910
}

compiler/rustc_codegen_ssa/src/traits/type_.rs

+11
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,22 @@ pub trait DerivedTypeMethods<'tcx>: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {
100100
impl<'tcx, T> DerivedTypeMethods<'tcx> for T where Self: BaseTypeMethods<'tcx> + MiscMethods<'tcx> {}
101101

102102
pub trait LayoutTypeMethods<'tcx>: Backend<'tcx> {
103+
/// The backend type used for a rust type when it's in memory,
104+
/// such as when it's stack-allocated or when it's being loaded or stored.
103105
fn backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type;
104106
fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type;
105107
fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type;
106108
fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type;
107109
fn reg_backend_type(&self, ty: &Reg) -> Self::Type;
110+
/// The backend type used for a rust type when it's in an SSA register.
111+
///
112+
/// For nearly all types this is the same as the [`Self::backend_type`], however
113+
/// `bool` (and other `0`-or-`1` values) are kept as [`BaseTypeMethods::type_i1`]
114+
/// in registers but as [`BaseTypeMethods::type_i8`] in memory.
115+
///
116+
/// Converting values between the two different backend types is done using
117+
/// [`from_immediate`](super::BuilderMethods::from_immediate) and
118+
/// [`to_immediate_scalar`](super::BuilderMethods::to_immediate_scalar).
108119
fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> Self::Type;
109120
fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool;
110121
fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool;

0 commit comments

Comments
 (0)