Skip to content

Commit 659779d

Browse files
SpriteOvOSpriteOvO
committed
Introduce adjust_for_rust_abi in rustc_target 
1 parent 8bf64f1 commit 659779d

File tree

3 files changed

+158
-132
lines changed

3 files changed

+158
-132
lines changed

‎compiler/rustc_target/src/callconv/mod.rs

+114-1
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
1-
use std::fmt;
21
use std::str::FromStr;
2+
use std::{fmt, iter};
33

4+
use rustc_abi::AddressSpace;
5+
use rustc_abi::Primitive::Pointer;
46
pub use rustc_abi::{Reg, RegKind};
57
use rustc_macros::HashStable_Generic;
68
use rustc_span::Symbol;
79

810
use crate::abi::{self, Abi, Align, HasDataLayout, Size, TyAbiInterface, TyAndLayout};
11+
use crate::spec::abi::Abi as SpecAbi;
912
use crate::spec::{self, HasTargetSpec, HasWasmCAbiOpt, HasX86AbiOpt, WasmCAbi};
1013

1114
mod aarch64;
@@ -720,6 +723,116 @@ impl<'a, Ty> FnAbi<'a, Ty> {
720723

721724
Ok(())
722725
}
726+
727+
pub fn adjust_for_rust_abi<C>(&mut self, cx: &C, abi: SpecAbi)
728+
where
729+
Ty: TyAbiInterface<'a, C> + Copy,
730+
C: HasDataLayout + HasTargetSpec,
731+
{
732+
let spec = cx.target_spec();
733+
match &spec.arch[..] {
734+
"x86" => x86::compute_rust_abi_info(cx, self, abi),
735+
_ => {}
736+
};
737+
738+
for (arg_idx, arg) in self
739+
.args
740+
.iter_mut()
741+
.enumerate()
742+
.map(|(idx, arg)| (Some(idx), arg))
743+
.chain(iter::once((None, &mut self.ret)))
744+
{
745+
if arg.is_ignore() {
746+
continue;
747+
}
748+
749+
if arg_idx.is_none() && arg.layout.size > Pointer(AddressSpace::DATA).size(cx) * 2 {
750+
// Return values larger than 2 registers using a return area
751+
// pointer. LLVM and Cranelift disagree about how to return
752+
// values that don't fit in the registers designated for return
753+
// values. LLVM will force the entire return value to be passed
754+
// by return area pointer, while Cranelift will look at each IR level
755+
// return value independently and decide to pass it in a
756+
// register or not, which would result in the return value
757+
// being passed partially in registers and partially through a
758+
// return area pointer.
759+
//
760+
// While Cranelift may need to be fixed as the LLVM behavior is
761+
// generally more correct with respect to the surface language,
762+
// forcing this behavior in rustc itself makes it easier for
763+
// other backends to conform to the Rust ABI and for the C ABI
764+
// rustc already handles this behavior anyway.
765+
//
766+
// In addition LLVM's decision to pass the return value in
767+
// registers or using a return area pointer depends on how
768+
// exactly the return type is lowered to an LLVM IR type. For
769+
// example `Option<u128>` can be lowered as `{ i128, i128 }`
770+
// in which case the x86_64 backend would use a return area
771+
// pointer, or it could be passed as `{ i32, i128 }` in which
772+
// case the x86_64 backend would pass it in registers by taking
773+
// advantage of an LLVM ABI extension that allows using 3
774+
// registers for the x86_64 sysv call conv rather than the
775+
// officially specified 2 registers.
776+
//
777+
// FIXME: Technically we should look at the amount of available
778+
// return registers rather than guessing that there are 2
779+
// registers for return values. In practice only a couple of
780+
// architectures have less than 2 return registers. None of
781+
// which supported by Cranelift.
782+
//
783+
// NOTE: This adjustment is only necessary for the Rust ABI as
784+
// for other ABI's the calling convention implementations in
785+
// rustc_target already ensure any return value which doesn't
786+
// fit in the available amount of return registers is passed in
787+
// the right way for the current target.
788+
arg.make_indirect();
789+
continue;
790+
}
791+
792+
match arg.layout.abi {
793+
Abi::Aggregate { .. } => {}
794+
795+
// This is a fun case! The gist of what this is doing is
796+
// that we want callers and callees to always agree on the
797+
// ABI of how they pass SIMD arguments. If we were to *not*
798+
// make these arguments indirect then they'd be immediates
799+
// in LLVM, which means that they'd used whatever the
800+
// appropriate ABI is for the callee and the caller. That
801+
// means, for example, if the caller doesn't have AVX
802+
// enabled but the callee does, then passing an AVX argument
803+
// across this boundary would cause corrupt data to show up.
804+
//
805+
// This problem is fixed by unconditionally passing SIMD
806+
// arguments through memory between callers and callees
807+
// which should get them all to agree on ABI regardless of
808+
// target feature sets. Some more information about this
809+
// issue can be found in #44367.
810+
//
811+
// Note that the intrinsic ABI is exempt here as
812+
// that's how we connect up to LLVM and it's unstable
813+
// anyway, we control all calls to it in libstd.
814+
Abi::Vector { .. } if abi != SpecAbi::RustIntrinsic && spec.simd_types_indirect => {
815+
arg.make_indirect();
816+
continue;
817+
}
818+
819+
_ => continue,
820+
}
821+
// Compute `Aggregate` ABI.
822+
823+
let is_indirect_not_on_stack =
824+
matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
825+
assert!(is_indirect_not_on_stack);
826+
827+
let size = arg.layout.size;
828+
if !arg.layout.is_unsized() && size <= Pointer(AddressSpace::DATA).size(cx) {
829+
// We want to pass small aggregates as immediates, but using
830+
// an LLVM aggregate type for this leads to bad optimizations,
831+
// so we pick an appropriately sized integer type instead.
832+
arg.cast_to(Reg { kind: RegKind::Integer, size });
833+
}
834+
}
835+
}
723836
}
724837

725838
impl FromStr for Conv {

‎compiler/rustc_target/src/callconv/x86.rs

+36-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
use crate::abi::call::{ArgAttribute, FnAbi, PassMode, Reg, RegKind};
2-
use crate::abi::{Abi, Align, HasDataLayout, TyAbiInterface, TyAndLayout};
2+
use crate::abi::{
3+
Abi, AddressSpace, Align, Float, HasDataLayout, Pointer, TyAbiInterface, TyAndLayout,
4+
};
35
use crate::spec::HasTargetSpec;
6+
use crate::spec::abi::Abi as SpecAbi;
47

58
#[derive(PartialEq)]
69
pub(crate) enum Flavor {
@@ -207,3 +210,35 @@ pub(crate) fn fill_inregs<'a, Ty, C>(
207210
}
208211
}
209212
}
213+
214+
pub(crate) fn compute_rust_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, abi: SpecAbi)
215+
where
216+
Ty: TyAbiInterface<'a, C> + Copy,
217+
C: HasDataLayout + HasTargetSpec,
218+
{
219+
// Avoid returning floats in x87 registers on x86 as loading and storing from x87
220+
// registers will quiet signalling NaNs. Also avoid using SSE registers since they
221+
// are not always available (depending on target features).
222+
if !fn_abi.ret.is_ignore()
223+
// Intrinsics themselves are not actual "real" functions, so theres no need to change their ABIs.
224+
&& abi != SpecAbi::RustIntrinsic
225+
{
226+
let has_float = match fn_abi.ret.layout.abi {
227+
Abi::Scalar(s) => matches!(s.primitive(), Float(_)),
228+
Abi::ScalarPair(s1, s2) => {
229+
matches!(s1.primitive(), Float(_)) || matches!(s2.primitive(), Float(_))
230+
}
231+
_ => false, // anyway not passed via registers on x86
232+
};
233+
if has_float {
234+
if fn_abi.ret.layout.size <= Pointer(AddressSpace::DATA).size(cx) {
235+
// Same size or smaller than pointer, return in a register.
236+
fn_abi.ret.cast_to(Reg { kind: RegKind::Integer, size: fn_abi.ret.layout.size });
237+
} else {
238+
// Larger than a pointer, return indirectly.
239+
fn_abi.ret.make_indirect();
240+
}
241+
return;
242+
}
243+
}
244+
}

‎compiler/rustc_ty_utils/src/abi.rs

+8-130
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use std::iter;
22

3-
use rustc_abi::Primitive::{Float, Pointer};
4-
use rustc_abi::{Abi, AddressSpace, PointerKind, Scalar, Size};
3+
use rustc_abi::Primitive::Pointer;
4+
use rustc_abi::{Abi, PointerKind, Scalar, Size};
55
use rustc_hir as hir;
66
use rustc_hir::lang_items::LangItem;
77
use rustc_middle::bug;
@@ -13,8 +13,7 @@ use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt};
1313
use rustc_session::config::OptLevel;
1414
use rustc_span::def_id::DefId;
1515
use rustc_target::abi::call::{
16-
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, Reg, RegKind,
17-
RiscvInterruptKind,
16+
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, RiscvInterruptKind,
1817
};
1918
use rustc_target::spec::abi::Abi as SpecAbi;
2019
use tracing::debug;
@@ -678,6 +677,8 @@ fn fn_abi_adjust_for_abi<'tcx>(
678677
let tcx = cx.tcx();
679678

680679
if abi == SpecAbi::Rust || abi == SpecAbi::RustCall || abi == SpecAbi::RustIntrinsic {
680+
fn_abi.adjust_for_rust_abi(cx, abi);
681+
681682
// Look up the deduced parameter attributes for this function, if we have its def ID and
682683
// we're optimizing in non-incremental mode. We'll tag its parameters with those attributes
683684
// as appropriate.
@@ -688,135 +689,17 @@ fn fn_abi_adjust_for_abi<'tcx>(
688689
&[]
689690
};
690691

691-
let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>, arg_idx: Option<usize>| {
692+
for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() {
692693
if arg.is_ignore() {
693-
return;
694-
}
695-
696-
// Avoid returning floats in x87 registers on x86 as loading and storing from x87
697-
// registers will quiet signalling NaNs. Also avoid using SSE registers since they
698-
// are not always available (depending on target features).
699-
if tcx.sess.target.arch == "x86"
700-
&& arg_idx.is_none()
701-
// Intrinsics themselves are not actual "real" functions, so theres no need to
702-
// change their ABIs.
703-
&& abi != SpecAbi::RustIntrinsic
704-
{
705-
let has_float = match arg.layout.abi {
706-
Abi::Scalar(s) => matches!(s.primitive(), Float(_)),
707-
Abi::ScalarPair(s1, s2) => {
708-
matches!(s1.primitive(), Float(_)) || matches!(s2.primitive(), Float(_))
709-
}
710-
_ => false, // anyway not passed via registers on x86
711-
};
712-
if has_float {
713-
if arg.layout.size <= Pointer(AddressSpace::DATA).size(cx) {
714-
// Same size or smaller than pointer, return in a register.
715-
arg.cast_to(Reg { kind: RegKind::Integer, size: arg.layout.size });
716-
} else {
717-
// Larger than a pointer, return indirectly.
718-
arg.make_indirect();
719-
}
720-
return;
721-
}
722-
}
723-
724-
if arg_idx.is_none() && arg.layout.size > Pointer(AddressSpace::DATA).size(cx) * 2 {
725-
// Return values larger than 2 registers using a return area
726-
// pointer. LLVM and Cranelift disagree about how to return
727-
// values that don't fit in the registers designated for return
728-
// values. LLVM will force the entire return value to be passed
729-
// by return area pointer, while Cranelift will look at each IR level
730-
// return value independently and decide to pass it in a
731-
// register or not, which would result in the return value
732-
// being passed partially in registers and partially through a
733-
// return area pointer.
734-
//
735-
// While Cranelift may need to be fixed as the LLVM behavior is
736-
// generally more correct with respect to the surface language,
737-
// forcing this behavior in rustc itself makes it easier for
738-
// other backends to conform to the Rust ABI and for the C ABI
739-
// rustc already handles this behavior anyway.
740-
//
741-
// In addition LLVM's decision to pass the return value in
742-
// registers or using a return area pointer depends on how
743-
// exactly the return type is lowered to an LLVM IR type. For
744-
// example `Option<u128>` can be lowered as `{ i128, i128 }`
745-
// in which case the x86_64 backend would use a return area
746-
// pointer, or it could be passed as `{ i32, i128 }` in which
747-
// case the x86_64 backend would pass it in registers by taking
748-
// advantage of an LLVM ABI extension that allows using 3
749-
// registers for the x86_64 sysv call conv rather than the
750-
// officially specified 2 registers.
751-
//
752-
// FIXME: Technically we should look at the amount of available
753-
// return registers rather than guessing that there are 2
754-
// registers for return values. In practice only a couple of
755-
// architectures have less than 2 return registers. None of
756-
// which supported by Cranelift.
757-
//
758-
// NOTE: This adjustment is only necessary for the Rust ABI as
759-
// for other ABI's the calling convention implementations in
760-
// rustc_target already ensure any return value which doesn't
761-
// fit in the available amount of return registers is passed in
762-
// the right way for the current target.
763-
arg.make_indirect();
764-
return;
765-
}
766-
767-
match arg.layout.abi {
768-
Abi::Aggregate { .. } => {}
769-
770-
// This is a fun case! The gist of what this is doing is
771-
// that we want callers and callees to always agree on the
772-
// ABI of how they pass SIMD arguments. If we were to *not*
773-
// make these arguments indirect then they'd be immediates
774-
// in LLVM, which means that they'd used whatever the
775-
// appropriate ABI is for the callee and the caller. That
776-
// means, for example, if the caller doesn't have AVX
777-
// enabled but the callee does, then passing an AVX argument
778-
// across this boundary would cause corrupt data to show up.
779-
//
780-
// This problem is fixed by unconditionally passing SIMD
781-
// arguments through memory between callers and callees
782-
// which should get them all to agree on ABI regardless of
783-
// target feature sets. Some more information about this
784-
// issue can be found in #44367.
785-
//
786-
// Note that the intrinsic ABI is exempt here as
787-
// that's how we connect up to LLVM and it's unstable
788-
// anyway, we control all calls to it in libstd.
789-
Abi::Vector { .. }
790-
if abi != SpecAbi::RustIntrinsic && tcx.sess.target.simd_types_indirect =>
791-
{
792-
arg.make_indirect();
793-
return;
794-
}
795-
796-
_ => return,
797-
}
798-
// Compute `Aggregate` ABI.
799-
800-
let is_indirect_not_on_stack =
801-
matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
802-
assert!(is_indirect_not_on_stack, "{:?}", arg);
803-
804-
let size = arg.layout.size;
805-
if !arg.layout.is_unsized() && size <= Pointer(AddressSpace::DATA).size(cx) {
806-
// We want to pass small aggregates as immediates, but using
807-
// an LLVM aggregate type for this leads to bad optimizations,
808-
// so we pick an appropriately sized integer type instead.
809-
arg.cast_to(Reg { kind: RegKind::Integer, size });
694+
continue;
810695
}
811696

812697
// If we deduced that this parameter was read-only, add that to the attribute list now.
813698
//
814699
// The `readonly` parameter only applies to pointers, so we can only do this if the
815700
// argument was passed indirectly. (If the argument is passed directly, it's an SSA
816701
// value, so it's implicitly immutable.)
817-
if let (Some(arg_idx), &mut PassMode::Indirect { ref mut attrs, .. }) =
818-
(arg_idx, &mut arg.mode)
819-
{
702+
if let &mut PassMode::Indirect { ref mut attrs, .. } = &mut arg.mode {
820703
// The `deduced_param_attrs` list could be empty if this is a type of function
821704
// we can't deduce any parameters for, so make sure the argument index is in
822705
// bounds.
@@ -827,11 +710,6 @@ fn fn_abi_adjust_for_abi<'tcx>(
827710
}
828711
}
829712
}
830-
};
831-
832-
fixup(&mut fn_abi.ret, None);
833-
for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() {
834-
fixup(arg, Some(arg_idx));
835713
}
836714
} else {
837715
fn_abi

0 commit comments

Comments
 (0)