Skip to content

make std::intrinsics functions actually be intrinsics #139916

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+337 −353
Open

make std::intrinsics functions actually be intrinsics #139916

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
318 changes: 25 additions & 293 deletions library/core/src/intrinsics/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,7 @@
#![allow(missing_docs)]

use crate::marker::{DiscriminantKind, Tuple};
use crate::mem::SizedTypeProperties;
use crate::{ptr, ub_checks};
use crate::ptr;

pub mod fallback;
pub mod mir;
Expand All @@ -74,6 +73,7 @@ pub mod simd;
#[cfg(all(target_has_atomic = "8", target_has_atomic = "32", target_has_atomic = "ptr"))]
use crate::sync::atomic::{self, AtomicBool, AtomicI32, AtomicIsize, AtomicU32, Ordering};

/// This is an accidentally-stable alias to [`ptr::drop_in_place`]; use that instead.
#[stable(feature = "drop_in_place", since = "1.8.0")]
#[rustc_allowed_through_unstable_modules = "import this function via `std::ptr` instead"]
#[deprecated(note = "no longer an intrinsic - use `ptr::drop_in_place` directly", since = "1.52.0")]
Expand Down Expand Up @@ -3336,7 +3336,7 @@ pub const unsafe fn typed_swap_nonoverlapping<T>(x: *mut T, y: *mut T) {
/// `#[inline]`), gating assertions on `ub_checks()` rather than `cfg!(ub_checks)` means that
/// assertions are enabled whenever the *user crate* has UB checks enabled. However, if the
/// user has UB checks disabled, the checks will still get optimized out. This intrinsic is
/// primarily used by [`ub_checks::assert_unsafe_precondition`].
/// primarily used by [`crate::ub_checks::assert_unsafe_precondition`].
#[rustc_intrinsic_const_stable_indirect] // just for UB checks
#[inline(always)]
#[rustc_intrinsic]
Expand Down Expand Up @@ -3625,306 +3625,38 @@ impl<P: ?Sized, T: ptr::Thin> AggregateRawPtr<*mut T> for *mut P {
#[rustc_intrinsic]
pub const fn ptr_metadata<P: ptr::Pointee<Metadata = M> + ?Sized, M>(ptr: *const P) -> M;

// Some functions are defined here because they accidentally got made
// available in this module on stable. See <https://github.com/rust-lang/rust/issues/15702>.
// (`transmute` also falls into this category, but it cannot be wrapped due to the
// check that `T` and `U` have the same size.)

/// Copies `count * size_of::<T>()` bytes from `src` to `dst`. The source
/// and destination must *not* overlap.
///
/// For regions of memory which might overlap, use [`copy`] instead.
///
/// `copy_nonoverlapping` is semantically equivalent to C's [`memcpy`], but
/// with the source and destination arguments swapped,
/// and `count` counting the number of `T`s instead of bytes.
///
/// The copy is "untyped" in the sense that data may be uninitialized or otherwise violate the
/// requirements of `T`. The initialization state is preserved exactly.
///
/// [`memcpy`]: https://en.cppreference.com/w/c/string/byte/memcpy
///
/// # Safety
///
/// Behavior is undefined if any of the following conditions are violated:
///
/// * `src` must be [valid] for reads of `count * size_of::<T>()` bytes.
///
/// * `dst` must be [valid] for writes of `count * size_of::<T>()` bytes.
///
/// * Both `src` and `dst` must be properly aligned.
///
/// * The region of memory beginning at `src` with a size of `count *
/// size_of::<T>()` bytes must *not* overlap with the region of memory
/// beginning at `dst` with the same size.
///
/// Like [`read`], `copy_nonoverlapping` creates a bitwise copy of `T`, regardless of
/// whether `T` is [`Copy`]. If `T` is not [`Copy`], using *both* the values
/// in the region beginning at `*src` and the region beginning at `*dst` can
/// [violate memory safety][read-ownership].
///
/// Note that even if the effectively copied size (`count * size_of::<T>()`) is
/// `0`, the pointers must be properly aligned.
///
/// [`read`]: crate::ptr::read
/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value
/// [valid]: crate::ptr#safety
///
/// # Examples
///
/// Manually implement [`Vec::append`]:
///
/// ```
/// use std::ptr;
///
/// /// Moves all the elements of `src` into `dst`, leaving `src` empty.
/// fn append<T>(dst: &mut Vec<T>, src: &mut Vec<T>) {
/// let src_len = src.len();
/// let dst_len = dst.len();
///
/// // Ensure that `dst` has enough capacity to hold all of `src`.
/// dst.reserve(src_len);
///
/// unsafe {
/// // The call to add is always safe because `Vec` will never
/// // allocate more than `isize::MAX` bytes.
/// let dst_ptr = dst.as_mut_ptr().add(dst_len);
/// let src_ptr = src.as_ptr();
///
/// // Truncate `src` without dropping its contents. We do this first,
/// // to avoid problems in case something further down panics.
/// src.set_len(0);
///
/// // The two regions cannot overlap because mutable references do
/// // not alias, and two different vectors cannot own the same
/// // memory.
/// ptr::copy_nonoverlapping(src_ptr, dst_ptr, src_len);
///
/// // Notify `dst` that it now holds the contents of `src`.
/// dst.set_len(dst_len + src_len);
/// }
/// }
///
/// let mut a = vec!['r'];
/// let mut b = vec!['u', 's', 't'];
///
/// append(&mut a, &mut b);
///
/// assert_eq!(a, &['r', 'u', 's', 't']);
/// assert!(b.is_empty());
/// ```
///
/// [`Vec::append`]: ../../std/vec/struct.Vec.html#method.append
#[doc(alias = "memcpy")]
/// This is an accidentally-stable alias to [`ptr::copy_nonoverlapping`]; use that instead.
// Note (intentionally not in the doc comment): `ptr::copy_nonoverlapping` adds some extra
// debug assertions; if you are writing compiler tests or code inside the standard library
// that wants to avoid those debug assertions, directly call this intrinsic instead.
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_allowed_through_unstable_modules = "import this function via `std::ptr` instead"]
#[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.83.0")]
#[inline(always)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
#[rustc_diagnostic_item = "ptr_copy_nonoverlapping"]
pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize) {
#[rustc_intrinsic_const_stable_indirect]
#[rustc_nounwind]
#[rustc_intrinsic]
const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize);

ub_checks::assert_unsafe_precondition!(
check_language_ub,
"ptr::copy_nonoverlapping requires that both pointer arguments are aligned and non-null \
and the specified memory ranges do not overlap",
(
src: *const () = src as *const (),
dst: *mut () = dst as *mut (),
size: usize = size_of::<T>(),
align: usize = align_of::<T>(),
count: usize = count,
) => {
let zero_size = count == 0 || size == 0;
ub_checks::maybe_is_aligned_and_not_null(src, align, zero_size)
&& ub_checks::maybe_is_aligned_and_not_null(dst, align, zero_size)
&& ub_checks::maybe_is_nonoverlapping(src, dst, size, count)
}
);

// SAFETY: the safety contract for `copy_nonoverlapping` must be
// upheld by the caller.
unsafe { copy_nonoverlapping(src, dst, count) }
}
#[rustc_nounwind]
#[rustc_intrinsic]
pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: usize);

/// Copies `count * size_of::<T>()` bytes from `src` to `dst`. The source
/// and destination may overlap.
///
/// If the source and destination will *never* overlap,
/// [`copy_nonoverlapping`] can be used instead.
///
/// `copy` is semantically equivalent to C's [`memmove`], but
/// with the source and destination arguments swapped,
/// and `count` counting the number of `T`s instead of bytes.
/// Copying takes place as if the bytes were copied from `src`
/// to a temporary array and then copied from the array to `dst`.
///
/// The copy is "untyped" in the sense that data may be uninitialized or otherwise violate the
/// requirements of `T`. The initialization state is preserved exactly.
///
/// [`memmove`]: https://en.cppreference.com/w/c/string/byte/memmove
///
/// # Safety
///
/// Behavior is undefined if any of the following conditions are violated:
///
/// * `src` must be [valid] for reads of `count * size_of::<T>()` bytes.
///
/// * `dst` must be [valid] for writes of `count * size_of::<T>()` bytes, and must remain valid even
/// when `src` is read for `count * size_of::<T>()` bytes. (This means if the memory ranges
/// overlap, the `dst` pointer must not be invalidated by `src` reads.)
///
/// * Both `src` and `dst` must be properly aligned.
///
/// Like [`read`], `copy` creates a bitwise copy of `T`, regardless of
/// whether `T` is [`Copy`]. If `T` is not [`Copy`], using both the values
/// in the region beginning at `*src` and the region beginning at `*dst` can
/// [violate memory safety][read-ownership].
///
/// Note that even if the effectively copied size (`count * size_of::<T>()`) is
/// `0`, the pointers must be properly aligned.
///
/// [`read`]: crate::ptr::read
/// [read-ownership]: crate::ptr::read#ownership-of-the-returned-value
/// [valid]: crate::ptr#safety
///
/// # Examples
///
/// Efficiently create a Rust vector from an unsafe buffer:
///
/// ```
/// use std::ptr;
///
/// /// # Safety
/// ///
/// /// * `ptr` must be correctly aligned for its type and non-zero.
/// /// * `ptr` must be valid for reads of `elts` contiguous elements of type `T`.
/// /// * Those elements must not be used after calling this function unless `T: Copy`.
/// # #[allow(dead_code)]
/// unsafe fn from_buf_raw<T>(ptr: *const T, elts: usize) -> Vec<T> {
/// let mut dst = Vec::with_capacity(elts);
///
/// // SAFETY: Our precondition ensures the source is aligned and valid,
/// // and `Vec::with_capacity` ensures that we have usable space to write them.
/// unsafe { ptr::copy(ptr, dst.as_mut_ptr(), elts); }
///
/// // SAFETY: We created it with this much capacity earlier,
/// // and the previous `copy` has initialized these elements.
/// unsafe { dst.set_len(elts); }
/// dst
/// }
/// ```
#[doc(alias = "memmove")]
/// This is an accidentally-stable alias to [`ptr::copy`]; use that instead.
// Note (intentionally not in the doc comment): `ptr::copy` adds some extra
// debug assertions; if you are writing compiler tests or code inside the standard library
// that wants to avoid those debug assertions, directly call this intrinsic instead.
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_allowed_through_unstable_modules = "import this function via `std::ptr` instead"]
#[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.83.0")]
#[inline(always)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
#[rustc_diagnostic_item = "ptr_copy"]
pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) {
#[rustc_intrinsic_const_stable_indirect]
#[rustc_nounwind]
#[rustc_intrinsic]
const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize);

// SAFETY: the safety contract for `copy` must be upheld by the caller.
unsafe {
ub_checks::assert_unsafe_precondition!(
check_language_ub,
"ptr::copy requires that both pointer arguments are aligned and non-null",
(
src: *const () = src as *const (),
dst: *mut () = dst as *mut (),
align: usize = align_of::<T>(),
zero_size: bool = T::IS_ZST || count == 0,
) =>
ub_checks::maybe_is_aligned_and_not_null(src, align, zero_size)
&& ub_checks::maybe_is_aligned_and_not_null(dst, align, zero_size)
);
copy(src, dst, count)
}
}
#[rustc_nounwind]
#[rustc_intrinsic]
pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize);

/// Sets `count * size_of::<T>()` bytes of memory starting at `dst` to
/// `val`.
///
/// `write_bytes` is similar to C's [`memset`], but sets `count *
/// size_of::<T>()` bytes to `val`.
///
/// [`memset`]: https://en.cppreference.com/w/c/string/byte/memset
///
/// # Safety
///
/// Behavior is undefined if any of the following conditions are violated:
///
/// * `dst` must be [valid] for writes of `count * size_of::<T>()` bytes.
///
/// * `dst` must be properly aligned.
///
/// Note that even if the effectively copied size (`count * size_of::<T>()`) is
/// `0`, the pointer must be properly aligned.
///
/// Additionally, note that changing `*dst` in this way can easily lead to undefined behavior (UB)
/// later if the written bytes are not a valid representation of some `T`. For instance, the
/// following is an **incorrect** use of this function:
///
/// ```rust,no_run
/// unsafe {
/// let mut value: u8 = 0;
/// let ptr: *mut bool = &mut value as *mut u8 as *mut bool;
/// let _bool = ptr.read(); // This is fine, `ptr` points to a valid `bool`.
/// ptr.write_bytes(42u8, 1); // This function itself does not cause UB...
/// let _bool = ptr.read(); // ...but it makes this operation UB! ⚠️
/// }
/// ```
///
/// [valid]: crate::ptr#safety
///
/// # Examples
///
/// Basic usage:
///
/// ```
/// use std::ptr;
///
/// let mut vec = vec![0u32; 4];
/// unsafe {
/// let vec_ptr = vec.as_mut_ptr();
/// ptr::write_bytes(vec_ptr, 0xfe, 2);
/// }
/// assert_eq!(vec, [0xfefefefe, 0xfefefefe, 0, 0]);
/// ```
#[doc(alias = "memset")]
/// This is an accidentally-stable alias to [`ptr::write_bytes`]; use that instead.
// Note (intentionally not in the doc comment): `ptr::write_bytes` adds some extra
// debug assertions; if you are writing compiler tests or code inside the standard library
// that wants to avoid those debug assertions, directly call this intrinsic instead.
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_allowed_through_unstable_modules = "import this function via `std::ptr` instead"]
#[rustc_const_stable(feature = "const_ptr_write", since = "1.83.0")]
#[inline(always)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
#[rustc_diagnostic_item = "ptr_write_bytes"]
pub const unsafe fn write_bytes<T>(dst: *mut T, val: u8, count: usize) {
#[rustc_intrinsic_const_stable_indirect]
#[rustc_nounwind]
#[rustc_intrinsic]
const unsafe fn write_bytes<T>(dst: *mut T, val: u8, count: usize);

// SAFETY: the safety contract for `write_bytes` must be upheld by the caller.
unsafe {
ub_checks::assert_unsafe_precondition!(
check_language_ub,
"ptr::write_bytes requires that the destination pointer is aligned and non-null",
(
addr: *const () = dst as *const (),
align: usize = align_of::<T>(),
zero_size: bool = T::IS_ZST || count == 0,
) => ub_checks::maybe_is_aligned_and_not_null(addr, align, zero_size)
);
write_bytes(dst, val, count)
}
}
#[rustc_const_stable(feature = "const_intrinsic_copy", since = "1.83.0")]
#[rustc_nounwind]
#[rustc_intrinsic]
pub const unsafe fn write_bytes<T>(dst: *mut T, val: u8, count: usize);

/// Returns the minimum of two `f16` values.
///
Expand Down
2 changes: 2 additions & 0 deletions library/core/src/mem/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ mod transmutability;
#[unstable(feature = "transmutability", issue = "99571")]
pub use transmutability::{Assume, TransmuteFrom};

// This one has to be a re-export (rather than wrapping the underlying intrinsic) so that we can do
// the special magic "types have equal size" check at the call site.
#[stable(feature = "rust1", since = "1.0.0")]
#[doc(inline)]
pub use crate::intrinsics::transmute;
Expand Down
Loading
Loading