add const_allocate intrinsic

compiler/rustc_mir/src/const_eval/machine.rs

@@ -1,5 +1,6 @@
 use rustc_middle::mir;
 use rustc_middle::ty::layout::HasTyCtxt;
+use rustc_middle::ty::InstanceDef;
 use rustc_middle::ty::{self, Ty};
 use std::borrow::Borrow;
 use std::collections::hash_map::Entry;
@@ -37,6 +38,14 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
         if instance.def.requires_caller_location(self.tcx()) {
             return Ok(false);
         }
+        // Only memoize instrinsics. This was added in #79594 while adding the `const_allocate` intrinsic.
+        // We only memoize intrinsics because it would be unsound to memoize functions
+        // which might interact with the heap.
+        // Additionally, const_allocate intrinsic is impure and thus should not be memoized;
+        // it will not be memoized because it has non-ZST args
+        if !matches!(instance.def, InstanceDef::Intrinsic(_)) {
+            return Ok(false);
+        }
         // For the moment we only do this for functions which take no arguments
         // (or all arguments are ZSTs) so that we don't memoize too much.
         if args.iter().any(|a| !a.layout.is_zst()) {

compiler/rustc_mir/src/interpret/intern.rs

@@ -104,7 +104,10 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>>(
     // This match is just a canary for future changes to `MemoryKind`, which most likely need
     // changes in this function.
     match kind {
-        MemoryKind::Stack | MemoryKind::Vtable | MemoryKind::CallerLocation => {}
+        MemoryKind::Stack
+        | MemoryKind::ConstHeap
+        | MemoryKind::Vtable
+        | MemoryKind::CallerLocation => {}
     }
     // Set allocation mutability as appropriate. This is used by LLVM to put things into
     // read-only memory, and also by Miri when evaluating other globals that

compiler/rustc_mir/src/interpret/intrinsics.rs

@@ -14,10 +14,11 @@ use rustc_middle::ty;
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_span::symbol::{sym, Symbol};
-use rustc_target::abi::{Abi, LayoutOf as _, Primitive, Size};
+use rustc_target::abi::{Abi, Align, LayoutOf as _, Primitive, Size};
 
 use super::{
-    util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy,
+    util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, MemoryKind, OpTy,
+    PlaceTy,
 };
 
 mod caller_location;
@@ -337,6 +338,22 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let result = Scalar::from_uint(truncated_bits, layout.size);
                 self.write_scalar(result, dest)?;
             }
+            sym::const_allocate => {
+                let size = self.read_scalar(args[0])?.to_machine_usize(self)?;
+                let align = self.read_scalar(args[1])?.to_machine_usize(self)?;
+
+                let align = match Align::from_bytes(align) {
+                    Ok(a) => a,
+                    Err(err) => throw_ub_format!("align has to be a power of 2, {}", err),
+                };
+
+                let ptr = self.memory.allocate(
+                    Size::from_bytes(size as u64),
+                    align,
+                    MemoryKind::ConstHeap,
+                );
+                self.write_scalar(Scalar::Ptr(ptr), dest)?;
+            }
             sym::offset => {
                 let ptr = self.read_scalar(args[0])?.check_init()?;
                 let offset_count = self.read_scalar(args[1])?.to_machine_isize(self)?;

compiler/rustc_mir/src/interpret/memory.rs

@@ -27,6 +27,8 @@ use crate::util::pretty;
 pub enum MemoryKind<T> {
     /// Stack memory. Error if deallocated except during a stack pop.
     Stack,
+    /// Heap memory.
+    ConstHeap,
     /// Memory backing vtables. Error if ever deallocated.
     Vtable,
     /// Memory allocated by `caller_location` intrinsic. Error if ever deallocated.
@@ -40,6 +42,7 @@ impl<T: MayLeak> MayLeak for MemoryKind<T> {
     fn may_leak(self) -> bool {
         match self {
             MemoryKind::Stack => false,
+            MemoryKind::ConstHeap => false,
             MemoryKind::Vtable => true,
             MemoryKind::CallerLocation => true,
             MemoryKind::Machine(k) => k.may_leak(),
@@ -51,6 +54,7 @@ impl<T: fmt::Display> fmt::Display for MemoryKind<T> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             MemoryKind::Stack => write!(f, "stack variable"),
+            MemoryKind::ConstHeap => write!(f, "heap allocation"),
             MemoryKind::Vtable => write!(f, "vtable"),
             MemoryKind::CallerLocation => write!(f, "caller location"),
             MemoryKind::Machine(m) => write!(f, "{}", m),

compiler/rustc_span/src/symbol.rs

@@ -356,6 +356,7 @@ symbols! {
         concat_idents,
         conservative_impl_trait,
         console,
+        const_allocate,
         const_compare_raw_pointers,
         const_constructor,
         const_eval_limit,

compiler/rustc_typeck/src/check/intrinsic.rs

@@ -286,6 +286,10 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                 (1, vec![tcx.mk_imm_ptr(param(0)), tcx.mk_imm_ptr(param(0))], tcx.types.bool)
             }
 
+            sym::const_allocate => {
+                (0, vec![tcx.types.usize, tcx.types.usize], tcx.mk_mut_ptr(tcx.types.u8))
+            }
+
             sym::ptr_offset_from => {
                 (1, vec![tcx.mk_imm_ptr(param(0)), tcx.mk_imm_ptr(param(0))], tcx.types.isize)
             }

library/core/src/intrinsics.rs

@@ -1732,6 +1732,11 @@ extern "rust-intrinsic" {
     /// See documentation of `<*const T>::guaranteed_ne` for details.
     #[rustc_const_unstable(feature = "const_raw_ptr_comparison", issue = "53020")]
     pub fn ptr_guaranteed_ne<T>(ptr: *const T, other: *const T) -> bool;
+
+    /// Allocate at compile time. Should not be called at runtime.
+    #[rustc_const_unstable(feature = "const_heap", issue = "79597")]
+    #[cfg(not(bootstrap))]
+    pub fn const_allocate(size: usize, align: usize) -> *mut u8;
 }
 
 // Some functions are defined here because they accidentally got made

library/core/src/lib.rs

@@ -68,6 +68,7 @@
 #![feature(arbitrary_self_types)]
 #![feature(asm)]
 #![feature(cfg_target_has_atomic)]
+#![cfg_attr(not(bootstrap), feature(const_heap))]
 #![feature(const_alloc_layout)]
 #![feature(const_discriminant)]
 #![feature(const_cell_into_inner)]

src/test/ui/const-generics/const-argument-if-length.full.stderr

@@ -11,12 +11,6 @@ LL |     if std::mem::size_of::<T>() == 0 {
 LL | pub const fn size_of<T>() -> usize {
    |                      - required by this bound in `std::mem::size_of`
 
-error[E0080]: evaluation of constant value failed
-  --> $DIR/const-argument-if-length.rs:19:15
-   |
-LL |     pad: [u8; is_zst::<T>()],
-   |               ^^^^^^^^^^^^^ referenced constant has errors
-
 error[E0277]: the size for values of type `T` cannot be known at compilation time
   --> $DIR/const-argument-if-length.rs:17:12
    |
@@ -36,7 +30,6 @@ help: the `Box` type always has a statically known size and allocates its conten
 LL |     value: Box<T>,
    |            ^^^^ ^
 
-error: aborting due to 3 previous errors
+error: aborting due to 2 previous errors
 
-Some errors have detailed explanations: E0080, E0277.
-For more information about an error, try `rustc --explain E0080`.
+For more information about this error, try `rustc --explain E0277`.

src/test/ui/const-generics/const-argument-if-length.rs

@@ -18,7 +18,6 @@ pub struct AtLeastByte<T: ?Sized> {
     //~^ ERROR the size for values of type `T` cannot be known at compilation time
     pad: [u8; is_zst::<T>()],
     //[min]~^ ERROR generic parameters may not be used in const operations
-    //[full]~^^ ERROR evaluation of constant value failed
 }
 
 fn main() {}

src/test/ui/consts/const-eval/erroneous-const.rs

@@ -9,11 +9,11 @@ impl<T> PrintName<T> {
 
 const fn no_codegen<T>() {
     if false {
-        let _ = PrintName::<T>::VOID; //~ERROR evaluation of constant value failed
+        let _ = PrintName::<T>::VOID; //~ERROR could not evaluate static initializer
     }
 }
 
-pub static FOO: () = no_codegen::<i32>(); //~ERROR could not evaluate static initializer
+pub static FOO: () = no_codegen::<i32>();
 
 fn main() {
     FOO

src/test/ui/consts/const-eval/erroneous-const.stderr

@@ -24,18 +24,18 @@ note: the lint level is defined here
 LL | #![warn(const_err, unconditional_panic)]
    |         ^^^^^^^^^
 
-error[E0080]: evaluation of constant value failed
+error[E0080]: could not evaluate static initializer
   --> $DIR/erroneous-const.rs:12:17
    |
 LL |         let _ = PrintName::<T>::VOID;
-   |                 ^^^^^^^^^^^^^^^^^^^^ referenced constant has errors
-
-error[E0080]: could not evaluate static initializer
-  --> $DIR/erroneous-const.rs:16:22
-   |
+   |                 ^^^^^^^^^^^^^^^^^^^^
+   |                 |
+   |                 referenced constant has errors
+   |                 inside `no_codegen::<i32>` at $DIR/erroneous-const.rs:12:17
+...
 LL | pub static FOO: () = no_codegen::<i32>();
-   |                      ^^^^^^^^^^^^^^^^^^^ referenced constant has errors
+   |                      ------------------- inside `FOO` at $DIR/erroneous-const.rs:16:22
 
-error: aborting due to 2 previous errors; 2 warnings emitted
+error: aborting due to previous error; 2 warnings emitted
 
 For more information about this error, try `rustc --explain E0080`.

src/test/ui/consts/const-eval/heap/alloc_intrinsic_errors.rs

@@ -0,0 +1,17 @@
+#![feature(core_intrinsics)]
+#![feature(const_heap)]
+#![feature(const_raw_ptr_deref)]
+#![feature(const_mut_refs)]
+use std::intrinsics;
+
+const FOO: i32 = foo();
+const fn foo() -> i32 {
+    unsafe {
+        let _ = intrinsics::const_allocate(4, 3) as * mut i32;
+        //~^ error: any use of this value will cause an error [const_err]
+    }
+    1
+
+}
+
+fn main() {}

src/test/ui/consts/const-eval/heap/alloc_intrinsic_errors.stderr

@@ -0,0 +1,17 @@
+error: any use of this value will cause an error
+  --> $DIR/alloc_intrinsic_errors.rs:10:17
+   |
+LL | const FOO: i32 = foo();
+   | -----------------------
+...
+LL |         let _ = intrinsics::const_allocate(4, 3) as * mut i32;
+   |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |                 |
+   |                 align has to be a power of 2, `3` is not a power of 2
+   |                 inside `foo` at $DIR/alloc_intrinsic_errors.rs:10:17
+   |                 inside `FOO` at $DIR/alloc_intrinsic_errors.rs:7:18
+   |
+   = note: `#[deny(const_err)]` on by default
+
+error: aborting due to previous error
+

src/test/ui/consts/const-eval/heap/alloc_intrinsic_nontransient.rs

@@ -0,0 +1,20 @@
+// run-pass
+#![feature(core_intrinsics)]
+#![feature(const_heap)]
+#![feature(const_raw_ptr_deref)]
+#![feature(const_mut_refs)]
+use std::intrinsics;
+
+const FOO: &i32 = foo();
+
+const fn foo() -> &'static i32 {
+    let t = unsafe {
+        let i = intrinsics::const_allocate(4, 4) as * mut i32;
+        *i = 20;
+        i
+    };
+    unsafe { &*t }
+}
+fn main() {
+    assert_eq!(*FOO, 20)
+}

src/test/ui/consts/const-eval/heap/alloc_intrinsic_nontransient_fail.rs

@@ -0,0 +1,19 @@
+#![feature(core_intrinsics)]
+#![feature(const_heap)]
+#![feature(const_raw_ptr_deref)]
+#![feature(const_mut_refs)]
+use std::intrinsics;
+
+const FOO: *const i32 = foo();
+//~^ ERROR untyped pointers are not allowed in constant
+
+const fn foo() -> &'static i32 {
+    let t = unsafe {
+        let i = intrinsics::const_allocate(4, 4) as * mut i32;
+        *i = 20;
+        i
+    };
+    unsafe { &*t }
+}
+fn main() {
+}

src/test/ui/consts/const-eval/heap/alloc_intrinsic_nontransient_fail.stderr

@@ -0,0 +1,8 @@
+error: untyped pointers are not allowed in constant
+  --> $DIR/alloc_intrinsic_nontransient_fail.rs:7:1
+   |
+LL | const FOO: *const i32 = foo();
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+

src/test/ui/consts/const-eval/heap/alloc_intrinsic_transient.rs

@@ -0,0 +1,20 @@
+// run-pass
+#![feature(core_intrinsics)]
+#![feature(const_heap)]
+#![feature(const_raw_ptr_deref)]
+#![feature(const_mut_refs)]
+use std::intrinsics;
+
+const FOO: i32 = foo();
+
+const fn foo() -> i32 {
+    let t = unsafe {
+        let i = intrinsics::const_allocate(4, 4) as * mut i32;
+        *i = 20;
+        i
+    };
+    unsafe { *t }
+}
+fn main() {
+    assert_eq!(FOO, 20);
+}

src/test/ui/consts/const-eval/heap/alloc_intrinsic_uninit.rs

@@ -0,0 +1,10 @@
+// compile-test
+#![feature(core_intrinsics)]
+#![feature(const_heap)]
+#![feature(const_raw_ptr_deref)]
+#![feature(const_mut_refs)]
+use std::intrinsics;
+
+const BAR: &i32 = unsafe { &*(intrinsics::const_allocate(4, 4) as *mut i32) };
+//~^ error: it is undefined behavior to use this value
+fn main() {}

src/test/ui/consts/const-eval/heap/alloc_intrinsic_uninit.stderr

@@ -0,0 +1,11 @@
+error[E0080]: it is undefined behavior to use this value
+  --> $DIR/alloc_intrinsic_uninit.rs:8:1
+   |
+LL | const BAR: &i32 = unsafe { &*(intrinsics::const_allocate(4, 4) as *mut i32) };
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered uninitialized bytes at .<deref>, but expected initialized plain (non-pointer) bytes
+   |
+   = note: The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior.
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0080`.

src/test/ui/consts/const-eval/heap/alloc_intrinsic_untyped.rs

@@ -0,0 +1,10 @@
+#![feature(core_intrinsics)]
+#![feature(const_heap)]
+#![feature(const_raw_ptr_deref)]
+#![feature(const_mut_refs)]
+use std::intrinsics;
+
+const BAR: *mut i32 = unsafe { intrinsics::const_allocate(4, 4) as *mut i32};
+//~^ error: untyped pointers are not allowed in constant
+
+fn main() {}

src/test/ui/consts/const-eval/heap/alloc_intrinsic_untyped.stderr

@@ -0,0 +1,8 @@
+error: untyped pointers are not allowed in constant
+  --> $DIR/alloc_intrinsic_untyped.rs:7:1
+   |
+LL | const BAR: *mut i32 = unsafe { intrinsics::const_allocate(4, 4) as *mut i32};
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+

src/test/ui/consts/const-eval/unwind-abort.rs

@@ -2,10 +2,10 @@
 
 #[unwind(aborts)]
 const fn foo() {
-    panic!() //~ evaluation of constant value failed
+    panic!() //~ ERROR any use of this value will cause an error [const_err]
 }
 
-const _: () = foo(); //~ any use of this value will cause an error
+const _: () = foo();
 // Ensure that the CTFE engine handles calls to `#[unwind(aborts)]` gracefully
 
 fn main() {