add extern "custom" functions

compiler/rustc_abi/src/extern_abi.rs

@@ -25,6 +25,10 @@ pub enum ExternAbi {
     Cdecl {
         unwind: bool,
     },
+    /// An ABI that rustc does not know how to call or define. Functions with this ABI can
+    /// only be created using `#[naked]` functions or `extern "custom"` blocks, and can only
+    /// be called from inline assembly.
+    Custom,
     Stdcall {
         unwind: bool,
     },
@@ -117,6 +121,7 @@ abi_impls! {
             AvrNonBlockingInterrupt =><= "avr-non-blocking-interrupt",
             Cdecl { unwind: false } =><= "cdecl",
             Cdecl { unwind: true } =><= "cdecl-unwind",
+            Custom =><= "custom",
             EfiApi =><= "efiapi",
             Fastcall { unwind: false } =><= "fastcall",
             Fastcall { unwind: true } =><= "fastcall-unwind",

compiler/rustc_ast/src/ast.rs

@@ -3479,6 +3479,37 @@ impl FnHeader {
             || matches!(constness, Const::Yes(_))
             || !matches!(ext, Extern::None)
     }
+
+    pub fn span(&self) -> Option<Span> {
+        fn append(a: &mut Option<Span>, b: Span) {
+            *a = match a {
+                None => Some(b),
+                Some(x) => Some(x.to(b)),
+            }
+        }
+
+        let mut full_span = None;
+
+        match self.safety {
+            Safety::Unsafe(span) | Safety::Safe(span) => append(&mut full_span, span),
+            Safety::Default => {}
+        };
+
+        if let Some(coroutine_kind) = self.coroutine_kind {
+            append(&mut full_span, coroutine_kind.span());
+        }
+
+        if let Const::Yes(span) = self.constness {
+            append(&mut full_span, span);
+        }
+
+        match self.ext {
+            Extern::Implicit(span) | Extern::Explicit(_, span) => append(&mut full_span, span),
+            Extern::None => {}
+        }
+
+        full_span
+    }
 }
 
 impl Default for FnHeader {

compiler/rustc_ast_lowering/src/stability.rs

@@ -134,5 +134,8 @@ pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {
             feature: sym::cmse_nonsecure_entry,
             explain: GateReason::Experimental,
         }),
+        ExternAbi::Custom => {
+            Err(UnstableAbi { abi, feature: sym::abi_custom, explain: GateReason::Experimental })
+        }
     }
 }

compiler/rustc_ast_passes/messages.ftl

@@ -1,3 +1,16 @@
+ast_passes_abi_custom_invalid_signature =
+    invalid signature for `extern "custom"` function
+    .note = functions with the `"custom"` ABI cannot have any parameters or return type
+    .suggestion = remove the parameters and return type
+
+ast_passes_abi_custom_safe_foreign_function =
+    foreign functions with the `"custom"` ABI cannot be safe
+    .suggestion = remove the `safe` keyword from this definition
+
+ast_passes_abi_custom_safe_function =
+    functions with the `"custom"` ABI must be unsafe
+    .suggestion = add the `unsafe` keyword to this definition
+
 ast_passes_assoc_const_without_body =
     associated constant in `impl` without body
     .suggestion = provide a definition for the constant

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -18,6 +18,7 @@
 
 use std::mem;
 use std::ops::{Deref, DerefMut};
+use std::str::FromStr;
 
 use itertools::{Either, Itertools};
 use rustc_abi::ExternAbi;
@@ -81,6 +82,7 @@ struct AstValidator<'a> {
 
     /// Used to ban explicit safety on foreign items when the extern block is not marked as unsafe.
     extern_mod_safety: Option<Safety>,
+    extern_mod_abi: Option<ExternAbi>,
 
     lint_node_id: NodeId,
 
@@ -121,10 +123,17 @@ impl<'a> AstValidator<'a> {
         self.outer_trait_or_trait_impl = old;
     }
 
-    fn with_in_extern_mod(&mut self, extern_mod_safety: Safety, f: impl FnOnce(&mut Self)) {
-        let old = mem::replace(&mut self.extern_mod_safety, Some(extern_mod_safety));
+    fn with_in_extern_mod(
+        &mut self,
+        extern_mod_safety: Safety,
+        abi: Option<ExternAbi>,
+        f: impl FnOnce(&mut Self),
+    ) {
+        let old_safety = mem::replace(&mut self.extern_mod_safety, Some(extern_mod_safety));
+        let old_abi = mem::replace(&mut self.extern_mod_abi, abi);
         f(self);
-        self.extern_mod_safety = old;
+        self.extern_mod_safety = old_safety;
+        self.extern_mod_abi = old_abi;
     }
 
     fn with_tilde_const(
@@ -370,6 +379,49 @@ impl<'a> AstValidator<'a> {
         }
     }
 
+    /// An `extern "custom"` function must be unsafe, and must not have any parameters or return
+    /// type.
+    fn check_custom_abi(&self, ctxt: FnCtxt, ident: &Ident, sig: &FnSig) {
+        let dcx = self.dcx();
+
+        // An `extern "custom"` function must be unsafe.
+        match sig.header.safety {
+            Safety::Unsafe(_) => { /* all good */ }
+            Safety::Safe(safe_span) => {
+                let safe_span = safe_span.with_hi(safe_span.hi() + rustc_span::BytePos(1));
+                dcx.emit_err(errors::AbiCustomSafeForeignFunction { span: sig.span, safe_span });
+            }
+            Safety::Default => match ctxt {
+                FnCtxt::Foreign => { /* all good */ }
+                FnCtxt::Free | FnCtxt::Assoc(_) => {
+                    self.dcx().emit_err(errors::AbiCustomSafeFunction {
+                        span: sig.span,
+                        unsafe_span: sig.span.shrink_to_lo(),
+                    });
+                }
+            },
+        }
+
+        // An `extern "custom"` function must not have any parameters or return type.
+        let mut spans: Vec<_> = sig.decl.inputs.iter().map(|p| p.span).collect();
+        if let FnRetTy::Ty(ref ret_ty) = sig.decl.output {
+            spans.push(ret_ty.span);
+        }
+
+        if !spans.is_empty() {
+            let header_span = sig.header.span().unwrap_or(sig.span.shrink_to_lo());
+            let suggestion_span = sig.span.with_lo(header_span.hi());
+            let padding = if header_span.is_empty() { "" } else { " " };
+
+            self.dcx().emit_err(errors::AbiCustomInvalidSignature {
+                spans,
+                symbol: ident.name,
+                suggestion_span,
+                padding,
+            });
+        }
+    }
+
     /// This ensures that items can only be `unsafe` (or unmarked) outside of extern
     /// blocks.
     ///
@@ -1005,7 +1057,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 if abi.is_none() {
                     self.handle_missing_abi(*extern_span, item.id);
                 }
-                self.with_in_extern_mod(*safety, |this| {
+
+                let extern_abi = abi.and_then(|abi| ExternAbi::from_str(abi.symbol.as_str()).ok());
+                self.with_in_extern_mod(*safety, extern_abi, |this| {
                     visit::walk_item(this, item);
                 });
                 self.extern_mod_span = old_item;
@@ -1145,6 +1199,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 self.check_foreign_fn_bodyless(*ident, body.as_deref());
                 self.check_foreign_fn_headerless(sig.header);
                 self.check_foreign_item_ascii_only(*ident);
+                if self.extern_mod_abi == Some(ExternAbi::Custom) {
+                    self.check_custom_abi(FnCtxt::Foreign, ident, sig);
+                }
             }
             ForeignItemKind::TyAlias(box TyAlias {
                 defaultness,
@@ -1352,6 +1409,13 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             self.check_item_safety(span, safety);
         }
 
+        if let FnKind::Fn(ctxt, _, fun) = fk
+            && let Extern::Explicit(str_lit, _) = fun.sig.header.ext
+            && let Ok(ExternAbi::Custom) = ExternAbi::from_str(str_lit.symbol.as_str())
+        {
+            self.check_custom_abi(ctxt, &fun.ident, &fun.sig);
+        }
+
         self.check_c_variadic_type(fk);
 
         // Functions cannot both be `const async` or `const gen`
@@ -1703,6 +1767,7 @@ pub fn check_crate(
         outer_impl_trait_span: None,
         disallow_tilde_const: Some(TildeConstReason::Item),
         extern_mod_safety: None,
+        extern_mod_abi: None,
         lint_node_id: CRATE_NODE_ID,
         is_sdylib_interface,
         lint_buffer: lints,

compiler/rustc_ast_passes/src/errors.rs

@@ -824,3 +824,51 @@ pub(crate) struct MissingAbi {
     #[suggestion(code = "extern \"<abi>\"", applicability = "has-placeholders")]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_abi_custom_safe_foreign_function)]
+pub(crate) struct AbiCustomSafeForeignFunction {
+    #[primary_span]
+    pub span: Span,
+
+    #[suggestion(
+        ast_passes_suggestion,
+        applicability = "maybe-incorrect",
+        code = "",
+        style = "verbose"
+    )]
+    pub safe_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_abi_custom_safe_function)]
+pub(crate) struct AbiCustomSafeFunction {
+    #[primary_span]
+    pub span: Span,
+
+    #[suggestion(
+        ast_passes_suggestion,
+        applicability = "maybe-incorrect",
+        code = "unsafe ",
+        style = "verbose"
+    )]
+    pub unsafe_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_abi_custom_invalid_signature)]
+#[note]
+pub(crate) struct AbiCustomInvalidSignature {
+    #[primary_span]
+    pub spans: Vec<Span>,
+
+    #[suggestion(
+        ast_passes_suggestion,
+        applicability = "maybe-incorrect",
+        code = "{padding}fn {symbol}()",
+        style = "verbose"
+    )]
+    pub suggestion_span: Span,
+    pub symbol: Symbol,
+    pub padding: &'static str,
+}

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -68,6 +68,11 @@ pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: Call
         Conv::Msp430Intr | Conv::GpuKernel | Conv::AvrInterrupt | Conv::AvrNonBlockingInterrupt => {
             unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");
         }
+
+        // Functions with this calling convention can only be called from assembly, but it is
+        // possible to declare an `extern "custom"` block, so the backend still needs a calling
+        // convention for declaring foreign functions.
+        Conv::Custom => default_call_conv,
     }
 }
 

compiler/rustc_codegen_gcc/src/abi.rs

@@ -291,6 +291,11 @@ pub fn conv_to_fn_attribute<'gcc>(conv: Conv, arch: &str) -> Option<FnAttribute<
         Conv::X86VectorCall => return None,
         Conv::X86_64SysV => FnAttribute::X86SysvAbi,
         Conv::X86_64Win64 => FnAttribute::X86MsAbi,
+
+        // Functions with this calling convention can only be called from assembly, but it is
+        // possible to declare an `extern "custom"` block, so the backend still needs a calling
+        // convention for declaring foreign functions.
+        Conv::Custom => return None,
     };
     Some(attribute)
 }

compiler/rustc_codegen_llvm/src/abi.rs

@@ -676,6 +676,11 @@ impl llvm::CallConv {
             Conv::X86VectorCall => llvm::X86_VectorCall,
             Conv::X86_64SysV => llvm::X86_64_SysV,
             Conv::X86_64Win64 => llvm::X86_64_Win64,
+
+            // Functions with this calling convention can only be called from assembly, but it is
+            // possible to declare an `extern "custom"` block, so the backend still needs a calling
+            // convention for declaring foreign functions.
+            Conv::Custom => llvm::CCallConv,
         }
     }
 }

compiler/rustc_feature/src/unstable.rs

@@ -354,6 +354,8 @@ declare_features! (
     (unstable, abi_avr_interrupt, "1.45.0", Some(69664)),
     /// Allows `extern "C-cmse-nonsecure-call" fn()`.
     (unstable, abi_c_cmse_nonsecure_call, "1.51.0", Some(81391)),
+    /// Allows `extern "custom" fn()`.
+    (unstable, abi_custom, "CURRENT_RUSTC_VERSION", Some(140829)),
     /// Allows `extern "gpu-kernel" fn()`.
     (unstable, abi_gpu_kernel, "1.86.0", Some(135467)),
     /// Allows `extern "msp430-interrupt" fn()`.

compiler/rustc_interface/src/passes.rs

@@ -960,6 +960,7 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
                     tcx.ensure_ok().check_mod_loops(module);
                     tcx.ensure_ok().check_mod_attrs(module);
                     tcx.ensure_ok().check_mod_naked_functions(module);
+                    tcx.ensure_ok().check_mod_custom_abi(module);
                     tcx.ensure_ok().check_mod_unstable_api_usage(module);
                 });
             },

compiler/rustc_middle/src/query/mod.rs

@@ -1117,6 +1117,10 @@ rustc_queries! {
         desc { |tcx| "checking naked functions in {}", describe_as_module(key, tcx) }
     }
 
+    query check_mod_custom_abi(key: LocalModDefId) {
+        desc { |tcx| "checking use of `extern \"custom\"` in {}", describe_as_module(key, tcx) }
+    }
+
     query check_mod_privacy(key: LocalModDefId) {
         desc { |tcx| "checking privacy in {}", describe_as_module(key.to_local_def_id(), tcx) }
     }

compiler/rustc_middle/src/ty/layout.rs

@@ -1264,6 +1264,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi)
         | RiscvInterruptS
         | CCmseNonSecureCall
         | CCmseNonSecureEntry
+        | Custom
         | Unadjusted => false,
         Rust | RustCall | RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind,
     }

compiler/rustc_passes/messages.ftl

@@ -4,6 +4,14 @@
 -passes_see_issue =
     see issue #{$issue} <https://github.com/rust-lang/rust/issues/{$issue}> for more information
 
+passes_abi_custom_call =
+    functions with the `"custom"` ABI cannot be called
+    .note = an `extern "custom"` function can only be called from within inline assembly
+
+passes_abi_custom_clothed_function =
+    functions with the `"custom"` ABI must be naked
+    .suggestion = add the `#[unsafe(naked)]` attribute to this function
+
 passes_abi_invalid_attribute =
     `#[rustc_abi]` can only be applied to function items, type aliases, and associated functions
 passes_abi_ne =