Soundness hole in pattern matching on enums with an uninhabited variant

[strawberry-choco]

Playground link: https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=7a8a62d7a736d6bd54273422da29d7af

Click to see the example code from above link

#[derive(Debug)]
enum A {
    B(B),
    C(C),
    A1,
    A2,
}

#[derive(Debug)]
enum B {
    B1,
    B2,
    B3,
    B4,
    B5,
}

#[derive(Debug)]
enum C {}

fn main() {
    let a = A::B(B::B5);
    match a {
        A::B(_) => println!("success"),
        _ => println!("error: {:?}", a),
    }
}

Expected output: success

Actual output:

timeout: the monitored command dumped core
/root/entrypoint.sh: line 8:     7 Illegal instruction     timeout --signal=KILL ${timeout} "$@"

Running the above code on both Rust stable and nightly on my machine returns illegal hardware instruction (core dumped) with some number different on each run prefixing the output.

Some experimentations:

• Implementing variants for enum C solves this issue.
• Remove variants A1 and A2 solves this issue.
• Use any variant of B other than B5 works.

I assume this is a bug in the compiler?

[Mark-Simulacrum]

cc @eddyb since potentially has something to do with enum optimizations?

[Centril]

Minimized:

#![feature(never_type, core_intrinsics)]

pub enum E1 {
    V1 { f: bool },
    V2 { f: ! },
    V3,
    V4,
}

fn main() {
    match (E1::V1 { f: true }) {
        E1::V2 { .. } => unsafe { ::core::intrinsics::unreachable() },
        _ => {},
    }
}

MIR:

// WARNING: This output format is intended for human consumers only
// and is subject to change without notice. Knock yourself out.
fn  main() -> () {
    let mut _0: ();                      // return place in scope 0 at src/main.rs:10:11: 10:11
    let mut _1: E1;                      // in scope 0 at src/main.rs:11:11: 11:31
    let mut _2: isize;                   // in scope 0 at src/main.rs:12:9: 12:22
    scope 1 {
    }

    bb0: {
        StorageLive(_1);                 // bb0[0]: scope 0 at src/main.rs:11:11: 11:31
        ((_1 as V1).0: bool) = const true; // bb0[1]: scope 0 at src/main.rs:11:11: 11:31
                                         // ty::Const
                                         // + ty: bool
                                         // + val: Scalar(0x01)
                                         // mir::Constant
                                         // + span: src/main.rs:11:24: 11:28
                                         // + ty: bool
                                         // + literal: Const { ty: bool, val: Scalar(0x01) }
        discriminant(_1) = 0;            // bb0[2]: scope 0 at src/main.rs:11:11: 11:31
        _2 = discriminant(_1);           // bb0[3]: scope 0 at src/main.rs:12:9: 12:22
        switchInt(move _2) -> [1isize: bb1, otherwise: bb2]; // bb0[4]: scope 0 at src/main.rs:12:9: 12:22
    }

    bb1: {
        const std::intrinsics::unreachable(); // bb1[0]: scope 1 at src/main.rs:12:35: 12:68
                                         // ty::Const
                                         // + ty: unsafe extern "rust-intrinsic" fn() -> ! {std::intrinsics::unreachable}
                                         // + val: Scalar(<ZST>)
                                         // mir::Constant
                                         // + span: src/main.rs:12:35: 12:66
                                         // + ty: unsafe extern "rust-intrinsic" fn() -> ! {std::intrinsics::unreachable}
                                         // + literal: Const { ty: unsafe extern "rust-intrinsic" fn() -> ! {std::intrinsics::unreachable}, val: Scalar(<ZST>) }
    }

    bb2: {
        StorageDead(_1);                 // bb2[0]: scope 0 at src/main.rs:15:1: 15:2
        return;                          // bb2[1]: scope 0 at src/main.rs:15:2: 15:2
    }
}

Debug ASM (does not reproduce with release profile):

```asm std::rt::lang_start: # @std::rt::lang_start # %bb.0: subq $56, %rsp leaq .L__unnamed_1(%rip), %rax movq %rdi, 24(%rsp) movq %rsi, 32(%rsp) movq %rdx, 40(%rsp) movq 24(%rsp), %rdx movq %rdx, 48(%rsp) leaq 48(%rsp), %rdx movq 32(%rsp), %rsi movq 40(%rsp), %rcx movq %rdx, %rdi movq %rsi, 16(%rsp) # 8-byte Spill movq %rax, %rsi movq 16(%rsp), %rdx # 8-byte Reload callq *std::rt::lang_start_internal@GOTPCREL(%rip) movq %rax, 8(%rsp) # 8-byte Spill # %bb.1: movq 8(%rsp), %rax # 8-byte Reload addq $56, %rsp retq # -- End function

std::rt::lang_start::{{closure}}: # @"std::rt::lang_start::{{closure}}"

%bb.0:

subq	$24, %rsp
movq	%rdi, 16(%rsp)
movq	16(%rsp), %rdi
callq	*(%rdi)

%bb.1:

callq	<() as std::process::Termination>::report
movl	%eax, 12(%rsp)          # 4-byte Spill

%bb.2:

movl	12(%rsp), %eax          # 4-byte Reload
addq	$24, %rsp
retq
                                    # -- End function

std::sys::unix::process::process_common::ExitCode::as_i32: # @std::sys::unix::process::process_common::ExitCode::as_i32

%bb.0:

pushq	%rax
movq	%rdi, (%rsp)
movq	(%rsp), %rdi
movzbl	(%rdi), %eax
popq	%rcx
retq
                                    # -- End function

core::ops::function::FnOnce::call_once{{vtable.shim}}: # @"core::ops::function::FnOnce::call_once{{vtable.shim}}"

%bb.0:

subq	$24, %rsp
movq	%rdi, 8(%rsp)
movq	8(%rsp), %rdi
movq	(%rdi), %rdi
callq	core::ops::function::FnOnce::call_once
movl	%eax, 4(%rsp)           # 4-byte Spill

%bb.1:

movl	4(%rsp), %eax           # 4-byte Reload
addq	$24, %rsp
retq
                                    # -- End function

core::ops::function::FnOnce::call_once: # @core::ops::function::FnOnce::call_once

%bb.0:

subq	$40, %rsp
movq	%rdi, 8(%rsp)
leaq	8(%rsp), %rdi
callq	std::rt::lang_start::{{closure}}
movl	%eax, 4(%rsp)           # 4-byte Spill
jmp	.LBB4_1

.LBB4_1:
jmp .LBB4_2

.LBB4_2:
movl 4(%rsp), %eax # 4-byte Reload
addq $40, %rsp
retq

.LBB4_3:
jmp .LBB4_4

.LBB4_4:
movq 24(%rsp), %rdi
callq _Unwind_Resume@PLT
ud2
movl %edx, %ecx
movq %rax, 24(%rsp)
movl %ecx, 32(%rsp)
jmp .LBB4_3
# -- End function

core::ptr::real_drop_in_place: # @core::ptr::real_drop_in_place

%bb.0:

pushq	%rax
movq	%rdi, (%rsp)
popq	%rax
retq
                                    # -- End function

<() as std::process::Termination>::report: # @"<() as std::process::Termination>::report"

%bb.0:

subq	$24, %rsp
xorl	%edi, %edi
callq	<std::process::ExitCode as std::process::Termination>::report
movl	%eax, 12(%rsp)          # 4-byte Spill

%bb.1:

movl	12(%rsp), %eax          # 4-byte Reload
addq	$24, %rsp
retq
                                    # -- End function

<std::process::ExitCode as std::process::Termination>::report: # @"<std::process::ExitCode as std::process::Termination>::report"

%bb.0:

pushq	%rax
movb	%dil, %al
movb	%al, 7(%rsp)
leaq	7(%rsp), %rdi
#DEBUG_VALUE: report:self <- [$rdi+0]
callq	std::sys::unix::process::process_common::ExitCode::as_i32
movl	%eax, (%rsp)            # 4-byte Spill

%bb.1:

movl	(%rsp), %eax            # 4-byte Reload
popq	%rcx
retq
                                    # -- End function

playground::main: # @playground::main

%bb.0:

subq	$1, %rsp
xorl	%eax, %eax
movl	%eax, %ecx
movb	$1, (%rsp)
movb	(%rsp), %dl
subb	$0, %dl
movzbl	%dl, %eax
movl	%eax, %esi
cmpb	$3, %dl
cmovbeq	%rsi, %rcx
cmpq	$1, %rcx
jne	.LBB8_2

%bb.1:

ud2

.LBB8_2:
addq $1, %rsp
retq
# -- End function

main: # @main

%bb.0:

subq	$24, %rsp
movb	__rustc_debug_gdb_scripts_section__(%rip), %al
movslq	%edi, %rcx
leaq	playground::main(%rip), %rdi
movq	%rsi, 16(%rsp)          # 8-byte Spill
movq	%rcx, %rsi
movq	16(%rsp), %rdx          # 8-byte Reload
movb	%al, 15(%rsp)           # 1-byte Spill
callq	std::rt::lang_start
movl	%eax, %r8d
movl	%r8d, %eax
addq	$24, %rsp
retq
                                    # -- End function

.L__unnamed_1:
.quad core::ptr::real_drop_in_place
.quad 8 # 0x8
.quad 8 # 0x8
.quad std::rt::lang_start::{{closure}}
.quad std::rt::lang_start::{{closure}}
.quad core::ops::function::FnOnce::call_once{{vtable.shim}}

rustc_debug_gdb_scripts_section:
.asciz "\001gdb_load_rust_pretty_printers.py"
# DW_AT_GNU_pubnames
# DW_AT_main_subprogram

</details>

[jonas-schievink]

Regression between 1.23.0 and 1.24.0

[Centril]

This seems like a rather serious soundness hole that I would suggest treating as P-high.

cc @rust-lang/compiler

[nagisa]

Given

#![feature(never_type, core_intrinsics)]
extern crate core;

pub enum E1 {
    V1 { f: bool },
    V2 { f: ! },
    V3,
    V4,
}

fn main() {
    match (E1::V1 { f: true }) {
        E1::V2 { .. } => unsafe { ::core::intrinsics::unreachable() },
        _ => {},
    }
}

LLVM for the interesting code is:

; test::main
; Function Attrs: nonlazybind uwtable
define internal void @_ZN4test4main17h4c216eb98fecdd36E() unnamed_addr #0 {
start:
  %_1 = alloca i8, align 1
  store i8 1, i8* %_1, align 1
  %0 = load i8, i8* %_1, align 1, !range !3    ; %0 = 1
  %1 = sub i8 %0, 0                            ; %1 = 1
  %2 = icmp ule i8 %1, 3                       ; %2 = 1 <= 3 = 1
  %3 = zext i8 %1 to i64                       ; %3 = 1
  %4 = select i1 %2, i64 %3, i64 0             ; %4 = 0
  %5 = icmp eq i64 %4, 1                       ; %5 = 0
  br i1 %5, label %bb1, label %bb2             ; -> bb1

bb1:                                              ; preds = %start
  unreachable

bb2:                                              ; preds = %start
  ret void
}

!3 = !{i8 0, i8 4}

[nagisa]

This is probably not A-LLVM because the generated LLVM IR is already wrong.