regression: UB with `extern fn() -> !` and LTO

[japaric]

STR

$ cargo new --lib ub && cd $_

$ edit src/lib.rs && cat $_

#![feature(lang_items)]
#![no_std]

use core::ptr;

#[no_mangle]
pub unsafe fn _start() {
    extern "C" {
        fn main() -> !;
    }

    // Just to have "something" before `main`
    ptr::read_volatile(0x0 as *const usize);

    main();
}

// stubs
#[lang = "eh_personality"]
extern "C" fn eh_personality() {}

#[lang = "panic_fmt"]
extern "C" fn panic_fmt() {}

$ edit examples/app.rs && cat $_

#![no_std]
#![no_main]

extern crate ub;

#[no_mangle]
pub fn main() -> ! {
    loop {}
}

Compiling with LTO produces:

$ cargo rustc --release --example app -- -C lto -C link-args=-nostartfiles

$ objdump -Cd target/release/examples/app
Disassembly of section .text:

00000000000002c0 <_start>:
 2c0:   48 8b 04 25 00 00 00    mov    0x0,%rax
 2c7:   00

Note that the infinite loop is missing. This program will execute stuff that's not in the .text section.

Whereas compiling without LTO produces:

$ cargo rustc --release --example app -- -C link-args=-nostartfiles

$ objdump -Cd target/release/examples/app
Disassembly of section .text:

00000000000002c0 <main>:
 2c0:   eb fe                   jmp    2c0 <main>
 2c2:   66 2e 0f 1f 84 00 00    nopw   %cs:0x0(%rax,%rax,1)
 2c9:   00 00 00
 2cc:   0f 1f 40 00             nopl   0x0(%rax)

00000000000002d0 <_start>:
 2d0:   50                      push   %rax
 2d1:   48 8b 04 25 00 00 00    mov    0x0,%rax
 2d8:   00
 2d9:   e8 e2 ff ff ff          callq  2c0 <main>

Both main and the infinite loop are preserved.

Meta

$ rustc -V
rustc 1.16.0-nightly (a52da95ce 2017-01-20)

---

This is a regression because nightly-2016-08-01 (I haven't bisected) preserved the infinite loop in presence of LTO.

$ cargo +nightly-2016-08-01 rustc --release --example app -- -C lto -C link-args=-nostartfiles

$ objdump -Cd target/release/examples/app
Disassembly of section .text:

00000000000002c0 <_start>:
 2c0:   48 8b 04 25 00 00 00    mov    0x0,%rax
 2c7:   00
 2c8:   0f 1f 84 00 00 00 00    nopl   0x0(%rax,%rax,1)
 2cf:   00
 2d0:   eb fe                   jmp    2d0 <_start+0x10>

Looking at the emitted object files, before they are sent to the linker, reveals more information:

$ cargo rustc --release --example app -- -C lto --emit=obj -C link-arg=-nostartfiles

$ objdump -Cd ./target/release/examples/app-e892a9c04992a722.o
Disassembly of section .text.main:

0000000000000000 <main>:
   0:   eb fe                   jmp    0 <main>

Disassembly of section .text._start:

0000000000000000 <_start>:
   0:   48 8b 04 25 00 00 00    mov    0x0,%rax
   7:   00

Note that today _start is simply not calling main at all.

$ cargo +nightly-2016-08-01 rustc --release --example app -- -C lto -C link-args=-nostartfiles --emit=obj

$ objdump -Cd ./target/release/examples/app.o
Disassembly of section .text.main:

0000000000000000 <main>:
   0:   eb fe                   jmp    0 <main>

Disassembly of section .text._start:

0000000000000000 <_start>:
   0:   48 8b 04 25 00 00 00    mov    0x0,%rax
   7:   00
   8:   0f 1f 84 00 00 00 00    nopl   0x0(%rax,%rax,1)
   f:   00
  10:   eb fe                   jmp    10 <_start+0x10>

[japaric]

Results of bisecting

Good: nightly-2016-08-01 - 7333c4a
Bad: nightly-2016-08-02 - 28ce3e8

PRs merged between those nightlies:

#35163 - Rollup of 8 pull requests
#34743 - LLVM upgrade
#34830 - trans: Avoid weak linkage for closures when linking with MinGW.
#35130 - Suppress unused type parameter error when type has error field
#35151 - Add libarena from local rust to stage0

So most likely the LLVM upgrade is the cause.

[strega-nil]

This has (Rust) UB, and therefore, there is no guarantee about what will happen. What happens if you remove the (Rust) undefined behavior (namely, writing to null)?

[japaric]

@ubsan Same thing

#![feature(lang_items)]
#![no_std]

#[no_mangle]
pub unsafe fn _start() {
    extern "C" {
        fn main() -> !;
    }

    main();
}

// stubs
#[lang = "eh_personality"]
extern "C" fn eh_personality() {}

#[lang = "panic_fmt"]
extern "C" fn panic_fmt() {}

$ objdump -Cd target/release/examples/app

target/release/examples/app:     file format elf64-x86-64

(yes, it's empty)

---

#![feature(lang_items)]
#![no_std]

use core::ptr;

#[no_mangle]
pub unsafe fn _start() {
    extern "C" {
        fn main() -> !;
    }

    ptr::read_volatile(_start as *const usize);

    main();
}

// stubs
#[lang = "eh_personality"]
extern "C" fn eh_personality() {}

#[lang = "panic_fmt"]
extern "C" fn panic_fmt() {}

$ objdump -Cd target/release/examples/app

target/release/examples/app:     file format elf64-x86-64


Disassembly of section .text:

00000000000002c0 <_start>:
 2c0:   48 8b 05 f9 ff ff ff    mov    -0x7(%rip),%rax        # 2c0 <_start>

[strega-nil]

@japaric thanks!

Relevant LLVM bug: https://llvm.org/bugs/show_bug.cgi?id=965

[ranma42]

Related to #18785