Reorder MIR passes.

Author: cjgillot

compiler/rustc_mir_transform/src/dead_store_elimination.rs

@@ -127,13 +127,15 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 
 pub enum DeadStoreElimination {
     Initial,
+    AfterJumpThreading,
     Final,
 }
 
 impl<'tcx> MirPass<'tcx> for DeadStoreElimination {
     fn name(&self) -> &'static str {
         match self {
             DeadStoreElimination::Initial => "DeadStoreElimination-initial",
+            DeadStoreElimination::AfterJumpThreading => "DeadStoreElimination-after-jump-threading",
             DeadStoreElimination::Final => "DeadStoreElimination-final",
         }
     }

compiler/rustc_mir_transform/src/gvn.rs

@@ -105,9 +105,19 @@ use crate::dataflow_const_prop::DummyMachine;
 use crate::ssa::{AssignedValue, SsaLocals};
 use either::Either;
 
-pub struct GVN;
+pub enum GVN {
+    Initial,
+    Final,
+}
 
 impl<'tcx> MirPass<'tcx> for GVN {
+    fn name(&self) -> &'static str {
+        match self {
+            GVN::Initial => "GVN",
+            GVN::Final => "GVN-final",
+        }
+    }
+
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
         sess.mir_opt_level() >= 2
     }

compiler/rustc_mir_transform/src/lib.rs

@@ -566,7 +566,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &check_alignment::CheckAlignment,
             // Before inlining: trim down MIR with passes to reduce inlining work.
 
-            // Has to be done before inlining, otherwise actual call will be almost always inlined.
+            // has to be done before inlining, otherwise actual call will be almost always inlined.
             // Also simple, so can just do first
             &lower_slice_len::LowerSliceLenCalls,
             // Perform inlining, which may add a lot of code.
@@ -583,30 +583,51 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             // Inlining may have introduced a lot of redundant code and a large move pattern.
             // Now, we need to shrink the generated MIR.
 
-            // Has to run after `slice::len` lowering
-            &normalize_array_len::NormalizeArrayLen,
+            // Replace deref of borrow with direct uses.
             &ref_prop::ReferencePropagation,
-            &sroa::ScalarReplacementOfAggregates,
+            // Remove dead stores to help SSA-based analyses.
+            &dead_store_elimination::DeadStoreElimination::Initial,
+            // Deduplicate computations.
+            &gvn::GVN::Initial,
+            // GVN may have moved constant assignments in terminators.
+            &simplify_branches::SimplifyConstCondition::AfterGVN,
+            // Use const-prop results in debuginfo, to remove some locals' uses.
+            &const_debuginfo::ConstDebugInfo,
+            // Replace `if x == constant { true/false } else { false/true }` by `Eq/Ne(x, constant)`.
             &match_branches::MatchBranchSimplification,
             // inst combine is after MatchBranchSimplification to clean up Ne(_1, false)
-            &multiple_return_terminators::MultipleReturnTerminators,
             &instsimplify::InstSimplify,
+            // Turn non-aliasing places into locals.
+            // Helps `DeadStoreElimination` to remove them and `CopyProp` to merge them.
+            &simplify::SimplifyLocals::BeforeSROA,
+            &sroa::ScalarReplacementOfAggregates,
+            // Compute `Len` when it comes from an unsizing cast.
+            &normalize_array_len::NormalizeArrayLen,
+            // Perform a first round of constant propagation.
+            &o1(simplify::SimplifyCfg::BeforeConstProp),
             &simplify::SimplifyLocals::BeforeConstProp,
-            &dead_store_elimination::DeadStoreElimination::Initial,
-            &gvn::GVN,
-            &simplify::SimplifyLocals::AfterGVN,
-            &dataflow_const_prop::DataflowConstProp,
-            &const_debuginfo::ConstDebugInfo,
-            &o1(simplify_branches::SimplifyConstCondition::AfterConstProp),
-            &jump_threading::JumpThreading,
-            &early_otherwise_branch::EarlyOtherwiseBranch,
+            // Turn `SwitchInt(Eq(x, constant))` into `SwitchInt(x)`.
+            // Runs after ConstProp to increase probability to have constants.
             &simplify_comparison_integral::SimplifyComparisonIntegral,
-            &o1(simplify_branches::SimplifyConstCondition::Final),
+            &early_otherwise_branch::EarlyOtherwiseBranch,
+            // Merge locals that just copy each another.
+            &copy_prop::CopyProp,
+            // Thread jumps when possible. Note that no SSA-based pass should happen
+            // post-jump-threading, as this pass clones blocks and demotes SSA locals.
+            &jump_threading::JumpThreading,
+            &dataflow_const_prop::DataflowConstProp,
+            &simplify_branches::SimplifyConstCondition::Final,
+            // Jump threading may have created dead stores, notably enums which discriminant we
+            // don't read any more. Remove them.
+            &dead_store_elimination::DeadStoreElimination::AfterJumpThreading,
+            // Jump threading may have created wrap-upwrap sequences in enums. Remove those.
+            &gvn::GVN::Final,
             &o1(remove_noop_landing_pads::RemoveNoopLandingPads),
             &o1(simplify::SimplifyCfg::Final),
-            &copy_prop::CopyProp,
-            &dead_store_elimination::DeadStoreElimination::Final,
+            &simplify::SimplifyLocals::AfterGVN,
             &dest_prop::DestinationPropagation,
+            // Attempt to promote call operands from copies to moves.
+            &dead_store_elimination::DeadStoreElimination::Final,
             &simplify::SimplifyLocals::Final,
             &multiple_return_terminators::MultipleReturnTerminators,
             &deduplicate_blocks::DeduplicateBlocks,

compiler/rustc_mir_transform/src/simplify.rs

@@ -42,6 +42,7 @@ pub enum SimplifyCfg {
     Final,
     MakeShim,
     AfterUninhabitedEnumBranching,
+    BeforeConstProp,
 }
 
 impl SimplifyCfg {
@@ -57,6 +58,7 @@ impl SimplifyCfg {
             SimplifyCfg::AfterUninhabitedEnumBranching => {
                 "SimplifyCfg-after-uninhabited-enum-branching"
             }
+            SimplifyCfg::BeforeConstProp => "SimplifyCfg-before-const-prop",
         }
     }
 }
@@ -346,6 +348,7 @@ pub(crate) fn remove_dead_blocks(body: &mut Body<'_>) {
 }
 
 pub enum SimplifyLocals {
+    BeforeSROA,
     BeforeConstProp,
     AfterGVN,
     Final,
@@ -354,6 +357,7 @@ pub enum SimplifyLocals {
 impl<'tcx> MirPass<'tcx> for SimplifyLocals {
     fn name(&self) -> &'static str {
         match &self {
+            SimplifyLocals::BeforeSROA => "SimplifyLocals-before-sroa",
             SimplifyLocals::BeforeConstProp => "SimplifyLocals-before-const-prop",
             SimplifyLocals::AfterGVN => "SimplifyLocals-after-value-numbering",
             SimplifyLocals::Final => "SimplifyLocals-final",

compiler/rustc_mir_transform/src/simplify_branches.rs

@@ -2,18 +2,25 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
 pub enum SimplifyConstCondition {
-    AfterConstProp,
+    AfterGVN,
     Final,
 }
 /// A pass that replaces a branch with a goto when its condition is known.
 impl<'tcx> MirPass<'tcx> for SimplifyConstCondition {
     fn name(&self) -> &'static str {
         match self {
-            SimplifyConstCondition::AfterConstProp => "SimplifyConstCondition-after-const-prop",
+            SimplifyConstCondition::AfterGVN => "SimplifyConstCondition-after-gvn",
             SimplifyConstCondition::Final => "SimplifyConstCondition-final",
         }
     }
 
+    fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
+        match self {
+            SimplifyConstCondition::Final => sess.mir_opt_level() >= 1,
+            SimplifyConstCondition::AfterGVN => sess.mir_opt_level() >= 2,
+        }
+    }
+
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         trace!("Running SimplifyConstCondition on {:?}", body.source);
         let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());

tests/coverage/match_or_pattern.cov-map

@@ -1,5 +1,5 @@
 Function name: match_or_pattern::main
-Raw bytes (202): 0x[01, 01, 23, 01, 05, 05, 02, 09, 0d, 2f, 00, 09, 0d, 2b, 15, 2f, 00, 09, 0d, 15, 26, 2b, 15, 2f, 00, 09, 0d, 19, 1d, 57, 00, 19, 1d, 53, 25, 57, 00, 19, 1d, 25, 4e, 53, 25, 57, 00, 19, 1d, 29, 2d, 7f, 31, 29, 2d, 7b, 35, 7f, 31, 29, 2d, 35, 76, 7b, 35, 7f, 31, 29, 2d, 39, 3d, 8b, 01, 41, 39, 3d, 19, 01, 01, 01, 08, 0f, 05, 08, 10, 03, 06, 02, 03, 06, 00, 07, 07, 01, 0b, 00, 11, 00, 03, 1b, 00, 1d, 2f, 01, 0e, 00, 10, 2b, 02, 08, 00, 0f, 15, 00, 10, 03, 06, 26, 03, 06, 00, 07, 23, 01, 0b, 00, 11, 00, 01, 1b, 00, 1d, 57, 01, 0e, 00, 10, 53, 02, 08, 00, 0f, 25, 00, 10, 03, 06, 4e, 03, 06, 00, 07, 4b, 01, 0b, 00, 11, 31, 01, 1b, 00, 1d, 7f, 01, 0e, 00, 10, 7b, 02, 08, 00, 0f, 35, 00, 10, 03, 06, 76, 03, 06, 00, 07, 73, 01, 0b, 00, 11, 41, 01, 1b, 00, 1d, 8b, 01, 01, 0e, 00, 10, 87, 01, 02, 01, 00, 02]
+Raw bytes (202): 0x[01, 01, 23, 01, 05, 05, 02, 09, 0d, 2f, 00, 09, 0d, 2b, 15, 2f, 00, 09, 0d, 15, 26, 2b, 15, 2f, 00, 09, 0d, 00, 1d, 57, 00, 00, 1d, 53, 25, 57, 00, 00, 1d, 25, 4e, 53, 25, 57, 00, 00, 1d, 29, 2d, 7f, 00, 29, 2d, 7b, 35, 7f, 00, 29, 2d, 35, 76, 7b, 35, 7f, 00, 29, 2d, 00, 3d, 8b, 01, 41, 00, 3d, 19, 01, 01, 01, 08, 0f, 05, 08, 10, 03, 06, 02, 03, 06, 00, 07, 07, 01, 0b, 00, 11, 00, 03, 1b, 00, 1d, 2f, 01, 0e, 00, 10, 2b, 02, 08, 00, 0f, 15, 00, 10, 03, 06, 26, 03, 06, 00, 07, 23, 01, 0b, 00, 11, 00, 01, 1b, 00, 1d, 57, 01, 0e, 00, 10, 53, 02, 08, 00, 0f, 25, 00, 10, 03, 06, 4e, 03, 06, 00, 07, 4b, 01, 0b, 00, 11, 00, 01, 1b, 00, 1d, 7f, 01, 0e, 00, 10, 7b, 02, 08, 00, 0f, 35, 00, 10, 03, 06, 76, 03, 06, 00, 07, 73, 01, 0b, 00, 11, 41, 01, 1b, 00, 1d, 8b, 01, 01, 0e, 00, 10, 87, 01, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 35
@@ -15,29 +15,29 @@ Number of expressions: 35
 - expression 9 operands: lhs = Expression(10, Add), rhs = Counter(5)
 - expression 10 operands: lhs = Expression(11, Add), rhs = Zero
 - expression 11 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 12 operands: lhs = Counter(6), rhs = Counter(7)
+- expression 12 operands: lhs = Zero, rhs = Counter(7)
 - expression 13 operands: lhs = Expression(21, Add), rhs = Zero
-- expression 14 operands: lhs = Counter(6), rhs = Counter(7)
+- expression 14 operands: lhs = Zero, rhs = Counter(7)
 - expression 15 operands: lhs = Expression(20, Add), rhs = Counter(9)
 - expression 16 operands: lhs = Expression(21, Add), rhs = Zero
-- expression 17 operands: lhs = Counter(6), rhs = Counter(7)
+- expression 17 operands: lhs = Zero, rhs = Counter(7)
 - expression 18 operands: lhs = Counter(9), rhs = Expression(19, Sub)
 - expression 19 operands: lhs = Expression(20, Add), rhs = Counter(9)
 - expression 20 operands: lhs = Expression(21, Add), rhs = Zero
-- expression 21 operands: lhs = Counter(6), rhs = Counter(7)
+- expression 21 operands: lhs = Zero, rhs = Counter(7)
 - expression 22 operands: lhs = Counter(10), rhs = Counter(11)
-- expression 23 operands: lhs = Expression(31, Add), rhs = Counter(12)
+- expression 23 operands: lhs = Expression(31, Add), rhs = Zero
 - expression 24 operands: lhs = Counter(10), rhs = Counter(11)
 - expression 25 operands: lhs = Expression(30, Add), rhs = Counter(13)
-- expression 26 operands: lhs = Expression(31, Add), rhs = Counter(12)
+- expression 26 operands: lhs = Expression(31, Add), rhs = Zero
 - expression 27 operands: lhs = Counter(10), rhs = Counter(11)
 - expression 28 operands: lhs = Counter(13), rhs = Expression(29, Sub)
 - expression 29 operands: lhs = Expression(30, Add), rhs = Counter(13)
-- expression 30 operands: lhs = Expression(31, Add), rhs = Counter(12)
+- expression 30 operands: lhs = Expression(31, Add), rhs = Zero
 - expression 31 operands: lhs = Counter(10), rhs = Counter(11)
-- expression 32 operands: lhs = Counter(14), rhs = Counter(15)
+- expression 32 operands: lhs = Zero, rhs = Counter(15)
 - expression 33 operands: lhs = Expression(34, Add), rhs = Counter(16)
-- expression 34 operands: lhs = Counter(14), rhs = Counter(15)
+- expression 34 operands: lhs = Zero, rhs = Counter(15)
 Number of file 0 mappings: 25
 - Code(Counter(0)) at (prev + 1, 1) to (start + 8, 15)
 - Code(Counter(1)) at (prev + 8, 16) to (start + 3, 6)
@@ -57,27 +57,27 @@ Number of file 0 mappings: 25
     = (c5 + (((c2 + c3) + Zero) - c5))
 - Code(Zero) at (prev + 1, 27) to (start + 0, 29)
 - Code(Expression(21, Add)) at (prev + 1, 14) to (start + 0, 16)
-    = (c6 + c7)
+    = (Zero + c7)
 - Code(Expression(20, Add)) at (prev + 2, 8) to (start + 0, 15)
-    = ((c6 + c7) + Zero)
+    = ((Zero + c7) + Zero)
 - Code(Counter(9)) at (prev + 0, 16) to (start + 3, 6)
 - Code(Expression(19, Sub)) at (prev + 3, 6) to (start + 0, 7)
-    = (((c6 + c7) + Zero) - c9)
+    = (((Zero + c7) + Zero) - c9)
 - Code(Expression(18, Add)) at (prev + 1, 11) to (start + 0, 17)
-    = (c9 + (((c6 + c7) + Zero) - c9))
-- Code(Counter(12)) at (prev + 1, 27) to (start + 0, 29)
+    = (c9 + (((Zero + c7) + Zero) - c9))
+- Code(Zero) at (prev + 1, 27) to (start + 0, 29)
 - Code(Expression(31, Add)) at (prev + 1, 14) to (start + 0, 16)
     = (c10 + c11)
 - Code(Expression(30, Add)) at (prev + 2, 8) to (start + 0, 15)
-    = ((c10 + c11) + c12)
+    = ((c10 + c11) + Zero)
 - Code(Counter(13)) at (prev + 0, 16) to (start + 3, 6)
 - Code(Expression(29, Sub)) at (prev + 3, 6) to (start + 0, 7)
-    = (((c10 + c11) + c12) - c13)
+    = (((c10 + c11) + Zero) - c13)
 - Code(Expression(28, Add)) at (prev + 1, 11) to (start + 0, 17)
-    = (c13 + (((c10 + c11) + c12) - c13))
+    = (c13 + (((c10 + c11) + Zero) - c13))
 - Code(Counter(16)) at (prev + 1, 27) to (start + 0, 29)
 - Code(Expression(34, Add)) at (prev + 1, 14) to (start + 0, 16)
-    = (c14 + c15)
+    = (Zero + c15)
 - Code(Expression(33, Add)) at (prev + 2, 1) to (start + 0, 2)
-    = ((c14 + c15) + c16)
+    = ((Zero + c15) + c16)
 

tests/mir-opt/const_debuginfo.rs

@@ -1,5 +1,5 @@
 // unit-test: ConstDebugInfo
-// compile-flags: -C overflow-checks=no -Zmir-enable-passes=+GVN
+// compile-flags: -C overflow-checks=no -Zmir-enable-passes=+GVN -g
 
 struct Point {
     x: u32,

tests/mir-opt/const_prop/slice_len.main.GVN.32bit.panic-abort.diff

@@ -22,12 +22,9 @@
           StorageLive(_3);
           StorageLive(_4);
           _9 = const _;
--         _4 = _9;
--         _3 = _4;
--         _2 = move _3 as &[u32] (PointerCoercion(Unsize));
-+         _4 = const {ALLOC0<imm>: &[u32; 3]};
-+         _3 = const {ALLOC0<imm>: &[u32; 3]};
-+         _2 = const {ALLOC0<imm>: &[u32; 3]} as &[u32] (PointerCoercion(Unsize));
+          _4 = &(*_9);
+          _3 = &(*_4);
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize));
           StorageDead(_3);
           StorageLive(_6);
           _6 = const 1_usize;
@@ -49,9 +46,5 @@
           StorageDead(_1);
           return;
       }
-+ }
-+ 
-+ ALLOC0 (size: 12, align: 4) {
-+     01 00 00 00 02 00 00 00 03 00 00 00             │ ............
   }
 

tests/mir-opt/const_prop/slice_len.main.GVN.32bit.panic-unwind.diff

@@ -22,12 +22,9 @@
           StorageLive(_3);
           StorageLive(_4);
           _9 = const _;
--         _4 = _9;
--         _3 = _4;
--         _2 = move _3 as &[u32] (PointerCoercion(Unsize));
-+         _4 = const {ALLOC0<imm>: &[u32; 3]};
-+         _3 = const {ALLOC0<imm>: &[u32; 3]};
-+         _2 = const {ALLOC0<imm>: &[u32; 3]} as &[u32] (PointerCoercion(Unsize));
+          _4 = &(*_9);
+          _3 = &(*_4);
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize));
           StorageDead(_3);
           StorageLive(_6);
           _6 = const 1_usize;
@@ -49,9 +46,5 @@
           StorageDead(_1);
           return;
       }
-+ }
-+ 
-+ ALLOC0 (size: 12, align: 4) {
-+     01 00 00 00 02 00 00 00 03 00 00 00             │ ............
   }
 

tests/mir-opt/const_prop/slice_len.main.GVN.64bit.panic-abort.diff

@@ -22,12 +22,9 @@
           StorageLive(_3);
           StorageLive(_4);
           _9 = const _;
--         _4 = _9;
--         _3 = _4;
--         _2 = move _3 as &[u32] (PointerCoercion(Unsize));
-+         _4 = const {ALLOC0<imm>: &[u32; 3]};
-+         _3 = const {ALLOC0<imm>: &[u32; 3]};
-+         _2 = const {ALLOC0<imm>: &[u32; 3]} as &[u32] (PointerCoercion(Unsize));
+          _4 = &(*_9);
+          _3 = &(*_4);
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize));
           StorageDead(_3);
           StorageLive(_6);
           _6 = const 1_usize;
@@ -49,9 +46,5 @@
           StorageDead(_1);
           return;
       }
-+ }
-+ 
-+ ALLOC0 (size: 12, align: 4) {
-+     01 00 00 00 02 00 00 00 03 00 00 00             │ ............
   }
 

tests/mir-opt/const_prop/slice_len.main.GVN.64bit.panic-unwind.diff

@@ -22,12 +22,9 @@
           StorageLive(_3);
           StorageLive(_4);
           _9 = const _;
--         _4 = _9;
--         _3 = _4;
--         _2 = move _3 as &[u32] (PointerCoercion(Unsize));
-+         _4 = const {ALLOC0<imm>: &[u32; 3]};
-+         _3 = const {ALLOC0<imm>: &[u32; 3]};
-+         _2 = const {ALLOC0<imm>: &[u32; 3]} as &[u32] (PointerCoercion(Unsize));
+          _4 = &(*_9);
+          _3 = &(*_4);
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize));
           StorageDead(_3);
           StorageLive(_6);
           _6 = const 1_usize;
@@ -49,9 +46,5 @@
           StorageDead(_1);
           return;
       }
-+ }
-+ 
-+ ALLOC0 (size: 12, align: 4) {
-+     01 00 00 00 02 00 00 00 03 00 00 00             │ ............
   }
 

tests/mir-opt/const_prop/slice_len.rs

@@ -7,7 +7,6 @@
 fn main() {
     // CHECK-LABEL: fn main(
     // CHECK: debug a => [[a:_.*]];
-    // CHECK: [[slice:_.*]] = const {{.*}} as &[u32] (PointerCoercion(Unsize));
     // CHECK: assert(const true,
     // CHECK: [[a]] = const 2_u32;
     let a = (&[1u32, 2, 3] as &[u32])[1];

tests/mir-opt/const_prop/switch_int.main.SimplifyConstCondition-after-const-prop.panic-abort.diff renamed to tests/mir-opt/const_prop/switch_int.main.SimplifyConstCondition-after-gvn.panic-abort.diff

@@ -1,5 +1,5 @@
-- // MIR for `main` before SimplifyConstCondition-after-const-prop
-+ // MIR for `main` after SimplifyConstCondition-after-const-prop
+- // MIR for `main` before SimplifyConstCondition-after-gvn
++ // MIR for `main` after SimplifyConstCondition-after-gvn
 
   fn main() -> () {
       let mut _0: ();

tests/mir-opt/const_prop/switch_int.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff renamed to tests/mir-opt/const_prop/switch_int.main.SimplifyConstCondition-after-gvn.panic-unwind.diff

@@ -1,5 +1,5 @@
-- // MIR for `main` before SimplifyConstCondition-after-const-prop
-+ // MIR for `main` after SimplifyConstCondition-after-const-prop
+- // MIR for `main` before SimplifyConstCondition-after-gvn
++ // MIR for `main` after SimplifyConstCondition-after-gvn
 
   fn main() -> () {
       let mut _0: ();