Leverage the interval property to precompute borrow kill points.

Author: cjgillot

compiler/rustc_borrowck/src/dataflow.rs

@@ -156,10 +156,10 @@ impl<'tcx> OutOfScopePrecomputer<'_, 'tcx> {
         &mut self,
         borrow_index: BorrowIndex,
         borrow_region: RegionVid,
-        location: Location,
+        first_location: Location,
     ) {
         // We visit one BB at a time. The complication is that we may start in the
-        // middle of the first BB visited (the one containing `location`), in which
+        // middle of the first BB visited (the one containing `first_location`), in which
         // case we may have to later on process the first part of that BB if there
         // is a path back to its start.
 
@@ -168,61 +168,58 @@ impl<'tcx> OutOfScopePrecomputer<'_, 'tcx> {
         // `visited` once they are added to `stack`, before they are actually
         // processed, because this avoids the need to look them up again on
         // completion.
-        self.visited.insert(location.block);
+        self.visited.insert(first_location.block);
 
-        let mut first_lo = location.statement_index;
-        let first_hi = self.body[location.block].statements.len();
+        let first_block = first_location.block;
+        let mut first_lo = first_location.statement_index;
+        let first_hi = self.body[first_block].statements.len();
 
-        self.visit_stack.push(StackEntry { bb: location.block, lo: first_lo, hi: first_hi });
+        self.visit_stack.push(StackEntry { bb: first_block, lo: first_lo, hi: first_hi });
 
-        while let Some(StackEntry { bb, lo, hi }) = self.visit_stack.pop() {
-            // If we process the first part of the first basic block (i.e. we encounter that block
-            // for the second time), we no longer have to visit its successors again.
-            let mut finished_early = bb == location.block && hi != first_hi;
-            for i in lo..=hi {
-                let location = Location { block: bb, statement_index: i };
+        'preorder: while let Some(StackEntry { bb, lo, hi }) = self.visit_stack.pop() {
+            if let Some(kill_stmt) =
+                self.regioncx.first_non_contained_inclusive(borrow_region, bb, lo, hi)
+            {
+                let kill_location = Location { block: bb, statement_index: kill_stmt };
                 // If region does not contain a point at the location, then add to list and skip
                 // successor locations.
-                if !self.regioncx.region_contains(borrow_region, location) {
-                    debug!("borrow {:?} gets killed at {:?}", borrow_index, location);
-                    self.borrows_out_of_scope_at_location
-                        .entry(location)
-                        .or_default()
-                        .push(borrow_index);
-                    finished_early = true;
-                    break;
-                }
+                debug!("borrow {:?} gets killed at {:?}", borrow_index, kill_location);
+                self.borrows_out_of_scope_at_location
+                    .entry(kill_location)
+                    .or_default()
+                    .push(borrow_index);
+                continue 'preorder;
             }
 
-            if !finished_early {
-                // Add successor BBs to the work list, if necessary.
-                let bb_data = &self.body[bb];
-                debug_assert!(hi == bb_data.statements.len());
-                for succ_bb in bb_data.terminator().successors() {
-                    if !self.visited.insert(succ_bb) {
-                        if succ_bb == location.block && first_lo > 0 {
-                            // `succ_bb` has been seen before. If it wasn't
-                            // fully processed, add its first part to `stack`
-                            // for processing.
-                            self.visit_stack.push(StackEntry {
-                                bb: succ_bb,
-                                lo: 0,
-                                hi: first_lo - 1,
-                            });
-
-                            // And update this entry with 0, to represent the
-                            // whole BB being processed.
-                            first_lo = 0;
-                        }
-                    } else {
-                        // succ_bb hasn't been seen before. Add it to
-                        // `stack` for processing.
-                        self.visit_stack.push(StackEntry {
-                            bb: succ_bb,
-                            lo: 0,
-                            hi: self.body[succ_bb].statements.len(),
-                        });
+            // If we process the first part of the first basic block (i.e. we encounter that block
+            // for the second time), we no longer have to visit its successors again.
+            if bb == first_block && hi != first_hi {
+                continue;
+            }
+
+            // Add successor BBs to the work list, if necessary.
+            let bb_data = &self.body[bb];
+            debug_assert!(hi == bb_data.statements.len());
+            for succ_bb in bb_data.terminator().successors() {
+                if !self.visited.insert(succ_bb) {
+                    if succ_bb == first_block && first_lo > 0 {
+                        // `succ_bb` has been seen before. If it wasn't
+                        // fully processed, add its first part to `stack`
+                        // for processing.
+                        self.visit_stack.push(StackEntry { bb: succ_bb, lo: 0, hi: first_lo - 1 });
+
+                        // And update this entry with 0, to represent the
+                        // whole BB being processed.
+                        first_lo = 0;
                     }
+                } else {
+                    // succ_bb hasn't been seen before. Add it to
+                    // `stack` for processing.
+                    self.visit_stack.push(StackEntry {
+                        bb: succ_bb,
+                        lo: 0,
+                        hi: self.body[succ_bb].statements.len(),
+                    });
                 }
             }
         }

compiler/rustc_borrowck/src/region_infer/mod.rs

@@ -12,7 +12,7 @@ use rustc_infer::infer::outlives::test_type_match;
 use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound, VerifyIfEq};
 use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin};
 use rustc_middle::mir::{
-    Body, ClosureOutlivesRequirement, ClosureOutlivesSubject, ClosureOutlivesSubjectTy,
+    BasicBlock, Body, ClosureOutlivesRequirement, ClosureOutlivesSubject, ClosureOutlivesSubjectTy,
     ClosureRegionRequirements, ConstraintCategory, Local, Location, ReturnConstraint,
     TerminatorKind,
 };
@@ -598,6 +598,20 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         self.scc_values.contains(scc, p)
     }
 
+    /// Returns the lowest statement index in `start..=end` which is not contained by `r`.
+    ///
+    /// Panics if called before `solve()` executes.
+    pub(crate) fn first_non_contained_inclusive(
+        &self,
+        r: RegionVid,
+        block: BasicBlock,
+        start: usize,
+        end: usize,
+    ) -> Option<usize> {
+        let scc = self.constraint_sccs.scc(r);
+        self.scc_values.first_non_contained_inclusive(scc, block, start, end)
+    }
+
     /// Returns access to the value of `r` for debugging purposes.
     pub(crate) fn region_value_str(&self, r: RegionVid) -> String {
         let scc = self.constraint_sccs.scc(r);

compiler/rustc_borrowck/src/region_infer/values.rs

@@ -283,6 +283,22 @@ impl<N: Idx> RegionValues<N> {
         elem.contained_in_row(self, r)
     }
 
+    /// Returns the lowest statement index in `start..=end` which is not contained by `r`.
+    pub(crate) fn first_non_contained_inclusive(
+        &self,
+        r: N,
+        block: BasicBlock,
+        start: usize,
+        end: usize,
+    ) -> Option<usize> {
+        let row = self.points.row(r)?;
+        let block = self.elements.entry_point(block);
+        let start = block.plus(start);
+        let end = block.plus(end);
+        let first_unset = row.first_unset_in(start..=end)?;
+        Some(first_unset.index() - block.index())
+    }
+
     /// `self[to] |= values[from]`, essentially: that is, take all the
     /// elements for the region `from` from `values` and add them to
     /// the region `to` in `self`.

compiler/rustc_index/src/interval.rs

@@ -181,6 +181,30 @@ impl<I: Idx> IntervalSet<I> {
         self.map.is_empty()
     }
 
+    /// Equivalent to `range.iter().find(|i| !self.contains(i))`.
+    pub fn first_unset_in(&self, range: impl RangeBounds<I> + Clone) -> Option<I> {
+        let start = inclusive_start(range.clone());
+        let Some(end) = inclusive_end(self.domain, range) else {
+            // empty range
+            return None;
+        };
+        if start > end {
+            return None;
+        }
+        let Some(last) = self.map.partition_point(|r| r.0 <= start).checked_sub(1) else {
+            // All ranges in the map start after the new range's end
+            return Some(I::new(start as usize));
+        };
+        let (_, prev_end) = self.map[last];
+        if start > prev_end {
+            Some(I::new(start as usize))
+        } else if prev_end < end {
+            Some(I::new(prev_end as usize + 1))
+        } else {
+            None
+        }
+    }
+
     /// Returns the maximum (last) element present in the set from `range`.
     pub fn last_set_in(&self, range: impl RangeBounds<I> + Clone) -> Option<I> {
         let start = inclusive_start(range.clone());