record runtime predicates on each Dependence relation

Author: sebpop

llvm/include/llvm/Analysis/DependenceAnalysis.h

@@ -76,8 +76,12 @@ namespace llvm {
     Dependence &operator=(Dependence &&) = default;
 
   public:
-    Dependence(Instruction *Source, Instruction *Destination)
-        : Src(Source), Dst(Destination) {}
+    Dependence(Instruction *Source, Instruction *Destination,
+               ScalarEvolution *SE,
+               const SmallVectorImpl<const SCEVPredicate *> &Assumes)
+        : Src(Source), Dst(Destination), SE(SE) {
+      Assumptions.append(Assumes.begin(), Assumes.end());
+    }
     virtual ~Dependence() = default;
 
     /// Dependence::DVEntry - Each level in the distance/direction vector
@@ -205,6 +209,10 @@ namespace llvm {
     /// field.
     void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; }
 
+    /// getRuntimeAssumptions - Returns the runtime assumptions under which this
+    /// Dependence relation is valid.
+    SCEVUnionPredicate getRuntimeAssumptions() const;
+
     /// dump - For debugging purposes, dumps a dependence to OS.
     ///
     void dump(raw_ostream &OS) const;
@@ -213,6 +221,8 @@ namespace llvm {
     Instruction *Src, *Dst;
 
   private:
+    ScalarEvolution *SE;
+    SmallVector<const SCEVPredicate *, 4> Assumptions;
     const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr;
     friend class DependenceInfo;
   };
@@ -227,8 +237,10 @@ namespace llvm {
   /// input dependences are unordered.
   class FullDependence final : public Dependence {
   public:
-    FullDependence(Instruction *Src, Instruction *Dst, bool LoopIndependent,
-                   unsigned Levels);
+    FullDependence(Instruction *Source, Instruction *Destination,
+                   ScalarEvolution *SE,
+                   const SmallVectorImpl<const SCEVPredicate *> &Assumes,
+                   bool PossiblyLoopIndependent, unsigned Levels);
 
     /// isLoopIndependent - Returns true if this is a loop-independent
     /// dependence.
@@ -304,9 +316,13 @@ namespace llvm {
 
     /// depends - Tests for a dependence between the Src and Dst instructions.
     /// Returns NULL if no dependence; otherwise, returns a Dependence (or a
-    /// FullDependence) with as much information as can be gleaned.
-    std::unique_ptr<Dependence> depends(Instruction *Src,
-                                        Instruction *Dst);
+    /// FullDependence) with as much information as can be gleaned. By default,
+    /// the dependence test collects a set of runtime assumptions that cannot be
+    /// solved at compilation time. Set UnderRuntimeAssumptions to false for a
+    /// safe approximation of the dependence relation that does not require
+    /// runtime checks.
+    std::unique_ptr<Dependence> depends(Instruction *Src, Instruction *Dst,
+                                        bool UnderRuntimeAssumptions = true);
 
     /// getSplitIteration - Give a dependence that's splittable at some
     /// particular level, return the iteration that should be used to split
@@ -351,7 +367,10 @@ namespace llvm {
     const SCEV *getSplitIteration(const Dependence &Dep, unsigned Level);
 
     Function *getFunction() const { return F; }
-    SCEVUnionPredicate getRuntimeAssumptions();
+
+    /// getRuntimeAssumptions - Returns all the runtime assumptions under which
+    /// the dependence test is valid.
+    SCEVUnionPredicate getRuntimeAssumptions() const;
 
   private:
     AAResults *AA;

llvm/lib/Analysis/DependenceAnalysis.cpp

@@ -268,10 +268,11 @@ bool Dependence::isScalar(unsigned level) const {
 //===----------------------------------------------------------------------===//
 // FullDependence methods
 
-FullDependence::FullDependence(Instruction *Source, Instruction *Destination,
-                               bool PossiblyLoopIndependent,
-                               unsigned CommonLevels)
-    : Dependence(Source, Destination), Levels(CommonLevels),
+FullDependence::FullDependence(
+    Instruction *Source, Instruction *Destination, ScalarEvolution *SE,
+    const SmallVectorImpl<const SCEVPredicate *> &Assumes,
+    bool PossiblyLoopIndependent, unsigned CommonLevels)
+    : Dependence(Source, Destination, SE, Assumes), Levels(CommonLevels),
       LoopIndependent(PossiblyLoopIndependent) {
   Consistent = true;
   if (CommonLevels)
@@ -711,6 +712,12 @@ void Dependence::dump(raw_ostream &OS) const {
       OS << " splitable";
   }
   OS << "!\n";
+
+  SCEVUnionPredicate Assumptions = getRuntimeAssumptions();
+  if (!Assumptions.isAlwaysTrue()) {
+    OS << "  Runtime Assumptions:\n";
+    Assumptions.print(OS, 2);
+  }
 }
 
 // Returns NoAlias/MayAliass/MustAlias for two memory locations based upon their
@@ -3574,7 +3581,11 @@ bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA,
          Inv.invalidate<LoopAnalysis>(F, PA);
 }
 
-SCEVUnionPredicate DependenceInfo::getRuntimeAssumptions() {
+SCEVUnionPredicate DependenceInfo::getRuntimeAssumptions() const {
+  return SCEVUnionPredicate(Assumptions, *SE);
+}
+
+SCEVUnionPredicate Dependence::getRuntimeAssumptions() const {
   return SCEVUnionPredicate(Assumptions, *SE);
 }
 
@@ -3590,7 +3601,9 @@ SCEVUnionPredicate DependenceInfo::getRuntimeAssumptions() {
 // Care is required to keep the routine below, getSplitIteration(),
 // up to date with respect to this routine.
 std::unique_ptr<Dependence>
-DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
+DependenceInfo::depends(Instruction *Src, Instruction *Dst,
+                        bool UnderRuntimeAssumptions) {
+  SmallVector<const SCEVPredicate *, 4> Assume;
   bool PossiblyLoopIndependent = true;
   if (Src == Dst)
     PossiblyLoopIndependent = false;
@@ -3602,7 +3615,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
   if (!isLoadOrStore(Src) || !isLoadOrStore(Dst)) {
     // can only analyze simple loads and stores, i.e., no calls, invokes, etc.
     LLVM_DEBUG(dbgs() << "can only handle simple loads and stores\n");
-    return std::make_unique<Dependence>(Src, Dst);
+    return std::make_unique<Dependence>(Src, Dst, SE, Assume);
   }
 
   const MemoryLocation &DstLoc = MemoryLocation::get(Dst);
@@ -3613,7 +3626,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
   case AliasResult::PartialAlias:
     // cannot analyse objects if we don't understand their aliasing.
     LLVM_DEBUG(dbgs() << "can't analyze may or partial alias\n");
-    return std::make_unique<Dependence>(Src, Dst);
+    return std::make_unique<Dependence>(Src, Dst, SE, Assume);
   case AliasResult::NoAlias:
     // If the objects noalias, they are distinct, accesses are independent.
     LLVM_DEBUG(dbgs() << "no alias\n");
@@ -3626,7 +3639,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
     // The dependence test gets confused if the size of the memory accesses
     // differ.
     LLVM_DEBUG(dbgs() << "can't analyze must alias with different sizes\n");
-    return std::make_unique<Dependence>(Src, Dst);
+    return std::make_unique<Dependence>(Src, Dst, SE, Assume);
   }
 
   Value *SrcPtr = getLoadStorePointerOperand(Src);
@@ -3645,7 +3658,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
     // We check this upfront so we don't crash in cases where getMinusSCEV()
     // returns a SCEVCouldNotCompute.
     LLVM_DEBUG(dbgs() << "can't analyze SCEV with different pointer base\n");
-    return std::make_unique<Dependence>(Src, Dst);
+    return std::make_unique<Dependence>(Src, Dst, SE, Assume);
   }
 
   uint64_t EltSize = SrcLoc.Size.toRaw();
@@ -3656,18 +3669,28 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
 
   if (Src != Dst) {
     // Check that memory access offsets are multiples of element sizes.
-    if (!SE->isKnownMultipleOf(SrcEv, EltSize, Assumptions) ||
-        !SE->isKnownMultipleOf(DstEv, EltSize, Assumptions)) {
+    if (!SE->isKnownMultipleOf(SrcEv, EltSize, Assume) ||
+        !SE->isKnownMultipleOf(DstEv, EltSize, Assume)) {
       LLVM_DEBUG(dbgs() << "can't analyze SCEV with different offsets\n");
-      return std::make_unique<Dependence>(Src, Dst);
+      return std::make_unique<Dependence>(Src, Dst, SE, Assume);
     }
   }
 
+  if (Assumptions.empty())
+    Assumptions.append(Assume.begin(), Assume.end());
+  else
+    // Add non-redundant assumptions.
+    for (auto *P : Assume)
+      for (auto *A : Assumptions)
+        if (!A->implies(P, *SE))
+          Assumptions.push_back(P);
+
   establishNestingLevels(Src, Dst);
   LLVM_DEBUG(dbgs() << "    common nesting levels = " << CommonLevels << "\n");
   LLVM_DEBUG(dbgs() << "    maximum nesting levels = " << MaxLevels << "\n");
 
-  FullDependence Result(Src, Dst, PossiblyLoopIndependent, CommonLevels);
+  FullDependence Result(Src, Dst, SE, Assume, PossiblyLoopIndependent,
+                        CommonLevels);
   ++TotalArrayPairs;
 
   unsigned Pairs = 1;
@@ -4065,7 +4088,7 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep,
   // establish loop nesting levels
   establishNestingLevels(Src, Dst);
 
-  FullDependence Result(Src, Dst, false, CommonLevels);
+  FullDependence Result(Src, Dst, SE, Dep.Assumptions, false, CommonLevels);
 
   unsigned Pairs = 1;
   SmallVector<Subscript, 2> Pair(Pairs);

llvm/test/Analysis/DependenceAnalysis/DifferentOffsets.ll

@@ -30,6 +30,8 @@ define i32 @alias_with_parametric_offset(ptr nocapture %A, i64 %n) {
 ; CHECK-NEXT:    da analyze - none!
 ; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx, align 1 --> Dst: %0 = load i32, ptr %A, align 1
 ; CHECK-NEXT:    da analyze - flow [|<]!
+; CHECK-NEXT:    Runtime Assumptions:
+; CHECK-NEXT:    Equal predicate: (zext i2 (trunc i64 %n to i2) to i64) == 0
 ; CHECK-NEXT:  Src: %0 = load i32, ptr %A, align 1 --> Dst: %0 = load i32, ptr %A, align 1
 ; CHECK-NEXT:    da analyze - none!
 ; CHECK-NEXT:  Runtime Assumptions:
@@ -48,6 +50,9 @@ define i32 @alias_with_parametric_expr(ptr nocapture %A, i64 %n, i64 %m) {
 ; CHECK-NEXT:    da analyze - none!
 ; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx, align 1 --> Dst: %0 = load i32, ptr %arrayidx1, align 1
 ; CHECK-NEXT:    da analyze - flow [|<]!
+; CHECK-NEXT:    Runtime Assumptions:
+; CHECK-NEXT:    Equal predicate: (zext i2 ((trunc i64 %m to i2) + (-2 * (trunc i64 %n to i2))) to i64) == 0
+; CHECK-NEXT:    Equal predicate: (zext i2 (-2 + (trunc i64 %m to i2)) to i64) == 0
 ; CHECK-NEXT:  Src: %0 = load i32, ptr %arrayidx1, align 1 --> Dst: %0 = load i32, ptr %arrayidx1, align 1
 ; CHECK-NEXT:    da analyze - none!
 ; CHECK-NEXT:  Runtime Assumptions:

llvm/test/Analysis/DependenceAnalysis/MIVCheckConst.ll

@@ -42,6 +42,9 @@ define void @test(ptr %A, ptr %B, i1 %arg, i32 %n, i32 %m) #0 align 2 {
 ; CHECK-NEXT:    da analyze - consistent input [0 S S]!
 ; CHECK-NEXT:  Src: %v27 = load <32 x i32>, ptr %v25, align 256 --> Dst: %v32 = load <32 x i32>, ptr %v30, align 128
 ; CHECK-NEXT:    da analyze - input [* S S|<]!
+; CHECK-NEXT:    Runtime Assumptions:
+; CHECK-NEXT:    Equal predicate: (zext i7 (4 * (trunc i32 %v1 to i7) * (1 + (trunc i32 %n to i7))) to i32) == 0
+; CHECK-NEXT:    Equal predicate: (8 * (zext i4 (trunc i32 %v1 to i4) to i32))<nuw><nsw> == 0
 ; CHECK-NEXT:  Src: %v32 = load <32 x i32>, ptr %v30, align 128 --> Dst: %v32 = load <32 x i32>, ptr %v30, align 128
 ; CHECK-NEXT:    da analyze - consistent input [0 S S]!
 ; CHECK-NEXT:  Runtime Assumptions: