[DA] handle memory accesses with different offsets and strides

llvm/include/llvm/Analysis/DependenceAnalysis.h

@@ -40,6 +40,7 @@
 #define LLVM_ANALYSIS_DEPENDENCEANALYSIS_H
 
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
@@ -49,8 +50,6 @@ namespace llvm {
   template <typename T> class ArrayRef;
   class Loop;
   class LoopInfo;
-  class ScalarEvolution;
-  class SCEV;
   class SCEVConstant;
   class raw_ostream;
 
@@ -74,8 +73,9 @@ namespace llvm {
     Dependence &operator=(Dependence &&) = default;
 
   public:
-    Dependence(Instruction *Source, Instruction *Destination)
-        : Src(Source), Dst(Destination) {}
+    Dependence(Instruction *Source, Instruction *Destination,
+               const SCEVUnionPredicate &A)
+        : Src(Source), Dst(Destination), Assumptions(A) {}
     virtual ~Dependence() = default;
 
     /// Dependence::DVEntry - Each level in the distance/direction vector
@@ -203,6 +203,10 @@ namespace llvm {
     /// field.
     void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; }
 
+    /// getRuntimeAssumptions - Returns the runtime assumptions under which this
+    /// Dependence relation is valid.
+    SCEVUnionPredicate getRuntimeAssumptions() const { return Assumptions; }
+
     /// dump - For debugging purposes, dumps a dependence to OS.
     ///
     void dump(raw_ostream &OS) const;
@@ -211,6 +215,7 @@ namespace llvm {
     Instruction *Src, *Dst;
 
   private:
+    SCEVUnionPredicate Assumptions;
     const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr;
     friend class DependenceInfo;
   };
@@ -225,8 +230,9 @@ namespace llvm {
   /// input dependences are unordered.
   class FullDependence final : public Dependence {
   public:
-    FullDependence(Instruction *Src, Instruction *Dst, bool LoopIndependent,
-                   unsigned Levels);
+    FullDependence(Instruction *Source, Instruction *Destination,
+                   const SCEVUnionPredicate &Assumes,
+                   bool PossiblyLoopIndependent, unsigned Levels);
 
     /// isLoopIndependent - Returns true if this is a loop-independent
     /// dependence.
@@ -302,9 +308,13 @@ namespace llvm {
 
     /// depends - Tests for a dependence between the Src and Dst instructions.
     /// Returns NULL if no dependence; otherwise, returns a Dependence (or a
-    /// FullDependence) with as much information as can be gleaned.
-    std::unique_ptr<Dependence> depends(Instruction *Src,
-                                        Instruction *Dst);
+    /// FullDependence) with as much information as can be gleaned. By default,
+    /// the dependence test collects a set of runtime assumptions that cannot be
+    /// solved at compilation time. By default UnderRuntimeAssumptions is false
+    /// for a safe approximation of the dependence relation that does not
+    /// require runtime checks.
+    std::unique_ptr<Dependence> depends(Instruction *Src, Instruction *Dst,
+                                        bool UnderRuntimeAssumptions = false);
 
     /// getSplitIteration - Give a dependence that's splittable at some
     /// particular level, return the iteration that should be used to split
@@ -350,11 +360,16 @@ namespace llvm {
 
     Function *getFunction() const { return F; }
 
+    /// getRuntimeAssumptions - Returns all the runtime assumptions under which
+    /// the dependence test is valid.
+    SCEVUnionPredicate getRuntimeAssumptions() const;
+
   private:
     AAResults *AA;
     ScalarEvolution *SE;
     LoopInfo *LI;
     Function *F;
+    SmallVector<const SCEVPredicate *, 4> Assumptions;
 
     /// Subscript - This private struct represents a pair of subscripts from
     /// a pair of potentially multi-dimensional array references. We use a

llvm/include/llvm/Analysis/ScalarEvolution.h

@@ -1044,6 +1044,13 @@ class ScalarEvolution {
   bool isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero = false,
                               bool OrNegative = false);
 
+  /// Check that \p S is a multiple of \p M. When \p S is an AddRecExpr, \p S is
+  /// a multiple of \p M if \p S starts with a multiple of \p M and at every
+  /// iteration step \p S only adds multiples of \p M. \p Assumptions records
+  /// the runtime predicates under which \p S is a multiple of \p M.
+  bool isKnownMultipleOf(const SCEV *S, uint64_t M,
+                         SmallVectorImpl<const SCEVPredicate *> &Assumptions);
+
   /// Splits SCEV expression \p S into two SCEVs. One of them is obtained from
   /// \p S by substitution of all AddRec sub-expression related to loop \p L
   /// with initial value of that SCEV. The second is obtained from \p S by

llvm/lib/Analysis/DependenceAnalysis.cpp

@@ -187,7 +187,8 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA,
         if (DstI->mayReadOrWriteMemory()) {
           OS << "Src:" << *SrcI << " --> Dst:" << *DstI << "\n";
           OS << "  da analyze - ";
-          if (auto D = DA->depends(&*SrcI, &*DstI)) {
+          if (auto D = DA->depends(&*SrcI, &*DstI,
+                                   /*UnderRuntimeAssumptions=*/true)) {
             // Normalize negative direction vectors if required by clients.
             if (NormalizeResults && D->normalize(&SE))
                 OS << "normalized - ";
@@ -199,13 +200,17 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA,
                 OS << "!\n";
               }
             }
-          }
-          else
+          } else
             OS << "none!\n";
         }
       }
     }
   }
+  SCEVUnionPredicate Assumptions = DA->getRuntimeAssumptions();
+  if (!Assumptions.isAlwaysTrue()) {
+    OS << "Runtime Assumptions:\n";
+    Assumptions.print(OS, 0);
+  }
 }
 
 void DependenceAnalysisWrapperPass::print(raw_ostream &OS,
@@ -264,9 +269,10 @@ bool Dependence::isScalar(unsigned level) const {
 // FullDependence methods
 
 FullDependence::FullDependence(Instruction *Source, Instruction *Destination,
+                               const SCEVUnionPredicate &Assumes,
                                bool PossiblyLoopIndependent,
                                unsigned CommonLevels)
-    : Dependence(Source, Destination), Levels(CommonLevels),
+    : Dependence(Source, Destination, Assumes), Levels(CommonLevels),
       LoopIndependent(PossiblyLoopIndependent) {
   Consistent = true;
   if (CommonLevels)
@@ -706,6 +712,12 @@ void Dependence::dump(raw_ostream &OS) const {
       OS << " splitable";
   }
   OS << "!\n";
+
+  SCEVUnionPredicate Assumptions = getRuntimeAssumptions();
+  if (!Assumptions.isAlwaysTrue()) {
+    OS << "  Runtime Assumptions:\n";
+    Assumptions.print(OS, 2);
+  }
 }
 
 // Returns NoAlias/MayAliass/MustAlias for two memory locations based upon their
@@ -3569,6 +3581,10 @@ bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA,
          Inv.invalidate<LoopAnalysis>(F, PA);
 }
 
+SCEVUnionPredicate DependenceInfo::getRuntimeAssumptions() const {
+  return SCEVUnionPredicate(Assumptions, *SE);
+}
+
 // depends -
 // Returns NULL if there is no dependence.
 // Otherwise, return a Dependence with as many details as possible.
@@ -3581,7 +3597,9 @@ bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA,
 // Care is required to keep the routine below, getSplitIteration(),
 // up to date with respect to this routine.
 std::unique_ptr<Dependence>
-DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
+DependenceInfo::depends(Instruction *Src, Instruction *Dst,
+                        bool UnderRuntimeAssumptions) {
+  SmallVector<const SCEVPredicate *, 4> Assume;
   bool PossiblyLoopIndependent = true;
   if (Src == Dst)
     PossiblyLoopIndependent = false;
@@ -3593,22 +3611,20 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
   if (!isLoadOrStore(Src) || !isLoadOrStore(Dst)) {
     // can only analyze simple loads and stores, i.e., no calls, invokes, etc.
     LLVM_DEBUG(dbgs() << "can only handle simple loads and stores\n");
-    return std::make_unique<Dependence>(Src, Dst);
+    return std::make_unique<Dependence>(Src, Dst,
+                                        SCEVUnionPredicate(Assume, *SE));
   }
 
-  assert(isLoadOrStore(Src) && "instruction is not load or store");
-  assert(isLoadOrStore(Dst) && "instruction is not load or store");
-  Value *SrcPtr = getLoadStorePointerOperand(Src);
-  Value *DstPtr = getLoadStorePointerOperand(Dst);
+  const MemoryLocation &DstLoc = MemoryLocation::get(Dst);
+  const MemoryLocation &SrcLoc = MemoryLocation::get(Src);
 
-  switch (underlyingObjectsAlias(AA, F->getDataLayout(),
-                                 MemoryLocation::get(Dst),
-                                 MemoryLocation::get(Src))) {
+  switch (underlyingObjectsAlias(AA, F->getDataLayout(), DstLoc, SrcLoc)) {
   case AliasResult::MayAlias:
   case AliasResult::PartialAlias:
     // cannot analyse objects if we don't understand their aliasing.
     LLVM_DEBUG(dbgs() << "can't analyze may or partial alias\n");
-    return std::make_unique<Dependence>(Src, Dst);
+    return std::make_unique<Dependence>(Src, Dst,
+                                        SCEVUnionPredicate(Assume, *SE));
   case AliasResult::NoAlias:
     // If the objects noalias, they are distinct, accesses are independent.
     LLVM_DEBUG(dbgs() << "no alias\n");
@@ -3617,30 +3633,75 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst) {
     break; // The underlying objects alias; test accesses for dependence.
   }
 
-  // establish loop nesting levels
-  establishNestingLevels(Src, Dst);
-  LLVM_DEBUG(dbgs() << "    common nesting levels = " << CommonLevels << "\n");
-  LLVM_DEBUG(dbgs() << "    maximum nesting levels = " << MaxLevels << "\n");
-
-  FullDependence Result(Src, Dst, PossiblyLoopIndependent, CommonLevels);
-  ++TotalArrayPairs;
+  if (DstLoc.Size != SrcLoc.Size || !DstLoc.Size.isPrecise() ||
+      !SrcLoc.Size.isPrecise()) {
+    // The dependence test gets confused if the size of the memory accesses
+    // differ.
+    LLVM_DEBUG(dbgs() << "can't analyze must alias with different sizes\n");
+    return std::make_unique<Dependence>(Src, Dst,
+                                        SCEVUnionPredicate(Assume, *SE));
+  }
 
-  unsigned Pairs = 1;
-  SmallVector<Subscript, 2> Pair(Pairs);
+  Value *SrcPtr = getLoadStorePointerOperand(Src);
+  Value *DstPtr = getLoadStorePointerOperand(Dst);
   const SCEV *SrcSCEV = SE->getSCEV(SrcPtr);
   const SCEV *DstSCEV = SE->getSCEV(DstPtr);
   LLVM_DEBUG(dbgs() << "    SrcSCEV = " << *SrcSCEV << "\n");
   LLVM_DEBUG(dbgs() << "    DstSCEV = " << *DstSCEV << "\n");
-  if (SE->getPointerBase(SrcSCEV) != SE->getPointerBase(DstSCEV)) {
+  const SCEV *SrcBase = SE->getPointerBase(SrcSCEV);
+  const SCEV *DstBase = SE->getPointerBase(DstSCEV);
+  if (SrcBase != DstBase) {
     // If two pointers have different bases, trying to analyze indexes won't
     // work; we can't compare them to each other. This can happen, for example,
     // if one is produced by an LCSSA PHI node.
     //
     // We check this upfront so we don't crash in cases where getMinusSCEV()
     // returns a SCEVCouldNotCompute.
     LLVM_DEBUG(dbgs() << "can't analyze SCEV with different pointer base\n");
-    return std::make_unique<Dependence>(Src, Dst);
+    return std::make_unique<Dependence>(Src, Dst,
+                                        SCEVUnionPredicate(Assume, *SE));
+  }
+
+  uint64_t EltSize = SrcLoc.Size.toRaw();
+  const SCEV *SrcEv = SE->getMinusSCEV(SrcSCEV, SrcBase);
+  const SCEV *DstEv = SE->getMinusSCEV(DstSCEV, DstBase);
+
+  if (Src != Dst) {
+    // Check that memory access offsets are multiples of element sizes.
+    if (!SE->isKnownMultipleOf(SrcEv, EltSize, Assume) ||
+        !SE->isKnownMultipleOf(DstEv, EltSize, Assume)) {
+      LLVM_DEBUG(dbgs() << "can't analyze SCEV with different offsets\n");
+      return std::make_unique<Dependence>(Src, Dst,
+                                          SCEVUnionPredicate(Assume, *SE));
+    }
+  }
+
+  if (!Assume.empty()) {
+    if (!UnderRuntimeAssumptions)
+      return std::make_unique<Dependence>(Src, Dst,
+                                          SCEVUnionPredicate(Assume, *SE));
+    // Add non-redundant assumptions.
+    unsigned N = Assumptions.size();
+    for (const SCEVPredicate *P : Assume) {
+      bool Implied = false;
+      for (unsigned I = 0; I != N && !Implied; I++)
+        if (Assumptions[I]->implies(P, *SE))
+          Implied = true;
+      if (!Implied)
+        Assumptions.push_back(P);
+    }
   }
+
+  establishNestingLevels(Src, Dst);
+  LLVM_DEBUG(dbgs() << "    common nesting levels = " << CommonLevels << "\n");
+  LLVM_DEBUG(dbgs() << "    maximum nesting levels = " << MaxLevels << "\n");
+
+  FullDependence Result(Src, Dst, SCEVUnionPredicate(Assume, *SE),
+                        PossiblyLoopIndependent, CommonLevels);
+  ++TotalArrayPairs;
+
+  unsigned Pairs = 1;
+  SmallVector<Subscript, 2> Pair(Pairs);
   Pair[0].Src = SrcSCEV;
   Pair[0].Dst = DstSCEV;
 
@@ -4034,7 +4095,7 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep,
   // establish loop nesting levels
   establishNestingLevels(Src, Dst);
 
-  FullDependence Result(Src, Dst, false, CommonLevels);
+  FullDependence Result(Src, Dst, Dep.Assumptions, false, CommonLevels);
 
   unsigned Pairs = 1;
   SmallVector<Subscript, 2> Pair(Pairs);

llvm/lib/Analysis/ScalarEvolution.cpp

@@ -10971,6 +10971,56 @@ bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
   return all_of(Mul->operands(), NonRecursive) && (OrZero || isKnownNonZero(S));
 }
 
+bool ScalarEvolution::isKnownMultipleOf(
+    const SCEV *S, uint64_t M,
+    SmallVectorImpl<const SCEVPredicate *> &Assumptions) {
+  if (M == 0)
+    return false;
+  if (M == 1)
+    return true;
+
+  // Recursively check AddRec operands. An AddRecExpr S is a multiple of M if S
+  // starts with a multiple of M and at every iteration step S only adds
+  // multiples of M.
+  if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S))
+    return isKnownMultipleOf(AddRec->getStart(), M, Assumptions) &&
+           isKnownMultipleOf(AddRec->getStepRecurrence(*this), M, Assumptions);
+
+  // For a constant, check that "S % M == 0".
+  if (auto *Cst = dyn_cast<SCEVConstant>(S)) {
+    APInt C = Cst->getAPInt();
+    return C.urem(M) == 0;
+  }
+
+  // TODO: Also check other SCEV expressions, i.e., SCEVAddRecExpr, etc.
+
+  // Basic tests have failed.
+  // Check "S % M == 0" at compile time and record runtime Assumptions.
+  auto *STy = dyn_cast<IntegerType>(S->getType());
+  const SCEV *SmodM =
+      getURemExpr(S, getConstant(ConstantInt::get(STy, M, false)));
+  const SCEV *Zero = getZero(STy);
+
+  // Check whether "S % M == 0" is known at compile time.
+  if (isKnownPredicate(ICmpInst::ICMP_EQ, SmodM, Zero))
+    return true;
+
+  // Check whether "S % M != 0" is known at compile time.
+  if (isKnownPredicate(ICmpInst::ICMP_NE, SmodM, Zero))
+    return false;
+
+  const SCEVPredicate *P = getComparePredicate(ICmpInst::ICMP_EQ, SmodM, Zero);
+
+  // Detect redundant predicates.
+  for (auto *A : Assumptions)
+    if (A->implies(P, *this))
+      return true;
+
+  // Only record non-redundant predicates.
+  Assumptions.push_back(P);
+  return true;
+}
+
 std::pair<const SCEV *, const SCEV *>
 ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
   // Compute SCEV on entry of loop L.

llvm/test/Analysis/DependenceAnalysis/DifferentAccessSize.ll

@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
+
+; The dependence test does not handle array accesses of different sizes: i32 and i64.
+; Bug 16183 - https://github.com/llvm/llvm-project/issues/16183
+
+define i64 @bug16183_alias(ptr nocapture %A) {
+; CHECK-LABEL: 'bug16183_alias'
+; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx, align 4 --> Dst: store i32 2, ptr %arrayidx, align 4
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx, align 4 --> Dst: %0 = load i64, ptr %A, align 8
+; CHECK-NEXT:    da analyze - confused!
+; CHECK-NEXT:  Src: %0 = load i64, ptr %A, align 8 --> Dst: %0 = load i64, ptr %A, align 8
+; CHECK-NEXT:    da analyze - none!
+;
+entry:
+  %arrayidx = getelementptr inbounds i32, ptr %A, i64 1
+  store i32 2, ptr %arrayidx, align 4
+  %0 = load i64, ptr %A, align 8
+  ret i64 %0
+}