!fixup address latest comments, thanks!

fhahn · fhahn · commit 4abc317c150b · 2024-08-14T21:05:40.000+01:00
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -166,8 +166,8 @@ class VPBuilder {
                             std::initializer_list<VPValue *> Operands,
                             DebugLoc DL = {}, const Twine &Name = "",
                             FastMathFlags FMFs = {}) {
-    auto *Op = new VPInstruction(Opcode, Operands, FMFs, DL, Name);
-    return tryInsertInstruction(Op);
+    return tryInsertInstruction(
+        new VPInstruction(Opcode, Operands, FMFs, DL, Name));
   }
 
   VPValue *createNot(VPValue *Operand, DebugLoc DL = {},
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7392,8 +7392,8 @@ LoopVectorizationPlanner::executePlan(
       "expanded SCEVs to reuse can only be used during epilogue vectorization");
   (void)IsEpilogueVectorization;
 
-  VPlanTransforms::interleave(BestVPlan, BestUF,
-                              OrigLoop->getHeader()->getModule()->getContext());
+  VPlanTransforms::interleaveByUF(
+      BestVPlan, BestUF, OrigLoop->getHeader()->getModule()->getContext());
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
 
   LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << BestVF
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -572,8 +572,7 @@ VPBasicBlock *VPBasicBlock::splitAt(iterator SplitAt) {
   return SplitBlock;
 }
 
-VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() {
-  VPRegionBlock *P = getParent();
+template <typename T> static T *getEnclosingLoopRegionImpl(T *P) {
   if (P && P->isReplicator()) {
     P = P->getParent();
     assert(!cast<VPRegionBlock>(P)->isReplicator() &&
@@ -582,6 +581,14 @@ VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() {
   return P;
 }
 
+const VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() const {
+  return getEnclosingLoopRegionImpl(getParent());
+}
+
+VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() {
+  return getEnclosingLoopRegionImpl(getParent());
+}
+
 static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
   if (VPBB->empty()) {
     assert(
@@ -933,7 +940,7 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
       createStepForVF(Builder, TripCountV->getType(), State.VF, State.UF));
   if (VF.getNumUsers() > 0) {
     VF.setUnderlyingValue(
-        createStepForVF(Builder, TripCountV->getType(), State.VF, 1));
+        getRuntimeVF(Builder, TripCountV->getType(), State.VF));
   }
 
   // When vectorizing the epilogue loop, the canonical induction start value
@@ -1053,10 +1060,12 @@ void VPlan::execute(VPTransformState *State) {
       // Move the last step to the end of the latch block. This ensures
       // consistent placement of all induction updates.
       Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
+      Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+
+      // When the VPlan has been unrolled, chain together the steps of the
+      // unrolled parts together.
       if (isa<VPWidenIntOrFpInductionRecipe>(&R) && R.getNumOperands() == 4)
         Inc->setOperand(0, State->get(R.getOperand(3), 0));
-
-      Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
       continue;
     }
 
@@ -1427,7 +1436,8 @@ void VPlanIngredient::print(raw_ostream &O) const {
 template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);
 
 bool VPValue::isDefinedOutsideVectorRegions() const {
-  return !hasDefiningRecipe() || !getDefiningRecipe()->getParent()->getParent();
+  return !hasDefiningRecipe() ||
+         !getDefiningRecipe()->getParent()->getEnclosingLoopRegion();
 }
 
 void VPValue::replaceAllUsesWith(VPValue *New) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -535,6 +535,7 @@ class VPBlockBase {
   VPBlocksTy &getSuccessors() { return Successors; }
 
   iterator_range<VPBlockBase **> successors() { return Successors; }
+  iterator_range<VPBlockBase **> predecessors() { return Predecessors; }
 
   const VPBlocksTy &getPredecessors() const { return Predecessors; }
   VPBlocksTy &getPredecessors() { return Predecessors; }
@@ -1400,7 +1401,7 @@ class VPInstruction : public VPRecipeWithIRFlags {
   /// result is also a single scalar.
   bool isSingleScalar() const;
 
-  /// Return the interleave count from the VPInstruction's last argument.
+  /// Return the interleave count from VPInstruction's last operand.
   unsigned getInterleaveCount() const;
 };
 
@@ -1690,7 +1691,7 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
                                      isInBounds(), getDebugLoc());
   }
 
-  /// Return the current part for this vector pointer.
+  /// Return the part associated with this vector pointer.
   unsigned getPartForRecipe() const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2034,7 +2035,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe {
   /// Returns true, if the phi is part of an in-loop reduction.
   bool isInLoop() const { return IsInLoop; }
 
-  /// Return the current part for this scalar step.
+  /// Return the part associated with this reduction phi.
   unsigned getPartForRecipe() const;
 };
 
@@ -2746,9 +2747,6 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
   /// Generate the canonical scalar induction phi of the vector loop.
   void execute(VPTransformState &State) override;
 
-  /// Return the current part for this scalar step.
-  unsigned getPartForRecipe() const;
-
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
@@ -2873,7 +2871,7 @@ class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe {
   /// step = <VF*UF, VF*UF, ..., VF*UF>.
   void execute(VPTransformState &State) override;
 
-  /// Return the current part for this scalar step.
+  /// Return the part associated with this widened IV.
   unsigned getPartForRecipe() const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -2989,7 +2987,7 @@ class VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
     return true;
   }
 
-  /// Return the current part for this scalar step.
+  /// Return the part associated with this scalar step
   unsigned getPartForRecipe() const;
 };
 
@@ -3093,6 +3091,7 @@ class VPBasicBlock : public VPBlockBase {
   VPBasicBlock *splitAt(iterator SplitAt);
 
   VPRegionBlock *getEnclosingLoopRegion();
+  const VPRegionBlock *getEnclosingLoopRegion() const;
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
@@ -3315,6 +3314,7 @@ class VPlan {
   /// Represents the loop-invariant VF * UF of the vector loop region.
   VPValue VFxUF;
 
+  /// Represents the loop-invariant VF of the vector loop region.
   VPValue VF;
 
   /// Holds a mapping between Values and their corresponding VPValue inside
@@ -3620,6 +3620,19 @@ class VPBlockUtils {
     connectBlocks(BlockPtr, NewBlock);
   }
 
+  static void insertBlockBefore(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
+    assert(NewBlock->getSuccessors().empty() &&
+           NewBlock->getPredecessors().empty() &&
+           "Can't insert new block with predecessors or successors.");
+    NewBlock->setParent(BlockPtr->getParent());
+    SmallVector<VPBlockBase *> Preds(BlockPtr->predecessors());
+    for (VPBlockBase *Pred : Preds) {
+      disconnectBlocks(Pred, BlockPtr);
+      connectBlocks(Pred, NewBlock);
+    }
+    connectBlocks(NewBlock, BlockPtr);
+  }
+
   /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
   /// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p
   /// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr
@@ -3850,25 +3863,36 @@ inline bool isUniformAfterVectorization(const VPValue *VPV) {
 /// Return true if \p V is a header mask in \p Plan.
 bool isHeaderMask(const VPValue *V, VPlan &Plan);
 
-/// Checks if \p C is uniform across all VFs and UFs. It is considered as such
-/// if it is either defined outside the vector region or its operand is known to
-/// be uniform across all VFs and UFs (e.g. VPDerivedIV or VPCanonicalIVPHI).
+/// Checks if \p C is uniform across all VF lanes and UF parts. It is considered
+/// as such if it is either loop invariant (defined outside the vector region)
+/// or its operand is known to be uniform across all VFs and UFs (e.g.
+/// VPDerivedIV or VPCanonicalIVPHI).
 inline bool isUniformAcrossVFsAndUFs(VPValue *V) {
-  if (V->isLiveIn())
+  // Loop invariants are uniform:
+  if (V->isDefinedOutsideVectorRegions())
     return true;
-  if (isa<VPCanonicalIVPHIRecipe, VPDerivedIVRecipe, VPExpandSCEVRecipe>(V))
+
+  auto *R = V->getDefiningRecipe();
+  // Canonical IV chain is uniform:
+  auto *CanonicalIV = R->getParent()->getPlan()->getCanonicalIV();
+  if (R == CanonicalIV || V == CanonicalIV->getBackedgeValue())
     return true;
-  auto *R = cast<VPSingleDefRecipe>(V->getDefiningRecipe());
-  if (R == R->getParent()->getPlan()->getCanonicalIV()->getBackedgeValue())
+
+  // DerivedIV is uniform:
+  if (isa<VPDerivedIVRecipe>(R))
     return true;
+
+  // Loads and stores that are uniform across VF lanes are handled by
+  // VPReplicateRecipe.IsUniform. They are also uniform across UF parts if all
+  // their operands are invariant:
   if (isa<VPReplicateRecipe>(V) && cast<VPReplicateRecipe>(V)->isUniform() &&
       (isa<LoadInst, StoreInst>(V->getUnderlyingValue())) &&
-      all_of(V->getDefiningRecipe()->operands(),
+      all_of(R->operands(),
              [](VPValue *Op) { return Op->isDefinedOutsideVectorRegions(); }))
     return true;
 
   return isa<VPScalarCastRecipe, VPWidenCastRecipe>(R) &&
-         (R->isDefinedOutsideVectorRegions() || R->getOperand(0)->isLiveIn() ||
+         (R->getOperand(0)->isLiveIn() ||
           isa<VPDerivedIVRecipe>(R->getOperand(0)) ||
           isa<VPCanonicalIVPHIRecipe>(R->getOperand(0)));
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -544,8 +544,6 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
     return CondBr;
   }
   case VPInstruction::ComputeReductionResult: {
-    unsigned NumParts = getNumOperands() - 1;
-
     // FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
     // and will be removed by breaking up the recipe further.
     auto *PhiR = cast<VPReductionPHIRecipe>(getOperand(0));
@@ -556,8 +554,11 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
     RecurKind RK = RdxDesc.getRecurrenceKind();
 
     Type *PhiTy = OrigPhi->getType();
+    // The recipe's operands are the reduction phi, followed by one operand for
+    // each part of the reduction.
+    unsigned NumParts = getNumOperands() - 1;
     VectorParts RdxParts(NumParts);
-    for (unsigned Part = 0; Part != NumParts; ++Part)
+    for (unsigned Part = 0; Part < NumParts; ++Part)
       RdxParts[Part] = State.get(getOperand(1 + Part), 0, PhiR->isInLoop());
 
     // If the vector reduction can be performed in a smaller type, we truncate
@@ -688,6 +689,9 @@ bool VPInstruction::isSingleScalar() const {
 }
 
 unsigned VPInstruction::getInterleaveCount() const {
+  assert((getOpcode() == VPInstruction::CalculateTripCountMinusVF ||
+          getOpcode() == VPInstruction::CanonicalIVIncrementForPart) &&
+         "used with unexpected opcode");
   return getNumOperands() == 1
              ? 1
              : cast<ConstantInt>(getOperand(1)->getLiveInIRValue())
@@ -1313,7 +1317,6 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
 
   Value *SplatVF;
   if (getNumOperands() == 4) {
-    // Need to create stuff in PH.
     SplatVF = State.get(getOperand(2), 0);
   } else {
     // Multiply the vectorization factor by the step using integer or
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
