Skip to content

Commit 2acecfe

Browse files
committed
[VPlan] Use VPBBs to look up masks for newly created recipes (NFC).
Update recipe construction to use VPBBs to look up masks, in preparation for #128420.
1 parent 3e393d9 commit 2acecfe

File tree

1 file changed

+9
-10
lines changed

1 file changed

+9
-10
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8418,7 +8418,7 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
84188418

84198419
VPValue *Mask = nullptr;
84208420
if (Legal->isMaskRequired(I))
8421-
Mask = getBlockInMask(I->getParent());
8421+
Mask = getBlockInMask(Builder.getInsertBlock());
84228422

84238423
// Determine if the pointer operand of the access is either consecutive or
84248424
// reverse consecutive.
@@ -8645,7 +8645,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
86458645
// all-true mask.
86468646
VPValue *Mask = nullptr;
86478647
if (Legal->isMaskRequired(CI))
8648-
Mask = getBlockInMask(CI->getParent());
8648+
Mask = getBlockInMask(Builder.getInsertBlock());
86498649
else
86508650
Mask = Plan.getOrAddLiveIn(
86518651
ConstantInt::getTrue(IntegerType::getInt1Ty(CI->getContext())));
@@ -8687,7 +8687,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
86878687
// div/rem operation itself. Otherwise fall through to general handling below.
86888688
if (CM.isPredicatedInst(I)) {
86898689
SmallVector<VPValue *> Ops(Operands);
8690-
VPValue *Mask = getBlockInMask(I->getParent());
8690+
VPValue *Mask = getBlockInMask(Builder.getInsertBlock());
86918691
VPValue *One =
86928692
Plan.getOrAddLiveIn(ConstantInt::get(I->getType(), 1u, false));
86938693
auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, I->getDebugLoc());
@@ -8769,7 +8769,7 @@ VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
87698769
// In case of predicated execution (due to tail-folding, or conditional
87708770
// execution, or both), pass the relevant mask.
87718771
if (Legal->isMaskRequired(HI->Store))
8772-
HGramOps.push_back(getBlockInMask(HI->Store->getParent()));
8772+
HGramOps.push_back(getBlockInMask(Builder.getInsertBlock()));
87738773

87748774
return new VPHistogramRecipe(Opcode, HGramOps, HI->Store->getDebugLoc());
87758775
}
@@ -8823,7 +8823,7 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
88238823
// added initially. Masked replicate recipes will later be placed under an
88248824
// if-then construct to prevent side-effects. Generate recipes to compute
88258825
// the block mask for this region.
8826-
BlockInMask = getBlockInMask(I->getParent());
8826+
BlockInMask = getBlockInMask(Builder.getInsertBlock());
88278827
}
88288828

88298829
// Note that there is some custom logic to mark some intrinsics as uniform
@@ -9067,7 +9067,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(Instruction *Reduction,
90679067
ReductionOpcode == Instruction::Sub) &&
90689068
"Expected an ADD or SUB operation for predicated partial "
90699069
"reductions (because the neutral element in the mask is zero)!");
9070-
Cond = getBlockInMask(Reduction->getParent());
9070+
Cond = getBlockInMask(Builder.getInsertBlock());
90719071
VPValue *Zero =
90729072
Plan.getOrAddLiveIn(ConstantInt::get(Reduction->getType(), 0));
90739073
BinOp = Builder.createSelect(Cond, BinOp, Zero, Reduction->getDebugLoc());
@@ -9847,10 +9847,9 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
98479847
"PreviousLink must be the operand other than VecOp");
98489848
}
98499849

9850-
BasicBlock *BB = CurrentLinkI->getParent();
98519850
VPValue *CondOp = nullptr;
9852-
if (CM.blockNeedsPredicationForAnyReason(BB))
9853-
CondOp = RecipeBuilder.getBlockInMask(BB);
9851+
if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent()))
9852+
CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent());
98549853

98559854
// Non-FP RdxDescs will have all fast math flags set, so clear them.
98569855
FastMathFlags FMFs = isa<FPMathOperator>(CurrentLinkI)
@@ -9893,7 +9892,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
98939892
// different numbers of lanes. Partial reductions mask the input instead.
98949893
if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
98959894
!isa<VPPartialReductionRecipe>(OrigExitingVPV->getDefiningRecipe())) {
9896-
VPValue *Cond = RecipeBuilder.getBlockInMask(OrigLoop->getHeader());
9895+
VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
98979896
Type *PhiTy = PhiR->getOperand(0)->getLiveInIRValue()->getType();
98989897
std::optional<FastMathFlags> FMFs =
98999898
PhiTy->isFloatingPointTy()

0 commit comments

Comments
 (0)