Skip to content

Commit 6fe32b9

Browse files
fhahnfhahn
committed
[LV] Add support for cmp reductions with decreasing IVs.
Similar to FindLastIV, add FindFirstIV to support select (icmp(), x, y) reductions where one of x or y is a decreasing induction. This is done via two new recurrence kinds FindFirstIV(UMin|SMin), which selects the first value from the reduction vector using umin/smin instead of the last value (FindLastIV). It uses unsigned max/signed max as sentinel value. The SMin version is used if the IV range does not include signed max. Otherwise UMin is used if the IV range does not include signed max. Alternatively we could consolidate FindFirstIV/FindLastIV in a single FindIV kind, and add a field indicating if whether we want to select the first or last value.
1 parent 7204141 commit 6fe32b9

File tree

12 files changed

+515
-113
lines changed

12 files changed

+515
-113
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,13 @@ enum class RecurKind {
5656
FindLastIV, ///< FindLast reduction with select(cmp(),x,y) where one of
5757
///< (x,y) is increasing loop induction, and both x and y are
5858
///< integer type.
59+
FindFirstIVUMin, /// FindFirst reduction with select(icmp(),x,y) where one of
60+
///< (x,y) is a decreasing loop induction, and both x and y
61+
///< are integer type.
62+
FindFirstIVSMin /// FindFirst reduction with select(icmp(),x,y) where one of
63+
///< (x,y) is a decreasing loop induction, and both x and y
64+
///< are integer type.
65+
5966
// clang-format on
6067
// TODO: Any_of and FindLast reduction need not be restricted to integer type
6168
// only.
@@ -160,12 +167,13 @@ class RecurrenceDescriptor {
160167
/// Returns a struct describing whether the instruction is either a
161168
/// Select(ICmp(A, B), X, Y), or
162169
/// Select(FCmp(A, B), X, Y)
163-
/// where one of (X, Y) is an increasing loop induction variable, and the
164-
/// other is a PHI value.
170+
/// where one of (X, Y) is an increasing (FindLast) or decreasing (FindFirst)
171+
/// loop induction variable, and the other is a PHI value.
165172
// TODO: Support non-monotonic variable. FindLast does not need be restricted
166173
// to increasing loop induction variables.
167-
static InstDesc isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
168-
Instruction *I, ScalarEvolution &SE);
174+
static InstDesc isFindIVPattern(RecurKind Kind, Loop *TheLoop,
175+
PHINode *OrigPhi, Instruction *I,
176+
ScalarEvolution &SE);
169177

170178
/// Returns a struct describing if the instruction is a
171179
/// Select(FCmp(X, Y), (Z = X op PHINode), PHINode) instruction pattern.
@@ -259,17 +267,33 @@ class RecurrenceDescriptor {
259267
return Kind == RecurKind::FindLastIV;
260268
}
261269

270+
/// Returns true if the recurrence kind is of the form
271+
/// select(cmp(),x,y) where one of (x,y) is an increasing or decreasing loop
272+
/// induction.
273+
static bool isFindIVRecurrenceKind(RecurKind Kind) {
274+
return Kind == RecurKind::FindLastIV ||
275+
Kind == RecurKind::FindFirstIVUMin ||
276+
Kind == RecurKind::FindFirstIVSMin;
277+
}
278+
262279
/// Returns the type of the recurrence. This type can be narrower than the
263280
/// actual type of the Phi if the recurrence has been type-promoted.
264281
Type *getRecurrenceType() const { return RecurrenceType; }
265282

266-
/// Returns the sentinel value for FindLastIV recurrences to replace the start
267-
/// value.
283+
/// Returns the sentinel value for FindFirstIV &FindLastIV recurrences to
284+
/// replace the start value.
268285
Value *getSentinelValue() const {
269-
assert(isFindLastIVRecurrenceKind(Kind) && "Unexpected recurrence kind");
270286
Type *Ty = StartValue->getType();
271-
return ConstantInt::get(Ty,
272-
APInt::getSignedMinValue(Ty->getIntegerBitWidth()));
287+
if (isFindLastIVRecurrenceKind(Kind)) {
288+
return ConstantInt::get(
289+
Ty, APInt::getSignedMinValue(Ty->getIntegerBitWidth()));
290+
} else if (Kind == RecurKind::FindFirstIVSMin) {
291+
return ConstantInt::get(
292+
Ty, APInt::getSignedMaxValue(Ty->getIntegerBitWidth()));
293+
} else {
294+
assert(Kind == RecurKind::FindFirstIVUMin);
295+
return ConstantInt::get(Ty, APInt::getMaxValue(Ty->getIntegerBitWidth()));
296+
}
273297
}
274298

275299
/// Returns a reference to the instructions used for type-promoting the

‎llvm/lib/Analysis/IVDescriptors.cpp

Lines changed: 52 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
5151
case RecurKind::UMin:
5252
case RecurKind::AnyOf:
5353
case RecurKind::FindLastIV:
54+
case RecurKind::FindFirstIVUMin:
55+
case RecurKind::FindFirstIVSMin:
5456
return true;
5557
}
5658
return false;
@@ -683,8 +685,9 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
683685
// value of the data type or a non-constant value by using mask and multiple
684686
// reduction operations.
685687
RecurrenceDescriptor::InstDesc
686-
RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
687-
Instruction *I, ScalarEvolution &SE) {
688+
RecurrenceDescriptor::isFindIVPattern(RecurKind Kind, Loop *TheLoop,
689+
PHINode *OrigPhi, Instruction *I,
690+
ScalarEvolution &SE) {
688691
// TODO: Support the vectorization of FindLastIV when the reduction phi is
689692
// used by more than one select instruction. This vectorization is only
690693
// performed when the SCEV of each increasing induction variable used by the
@@ -700,7 +703,7 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
700703
m_Value(NonRdxPhi)))))
701704
return InstDesc(false, I);
702705

703-
auto IsIncreasingLoopInduction = [&](Value *V) {
706+
auto IsSupportedLoopInduction = [&](Value *V, RecurKind Kind) {
704707
Type *Ty = V->getType();
705708
if (!SE.isSCEVable(Ty))
706709
return false;
@@ -710,21 +713,39 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
710713
return false;
711714

712715
const SCEV *Step = AR->getStepRecurrence(SE);
713-
if (!SE.isKnownPositive(Step))
716+
if (Kind == RecurKind::FindFirstIVUMin ||
717+
Kind == RecurKind::FindFirstIVSMin) {
718+
if (!SE.isKnownNegative(Step))
719+
return false;
720+
} else if (!SE.isKnownPositive(Step))
714721
return false;
715722

716723
const ConstantRange IVRange = SE.getSignedRange(AR);
717724
unsigned NumBits = Ty->getIntegerBitWidth();
718-
// Keep the minimum value of the recurrence type as the sentinel value.
719-
// The maximum acceptable range for the increasing induction variable,
720-
// called the valid range, will be defined as
725+
// Keep the minimum (FindLast) or maximum (FindFirst) value of the
726+
// recurrence type as the sentinel value. The maximum acceptable range for
727+
// the induction variable, called the valid range, will be defined as
721728
// [<sentinel value> + 1, <sentinel value>)
722-
// where <sentinel value> is SignedMin(<recurrence type>)
729+
// where <sentinel value> is SignedMin(<recurrence type>) for FindLast or
730+
// UnsignedMax(<recurrence type>) for FindFirst.
723731
// TODO: This range restriction can be lifted by adding an additional
724732
// virtual OR reduction.
725-
const APInt Sentinel = APInt::getSignedMinValue(NumBits);
726-
const ConstantRange ValidRange =
727-
ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
733+
const APInt Sentinel = Kind == RecurKind::FindFirstIVUMin
734+
? APInt::getMaxValue(NumBits)
735+
: (Kind == RecurKind::FindFirstIVSMin
736+
? APInt::getSignedMaxValue(NumBits)
737+
: APInt::getSignedMinValue(NumBits));
738+
ConstantRange ValidRange = ConstantRange::getEmpty(NumBits);
739+
if (Kind == RecurKind::FindFirstIVSMin)
740+
ValidRange =
741+
ConstantRange::getNonEmpty(APInt::getSignedMinValue(NumBits),
742+
APInt::getSignedMaxValue(NumBits) - 1);
743+
else {
744+
const APInt Sentinel = Kind == RecurKind::FindFirstIVUMin
745+
? APInt::getMaxValue(NumBits)
746+
: APInt::getSignedMinValue(NumBits);
747+
ValidRange = ConstantRange::getNonEmpty(Sentinel + 1, Sentinel);
748+
}
728749
LLVM_DEBUG(dbgs() << "LV: FindLastIV valid range is " << ValidRange
729750
<< ", and the signed range of " << *AR << " is "
730751
<< IVRange << "\n");
@@ -736,11 +757,18 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi,
736757
// We are looking for selects of the form:
737758
// select(cmp(), phi, increasing_loop_induction) or
738759
// select(cmp(), increasing_loop_induction, phi)
739-
// TODO: Support for monotonically decreasing induction variable
740-
if (!IsIncreasingLoopInduction(NonRdxPhi))
760+
if (Kind == RecurKind::FindLastIV) {
761+
if (IsSupportedLoopInduction(NonRdxPhi, Kind))
762+
return InstDesc(I, Kind);
741763
return InstDesc(false, I);
764+
}
765+
766+
if (IsSupportedLoopInduction(NonRdxPhi, RecurKind::FindFirstIVSMin))
767+
return InstDesc(I, RecurKind::FindFirstIVSMin);
768+
if (IsSupportedLoopInduction(NonRdxPhi, RecurKind::FindFirstIVUMin))
769+
return InstDesc(I, RecurKind::FindFirstIVUMin);
742770

743-
return InstDesc(I, RecurKind::FindLastIV);
771+
return InstDesc(false, I);
744772
}
745773

746774
RecurrenceDescriptor::InstDesc
@@ -875,8 +903,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
875903
if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
876904
Kind == RecurKind::Add || Kind == RecurKind::Mul)
877905
return isConditionalRdxPattern(Kind, I);
878-
if (isFindLastIVRecurrenceKind(Kind) && SE)
879-
return isFindLastIVPattern(L, OrigPhi, I, *SE);
906+
if (isFindIVRecurrenceKind(Kind) && SE)
907+
return isFindIVPattern(Kind, L, OrigPhi, I, *SE);
880908
[[fallthrough]];
881909
case Instruction::FCmp:
882910
case Instruction::ICmp:
@@ -990,6 +1018,12 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
9901018
LLVM_DEBUG(dbgs() << "Found a FindLastIV reduction PHI." << *Phi << "\n");
9911019
return true;
9921020
}
1021+
if (AddReductionVar(Phi, RecurKind::FindFirstIVUMin, TheLoop, FMF, RedDes, DB,
1022+
AC, DT, SE)) {
1023+
LLVM_DEBUG(dbgs() << "Found a FindFirstV reduction PHI." << *Phi << "\n");
1024+
return true;
1025+
}
1026+
9931027
if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT,
9941028
SE)) {
9951029
LLVM_DEBUG(dbgs() << "Found an FMult reduction PHI." << *Phi << "\n");
@@ -1153,6 +1187,8 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
11531187
case RecurKind::SMin:
11541188
case RecurKind::UMax:
11551189
case RecurKind::UMin:
1190+
case RecurKind::FindFirstIVUMin:
1191+
case RecurKind::FindFirstIVSMin:
11561192
return Instruction::ICmp;
11571193
case RecurKind::FMax:
11581194
case RecurKind::FMin:

‎llvm/lib/Transforms/Utils/LoopUtils.cpp

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1244,12 +1244,16 @@ Value *llvm::createAnyOfReduction(IRBuilderBase &Builder, Value *Src,
12441244
Value *llvm::createFindLastIVReduction(IRBuilderBase &Builder, Value *Src,
12451245
Value *Start,
12461246
const RecurrenceDescriptor &Desc) {
1247-
assert(RecurrenceDescriptor::isFindLastIVRecurrenceKind(
1248-
Desc.getRecurrenceKind()) &&
1249-
"Unexpected reduction kind");
1247+
assert(
1248+
RecurrenceDescriptor::isFindIVRecurrenceKind(Desc.getRecurrenceKind()) &&
1249+
"Unexpected reduction kind");
12501250
Value *Sentinel = Desc.getSentinelValue();
12511251
Value *MaxRdx = Src->getType()->isVectorTy()
1252-
? Builder.CreateIntMaxReduce(Src, true)
1252+
? (Desc.getRecurrenceKind() == RecurKind::FindLastIV
1253+
? Builder.CreateIntMaxReduce(Src, true)
1254+
: Builder.CreateIntMinReduce(
1255+
Src, Desc.getRecurrenceKind() ==
1256+
RecurKind::FindFirstIVSMin))
12531257
: Src;
12541258
// Correct the final reduction result back to the start value if the maximum
12551259
// reduction is sentinel value.
@@ -1345,8 +1349,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
13451349
Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src,
13461350
RecurKind Kind) {
13471351
assert(!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
1348-
!RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
1349-
"AnyOf or FindLastIV reductions are not supported.");
1352+
!RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
1353+
"AnyOf, FindFirstIV and FindLastIV reductions are not supported.");
13501354
Intrinsic::ID Id = getReductionIntrinsicID(Kind);
13511355
auto *SrcTy = cast<VectorType>(Src->getType());
13521356
Type *SrcEltTy = SrcTy->getElementType();

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5176,7 +5176,7 @@ LoopVectorizationCostModel::selectInterleaveCount(VPlan &Plan, ElementCount VF,
51765176
const RecurrenceDescriptor &RdxDesc = Reduction.second;
51775177
RecurKind RK = RdxDesc.getRecurrenceKind();
51785178
return RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
5179-
RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK);
5179+
RecurrenceDescriptor::isFindIVRecurrenceKind(RK);
51805180
});
51815181
if (HasSelectCmpReductions) {
51825182
LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
@@ -7549,7 +7549,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
75497549
auto *EpiRedResult = dyn_cast<VPInstruction>(R);
75507550
if (!EpiRedResult ||
75517551
(EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
7552-
EpiRedResult->getOpcode() != VPInstruction::ComputeFindLastIVResult))
7552+
EpiRedResult->getOpcode() != VPInstruction::ComputeFindIVResult))
75537553
return;
75547554

75557555
auto *EpiRedHeaderPhi =
@@ -7567,7 +7567,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
75677567
"AnyOf expected to start by comparing main resume value to original "
75687568
"start value");
75697569
MainResumeValue = Cmp->getOperand(0);
7570-
} else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
7570+
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(
75717571
RdxDesc.getRecurrenceKind())) {
75727572
using namespace llvm::PatternMatch;
75737573
Value *Cmp, *OrigResumeV, *CmpOp;
@@ -9360,7 +9360,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
93609360
RecurKind Kind = RdxDesc.getRecurrenceKind();
93619361
assert(
93629362
!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) &&
9363-
!RecurrenceDescriptor::isFindLastIVRecurrenceKind(Kind) &&
9363+
!RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) &&
93649364
"AnyOf and FindLast reductions are not allowed for in-loop reductions");
93659365

93669366
// Collect the chain of "link" recipes for the reduction starting at PhiR.
@@ -9517,7 +9517,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
95179517
(cast<VPInstruction>(&U)->getOpcode() ==
95189518
VPInstruction::ComputeReductionResult ||
95199519
cast<VPInstruction>(&U)->getOpcode() ==
9520-
VPInstruction::ComputeFindLastIVResult);
9520+
VPInstruction::ComputeFindIVResult);
95219521
});
95229522
if (CM.usePredicatedReductionSelect())
95239523
PhiR->setOperand(1, NewExitingVPV);
@@ -9562,11 +9562,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
95629562
VPInstruction *FinalReductionResult;
95639563
VPBuilder::InsertPointGuard Guard(Builder);
95649564
Builder.setInsertPoint(MiddleVPBB, IP);
9565-
if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
9565+
if (RecurrenceDescriptor::isFindIVRecurrenceKind(
95669566
RdxDesc.getRecurrenceKind())) {
95679567
VPValue *Start = PhiR->getStartValue();
95689568
FinalReductionResult =
9569-
Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
9569+
Builder.createNaryOp(VPInstruction::ComputeFindIVResult,
95709570
{PhiR, Start, NewExitingVPV}, ExitDL);
95719571
} else {
95729572
FinalReductionResult = Builder.createNaryOp(
@@ -9613,11 +9613,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
96139613
continue;
96149614
}
96159615

9616-
if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
9616+
if (RecurrenceDescriptor::isFindIVRecurrenceKind(
96179617
RdxDesc.getRecurrenceKind())) {
9618-
// Adjust the start value for FindLastIV recurrences to use the sentinel
9619-
// value after generating the ResumePhi recipe, which uses the original
9620-
// start value.
9618+
// Adjust the start value for FindFirstIV/FindLastIV recurrences to use
9619+
// the sentinel value after generating the ResumePhi recipe, which uses
9620+
// the original start value.
96219621
PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()));
96229622
}
96239623
}
@@ -9985,18 +9985,18 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
99859985
VPlanTransforms::runPass(VPlanTransforms::removeDeadRecipes, MainPlan);
99869986

99879987
using namespace VPlanPatternMatch;
9988-
// When vectorizing the epilogue, FindLastIV reductions can introduce multiple
9989-
// uses of undef/poison. If the reduction start value may be undef or poison
9990-
// it needs to be frozen and the frozen start has to be used when computing
9991-
// the reduction result. We also need to use the frozen value in the resume
9992-
// phi generated by the main vector loop, as this is also used to compute the
9993-
// reduction result after the epilogue vector loop.
9988+
// When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
9989+
// introduce multiple uses of undef/poison. If the reduction start value may
9990+
// be undef or poison it needs to be frozen and the frozen start has to be
9991+
// used when computing the reduction result. We also need to use the frozen
9992+
// value in the resume phi generated by the main vector loop, as this is also
9993+
// used to compute the reduction result after the epilogue vector loop.
99949994
auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
99959995
bool UpdateResumePhis) {
99969996
VPBuilder Builder(Plan.getEntry());
99979997
for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
99989998
auto *VPI = dyn_cast<VPInstruction>(&R);
9999-
if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindLastIVResult)
9999+
if (!VPI || VPI->getOpcode() != VPInstruction::ComputeFindIVResult)
1000010000
continue;
1000110001
VPValue *OrigStart = VPI->getOperand(1);
1000210002
if (isGuaranteedNotToBeUndefOrPoison(OrigStart->getLiveInIRValue()))
@@ -10107,17 +10107,17 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
1010710107
IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
1010810108
ResumeV =
1010910109
Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue());
10110-
} else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
10110+
} else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) {
1011110111
ToFrozen[RdxDesc.getRecurrenceStartValue()] =
1011210112
cast<PHINode>(ResumeV)->getIncomingValueForBlock(
1011310113
EPI.MainLoopIterationCountCheck);
1011410114

10115-
// VPReductionPHIRecipe for FindLastIV reductions requires an adjustment
10116-
// to the resume value. The resume value is adjusted to the sentinel
10117-
// value when the final value from the main vector loop equals the start
10118-
// value. This ensures correctness when the start value might not be
10119-
// less than the minimum value of a monotonically increasing induction
10120-
// variable.
10115+
// VPReductionPHIRecipe for FindFirstIV/FindLastIV reductions requires
10116+
// an adjustment to the resume value. The resume value is adjusted to
10117+
// the sentinel value when the final value from the main vector loop
10118+
// equals the start value. This ensures correctness when the start value
10119+
// might not be less than the minimum value of a monotonically
10120+
// increasing induction variable.
1012110121
BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
1012210122
IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
1012310123
Value *Cmp = Builder.CreateICmpEQ(

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23092,6 +23092,8 @@ class HorizontalReduction {
2309223092
case RecurKind::FMulAdd:
2309323093
case RecurKind::AnyOf:
2309423094
case RecurKind::FindLastIV:
23095+
case RecurKind::FindFirstIVUMin:
23096+
case RecurKind::FindFirstIVSMin:
2309523097
case RecurKind::FMaximumNum:
2309623098
case RecurKind::FMinimumNum:
2309723099
case RecurKind::None:
@@ -23226,6 +23228,8 @@ class HorizontalReduction {
2322623228
case RecurKind::FMulAdd:
2322723229
case RecurKind::AnyOf:
2322823230
case RecurKind::FindLastIV:
23231+
case RecurKind::FindFirstIVUMin:
23232+
case RecurKind::FindFirstIVSMin:
2322923233
case RecurKind::FMaximumNum:
2323023234
case RecurKind::FMinimumNum:
2323123235
case RecurKind::None:
@@ -23325,6 +23329,8 @@ class HorizontalReduction {
2332523329
case RecurKind::FMulAdd:
2332623330
case RecurKind::AnyOf:
2332723331
case RecurKind::FindLastIV:
23332+
case RecurKind::FindFirstIVUMin:
23333+
case RecurKind::FindFirstIVSMin:
2332823334
case RecurKind::FMaximumNum:
2332923335
case RecurKind::FMinimumNum:
2333023336
case RecurKind::None:

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -898,7 +898,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
898898
BranchOnCount,
899899
BranchOnCond,
900900
Broadcast,
901-
ComputeFindLastIVResult,
901+
ComputeFindIVResult,
902902
ComputeReductionResult,
903903
// Extracts the last lane from its operand if it is a vector, or the last
904904
// part if scalar. In the latter case, the recipe will be removed during

0 commit comments

Comments
 (0)