Skip to content

Commit d267411

Browse files
committed
!fixup, address comments.
1 parent ce95f18 commit d267411

File tree

4 files changed

+37
-33
lines changed

4 files changed

+37
-33
lines changed

llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1311,8 +1311,8 @@ class VPWidenRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
13111311
Opcode(I.getOpcode()) {}
13121312

13131313
template <typename IterT>
1314-
VPWidenRecipe(unsigned VPDefOpcode, unsigned Opcode,
1315-
iterator_range<IterT> Operands, bool NUW, bool NSW, DebugLoc DL)
1314+
VPWidenRecipe(unsigned VPDefOpcode, unsigned Opcode, ArrayRef<IterT> Operands,
1315+
bool NUW, bool NSW, DebugLoc DL)
13161316
: VPRecipeWithIRFlags(VPDefOpcode, Operands, WrapFlagsTy(NUW, NSW), DL),
13171317
Opcode(Opcode) {}
13181318

@@ -1321,8 +1321,8 @@ class VPWidenRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
13211321
: VPWidenRecipe(VPDef::VPWidenSC, I, Operands) {}
13221322

13231323
template <typename IterT>
1324-
VPWidenRecipe(unsigned Opcode, iterator_range<IterT> Operands, bool NUW,
1325-
bool NSW, DebugLoc DL)
1324+
VPWidenRecipe(unsigned Opcode, ArrayRef<IterT> Operands, bool NUW, bool NSW,
1325+
DebugLoc DL)
13261326
: VPWidenRecipe(VPDef::VPWidenSC, Opcode, Operands, NUW, NSW, DL) {}
13271327

13281328
~VPWidenRecipe() override = default;
@@ -2614,9 +2614,10 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
26142614
/// concrete recipes before codegen. The operands are {ChainOp, VecOp,
26152615
/// [Condition]}.
26162616
class VPExtendedReductionRecipe : public VPReductionRecipe {
2617-
/// Opcode of the extend recipe will be lowered to.
2617+
/// Opcode of the extend for VecOp.
26182618
Instruction::CastOps ExtOp;
26192619

2620+
/// The scalar type after extending.
26202621
Type *ResultTy;
26212622

26222623
/// For cloning VPExtendedReductionRecipe.
@@ -2637,10 +2638,8 @@ class VPExtendedReductionRecipe : public VPReductionRecipe {
26372638
ExtOp(Ext->getOpcode()), ResultTy(Ext->getResultType()) {
26382639
assert((ExtOp == Instruction::CastOps::ZExt ||
26392640
ExtOp == Instruction::CastOps::SExt) &&
2640-
"VPExtendedReductionRecipe only support zext and sext.");
2641+
"VPExtendedReductionRecipe only supports zext and sext.");
26412642

2642-
// Not all WidenCastRecipes contain nneg flag. Need to transfer flags from
2643-
// the original recipe to prevent setting wrong flags.
26442643
transferFlags(*Ext);
26452644
setUnderlyingValue(R->getUnderlyingValue());
26462645
}
@@ -2670,7 +2669,7 @@ class VPExtendedReductionRecipe : public VPReductionRecipe {
26702669
/// Is the extend ZExt?
26712670
bool isZExt() const { return getExtOpcode() == Instruction::ZExt; }
26722671

2673-
/// The opcode of extend recipe.
2672+
/// Get the opcode of the extend for VecOp.
26742673
Instruction::CastOps getExtOpcode() const { return ExtOp; }
26752674
};
26762675

@@ -2680,12 +2679,13 @@ class VPExtendedReductionRecipe : public VPReductionRecipe {
26802679
/// recipe is abstract and needs to be lowered to concrete recipes before
26812680
/// codegen. The operands are {ChainOp, VecOp1, VecOp2, [Condition]}.
26822681
class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
2683-
/// Opcode of the extend recipe.
2682+
/// Opcode of the extend for VecOp1 and VecOp2.
26842683
Instruction::CastOps ExtOp;
26852684

26862685
/// Non-neg flag of the extend recipe.
26872686
bool IsNonNeg = false;
26882687

2688+
/// The scalar type after extending.
26892689
Type *ResultTy;
26902690

26912691
/// For cloning VPMulAccumulateReductionRecipe.
@@ -2716,7 +2716,7 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
27162716
"be Add");
27172717
assert((ExtOp == Instruction::CastOps::ZExt ||
27182718
ExtOp == Instruction::CastOps::SExt) &&
2719-
"VPMulAccumulateReductionRecipe only support zext and sext.");
2719+
"VPMulAccumulateReductionRecipe only supports zext and sext.");
27202720
setUnderlyingValue(R->getUnderlyingValue());
27212721
// Only set the non-negative flag if the original recipe contains.
27222722
if (Ext0->hasNonNegFlag())
@@ -2762,24 +2762,26 @@ class VPMulAccumulateReductionRecipe : public VPReductionRecipe {
27622762

27632763
Type *getResultType() const {
27642764
assert(isExtended() && "Only support getResultType when this recipe "
2765-
"contains implicit extend.");
2765+
"is implicitly extend.");
27662766
return ResultTy;
27672767
}
27682768

2769-
/// The VPValue of the vector value to be extended and reduced.
2769+
/// The first vector value to be extended and reduced.
27702770
VPValue *getVecOp0() const { return getOperand(1); }
2771+
2772+
/// The second vector value to be extended and reduced.
27712773
VPValue *getVecOp1() const { return getOperand(2); }
27722774

2773-
/// Return if this MulAcc recipe contains extended operands.
2775+
/// Return true if this recipe contains extended operands.
27742776
bool isExtended() const { return ExtOp != Instruction::CastOps::CastOpsEnd; }
27752777

27762778
/// Return the opcode of the extends for the operands.
27772779
Instruction::CastOps getExtOpcode() const { return ExtOp; }
27782780

2779-
/// Return if the operands are zero extended.
2781+
/// Return if the operands are zero-extended.
27802782
bool isZExt() const { return ExtOp == Instruction::CastOps::ZExt; }
27812783

2782-
/// Return the non negative flag of the ext recipe.
2784+
/// Return true if the operand extends have the non-negative flag.
27832785
bool isNonNeg() const { return IsNonNeg; }
27842786
};
27852787

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2602,7 +2602,8 @@ void VPExtendedReductionRecipe::print(raw_ostream &O, const Twine &Indent,
26022602
RecurrenceDescriptor::getOpcode(getRecurrenceKind()))
26032603
<< " (";
26042604
getVecOp()->printAsOperand(O, SlotTracker);
2605-
O << " extended to " << *getResultType();
2605+
printFlags(O);
2606+
O << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType();
26062607
if (isConditional()) {
26072608
O << ", ";
26082609
getCondOp()->printAsOperand(O, SlotTracker);
@@ -2627,12 +2628,14 @@ void VPMulAccumulateReductionRecipe::print(raw_ostream &O, const Twine &Indent,
26272628
O << "(";
26282629
getVecOp0()->printAsOperand(O, SlotTracker);
26292630
if (isExtended())
2630-
O << " extended to " << *getResultType() << "), (";
2631+
O << " " << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType()
2632+
<< "), (";
26312633
else
26322634
O << ", ";
26332635
getVecOp1()->printAsOperand(O, SlotTracker);
26342636
if (isExtended())
2635-
O << " extended to " << *getResultType() << ")";
2637+
O << " " << Instruction::getOpcodeName(ExtOp) << " to " << *getResultType()
2638+
<< ")";
26362639
if (isConditional()) {
26372640
O << ", ";
26382641
getCondOp()->printAsOperand(O, SlotTracker);

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2419,7 +2419,7 @@ static void expandVPExtendedReduction(VPExtendedReductionRecipe *ExtRed) {
24192419
static void
24202420
expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) {
24212421
// Generate inner VPWidenCastRecipes if necessary.
2422-
// Note that we will drop the extend after mul which transform
2422+
// Note that we will drop the extend after mul which transforms
24232423
// reduce.add(ext(mul(ext, ext))) to reduce.add(mul(ext, ext)).
24242424
VPValue *Op0, *Op1;
24252425
if (MulAcc->isExtended()) {
@@ -2454,9 +2454,8 @@ expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) {
24542454

24552455
std::array<VPValue *, 2> MulOps = {Op0, Op1};
24562456
auto *Mul = new VPWidenRecipe(
2457-
Instruction::Mul, make_range(MulOps.begin(), MulOps.end()),
2458-
MulAcc->hasNoUnsignedWrap(), MulAcc->hasNoSignedWrap(),
2459-
MulAcc->getDebugLoc());
2457+
Instruction::Mul, ArrayRef(MulOps), MulAcc->hasNoUnsignedWrap(),
2458+
MulAcc->hasNoSignedWrap(), MulAcc->getDebugLoc());
24602459
Mul->insertBefore(MulAcc);
24612460

24622461
auto *Red = new VPReductionRecipe(
@@ -2688,6 +2687,10 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
26882687
VPCostContext &Ctx, VFRange &Range) {
26892688
using namespace VPlanPatternMatch;
26902689

2690+
unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
2691+
if (Opcode != Instruction::Add)
2692+
return nullptr;
2693+
26912694
Type *RedTy = Ctx.Types.inferScalarType(Red);
26922695

26932696
// Clamp the range if using multiply-accumulate-reduction is profitable.
@@ -2718,21 +2721,17 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
27182721
Range);
27192722
};
27202723

2721-
unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
2722-
if (Opcode != Instruction::Add)
2723-
return nullptr;
2724-
27252724
VPValue *VecOp = Red->getVecOp();
27262725
VPValue *A, *B;
2727-
// Try to match reduce.add(mul(...))
2726+
// Try to match reduce.add(mul(...)).
27282727
if (match(VecOp, m_Mul(m_VPValue(A), m_VPValue(B)))) {
27292728
auto *RecipeA =
27302729
dyn_cast_if_present<VPWidenCastRecipe>(A->getDefiningRecipe());
27312730
auto *RecipeB =
27322731
dyn_cast_if_present<VPWidenCastRecipe>(B->getDefiningRecipe());
27332732
auto *Mul = cast<VPWidenRecipe>(VecOp->getDefiningRecipe());
27342733

2735-
// Match reduce.add(mul(ext, ext))
2734+
// Match reduce.add(mul(ext, ext)).
27362735
if (RecipeA && RecipeB &&
27372736
(RecipeA->getOpcode() == RecipeB->getOpcode() || A == B) &&
27382737
match(RecipeA, m_ZExtOrSExt(m_VPValue())) &&
@@ -2742,11 +2741,11 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
27422741
Mul, RecipeA, RecipeB, nullptr))
27432742
return new VPMulAccumulateReductionRecipe(Red, Mul, RecipeA, RecipeB,
27442743
RecipeA->getResultType());
2745-
// Match reduce.add(mul)
2744+
// Match reduce.add(mul).
27462745
if (IsMulAccValidAndClampRange(true, Mul, nullptr, nullptr, nullptr))
27472746
return new VPMulAccumulateReductionRecipe(Red, Mul);
27482747
}
2749-
// Match reduce.add(ext(mul(ext(A), ext(B))))
2748+
// Match reduce.add(ext(mul(ext(A), ext(B)))).
27502749
// All extend recipes must have same opcode or A == B
27512750
// which can be transform to reduce.add(zext(mul(sext(A), sext(B)))).
27522751
if (match(VecOp, m_ZExtOrSExt(m_Mul(m_ZExtOrSExt(m_VPValue()),

llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
288288
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[STEPS]]>
289289
; CHECK-NEXT: vp<[[ADDR:%.+]]> = vector-pointer ir<%arrayidx>
290290
; CHECK-NEXT: WIDEN ir<[[LOAD:%.+]]> = load vp<[[ADDR]]>
291-
; CHECK-NEXT: EXTENDED-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (ir<[[LOAD]]> extended to i64)
291+
; CHECK-NEXT: EXTENDED-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (ir<[[LOAD]]> zext to i64)
292292
; CHECK-NEXT: EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
293293
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
294294
; CHECK-NEXT: No successors
@@ -384,7 +384,7 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
384384
; CHECK-NEXT: CLONE ir<[[ARRAYIDX1:%.+]]> = getelementptr inbounds ir<%y>, vp<[[STEPS]]>
385385
; CHECK-NEXT: vp<[[ADDR1:%.+]]> = vector-pointer ir<[[ARRAYIDX1]]>
386386
; CHECK-NEXT: WIDEN ir<[[LOAD1:%.+]]> = load vp<[[ADDR1]]>
387-
; CHECK-NEXT: MULACC-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (mul nsw (ir<[[LOAD0]]> extended to i64), (ir<[[LOAD1]]> extended to i64))
387+
; CHECK-NEXT: MULACC-REDUCE ir<[[RDX_NEXT:%.+]]> = ir<[[RDX]]> + reduce.add (mul nsw (ir<[[LOAD0]]> sext to i64), (ir<[[LOAD1]]> sext to i64))
388388
; CHECK-NEXT: EMIT vp<[[IV_NEXT]]> = add nuw vp<[[IV]]>, vp<[[VFxUF]]>
389389
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
390390
; CHECK-NEXT: No successors

0 commit comments

Comments
 (0)