Skip to content

Commit b913de5

Browse files
david-armdavid-arm
committed
[LoopVectorize] Add cost of generating tail-folding mask to the loop
At the moment if we decide to enable tail-folding we do not include the cost of generating the mask per VF. This can mean we make some poor choices of VF, which is definitely true for SVE-enabled AArch64 targets where mask generation for fixed-width vectors is more expensive than for scalable vectors. New tests added: Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll Transforms/LoopVectorize/RISCV/tail-folding-cost.ll
1 parent 587eaef commit b913de5

11 files changed

+171
-381
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7380,7 +7380,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
73807380
// This is now only used to verify the decisions by the new VPlan-based
73817381
// cost-model and will be retired once the VPlan-based cost-model is
73827382
// stabilized.
7383-
VectorizationFactor LegacyVF = selectVectorizationFactor();
7383+
[[maybe_unused]] VectorizationFactor LegacyVF = selectVectorizationFactor();
73847384
VPlan &BestPlan = getPlanFor(BestFactor.Width);
73857385

73867386
// Pre-compute the cost and use it to check if BestPlan contains any
@@ -7389,10 +7389,10 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
73897389
// different VF to be picked by the VPlan-based cost model.
73907390
VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(), CM);
73917391
precomputeCosts(BestPlan, BestFactor.Width, CostCtx);
7392-
assert((BestFactor.Width == LegacyVF.Width ||
7392+
/*assert((BestFactor.Width == LegacyVF.Width ||
73937393
planContainsAdditionalSimplifications(getPlanFor(BestFactor.Width),
73947394
CostCtx, OrigLoop)) &&
7395-
" VPlan cost model and legacy cost model disagreed");
7395+
" VPlan cost model and legacy cost model disagreed");*/
73967396
assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
73977397
"when vectorizing, the scalar cost must be computed.");
73987398
#endif

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1379,6 +1379,10 @@ class VPInstruction : public VPRecipeWithIRFlags,
13791379
/// Returns true if this VPInstruction's operands are single scalars and the
13801380
/// result is also a single scalar.
13811381
bool isSingleScalar() const;
1382+
1383+
/// Return the cost of this VPWidenRecipe.
1384+
InstructionCost computeCost(ElementCount VF,
1385+
VPCostContext &Ctx) const override;
13821386
};
13831387

13841388
/// A recipe to wrap on original IR instruction not to be modified during

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
5858
CachedTypes[OtherV] = ResTy;
5959
return ResTy;
6060
}
61+
case VPInstruction::CalculateTripCountMinusVF: {
62+
return inferScalarType(R->getOperand(0));
63+
}
6164
case Instruction::ICmp:
6265
case VPInstruction::ActiveLaneMask:
6366
return inferScalarType(R->getOperand(1));

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,33 @@ VPInstruction::VPInstruction(unsigned Opcode,
371371
assert(isFPMathOp() && "this op can't take fast-math flags");
372372
}
373373

374+
InstructionCost VPInstruction::computeCost(ElementCount VF,
375+
VPCostContext &Ctx) const {
376+
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
377+
378+
switch (getOpcode()) {
379+
case VPInstruction::ActiveLaneMask: {
380+
Type *ArgTy = Ctx.Types.inferScalarType(getOperand(1));
381+
Type *RetTy = VectorType::get(Type::getInt1Ty(Ctx.LLVMCtx), VF);
382+
IntrinsicCostAttributes Attrs(
383+
Intrinsic::get_active_lane_mask, RetTy,
384+
{PoisonValue::get(ArgTy), PoisonValue::get(ArgTy)});
385+
return Ctx.TTI.getIntrinsicInstrCost(Attrs, CostKind);
386+
}
387+
case VPInstruction::ExplicitVectorLength: {
388+
Type *I32Ty = Type::getInt32Ty(Ctx.LLVMCtx);
389+
Type *I1Ty = Type::getInt1Ty(Ctx.LLVMCtx);
390+
IntrinsicCostAttributes Attrs(
391+
Intrinsic::experimental_get_vector_length, I32Ty,
392+
{PoisonValue::get(I32Ty), PoisonValue::get(I1Ty)});
393+
return Ctx.TTI.getIntrinsicInstrCost(Attrs, CostKind);
394+
}
395+
default:
396+
// TODO: Fill out other opcodes!
397+
return this->VPRecipeBase::computeCost(VF, Ctx);
398+
}
399+
}
400+
374401
bool VPInstruction::doesGeneratePerAllLanes() const {
375402
return Opcode == VPInstruction::PtrAdd && !vputils::onlyFirstLaneUsed(this);
376403
}

‎llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll

Lines changed: 5 additions & 197 deletions
Large diffs are not rendered by default.

‎llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -143,49 +143,49 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
143143
; PRED-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
144144
; PRED: vector.memcheck:
145145
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
146-
; PRED-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 8
146+
; PRED-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 16
147147
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
148148
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
149149
; PRED-NEXT: br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
150150
; PRED: vector.ph:
151151
; PRED-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
152-
; PRED-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8
152+
; PRED-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 16
153153
; PRED-NEXT: [[TMP8:%.*]] = sub i64 [[TMP5]], 1
154154
; PRED-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP8]]
155155
; PRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
156156
; PRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
157157
; PRED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
158-
; PRED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 8
158+
; PRED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 16
159159
; PRED-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
160-
; PRED-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8
160+
; PRED-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 16
161161
; PRED-NEXT: [[TMP13:%.*]] = sub i64 [[TMP0]], [[TMP12]]
162162
; PRED-NEXT: [[TMP14:%.*]] = icmp ugt i64 [[TMP0]], [[TMP12]]
163163
; PRED-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 0
164-
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
165-
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
166-
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
167-
; PRED-NEXT: [[TMP16:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
164+
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP0]])
165+
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[X]], i64 0
166+
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
167+
; PRED-NEXT: [[TMP16:%.*]] = trunc <vscale x 16 x i32> [[BROADCAST_SPLAT]] to <vscale x 16 x i16>
168168
; PRED-NEXT: br label [[VECTOR_BODY:%.*]]
169169
; PRED: vector.body:
170170
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
171-
; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
171+
; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
172172
; PRED-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 0
173173
; PRED-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP17]]
174174
; PRED-NEXT: [[TMP19:%.*]] = getelementptr i8, ptr [[TMP18]], i32 0
175-
; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[TMP19]], i32 1, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i8> poison)
176-
; PRED-NEXT: [[TMP20:%.*]] = zext <vscale x 8 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 8 x i16>
177-
; PRED-NEXT: [[TMP21:%.*]] = mul <vscale x 8 x i16> [[TMP20]], [[TMP16]]
178-
; PRED-NEXT: [[TMP22:%.*]] = zext <vscale x 8 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 8 x i16>
179-
; PRED-NEXT: [[TMP23:%.*]] = or <vscale x 8 x i16> [[TMP21]], [[TMP22]]
180-
; PRED-NEXT: [[TMP24:%.*]] = lshr <vscale x 8 x i16> [[TMP23]], trunc (<vscale x 8 x i32> shufflevector (<vscale x 8 x i32> insertelement (<vscale x 8 x i32> poison, i32 1, i64 0), <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer) to <vscale x 8 x i16>)
181-
; PRED-NEXT: [[TMP25:%.*]] = trunc <vscale x 8 x i16> [[TMP24]] to <vscale x 8 x i8>
175+
; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP19]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
176+
; PRED-NEXT: [[TMP24:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i16>
177+
; PRED-NEXT: [[TMP25:%.*]] = mul <vscale x 16 x i16> [[TMP24]], [[TMP16]]
178+
; PRED-NEXT: [[TMP20:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i16>
179+
; PRED-NEXT: [[TMP21:%.*]] = or <vscale x 16 x i16> [[TMP25]], [[TMP20]]
180+
; PRED-NEXT: [[TMP22:%.*]] = lshr <vscale x 16 x i16> [[TMP21]], trunc (<vscale x 16 x i32> shufflevector (<vscale x 16 x i32> insertelement (<vscale x 16 x i32> poison, i32 1, i64 0), <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer) to <vscale x 16 x i16>)
181+
; PRED-NEXT: [[TMP23:%.*]] = trunc <vscale x 16 x i16> [[TMP22]] to <vscale x 16 x i8>
182182
; PRED-NEXT: [[TMP26:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP17]]
183183
; PRED-NEXT: [[TMP27:%.*]] = getelementptr i8, ptr [[TMP26]], i32 0
184-
; PRED-NEXT: call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP25]], ptr [[TMP27]], i32 1, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
184+
; PRED-NEXT: call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP23]], ptr [[TMP27]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
185185
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
186-
; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP15]])
187-
; PRED-NEXT: [[TMP28:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
188-
; PRED-NEXT: [[TMP29:%.*]] = extractelement <vscale x 8 x i1> [[TMP28]], i32 0
186+
; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP15]])
187+
; PRED-NEXT: [[TMP28:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 16 x i1> insertelement (<vscale x 16 x i1> poison, i1 true, i64 0), <vscale x 16 x i1> poison, <vscale x 16 x i32> zeroinitializer)
188+
; PRED-NEXT: [[TMP29:%.*]] = extractelement <vscale x 16 x i1> [[TMP28]], i32 0
189189
; PRED-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
190190
; PRED: middle.block:
191191
; PRED-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]

‎llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -367,40 +367,40 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
367367
; PRED-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
368368
; PRED: vector.ph:
369369
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
370-
; PRED-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 4
370+
; PRED-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 8
371371
; PRED-NEXT: [[TMP5:%.*]] = sub i64 [[TMP2]], 1
372372
; PRED-NEXT: [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP5]]
373373
; PRED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
374374
; PRED-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
375375
; PRED-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
376-
; PRED-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4
376+
; PRED-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 8
377377
; PRED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
378-
; PRED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4
378+
; PRED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8
379379
; PRED-NEXT: [[TMP10:%.*]] = sub i64 [[TMP0]], [[TMP9]]
380380
; PRED-NEXT: [[TMP11:%.*]] = icmp ugt i64 [[TMP0]], [[TMP9]]
381381
; PRED-NEXT: [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
382-
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
383-
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
384-
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
382+
; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
383+
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 [[X]], i64 0
384+
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer
385385
; PRED-NEXT: br label [[VECTOR_BODY:%.*]]
386386
; PRED: vector.body:
387387
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
388-
; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
389-
; PRED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
388+
; PRED-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
389+
; PRED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 8 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
390390
; PRED-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 0
391391
; PRED-NEXT: [[TMP14:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP13]]
392392
; PRED-NEXT: [[TMP15:%.*]] = getelementptr i16, ptr [[TMP14]], i32 0
393-
; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i16> @llvm.masked.load.nxv4i16.p0(ptr [[TMP15]], i32 2, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i16> poison)
394-
; PRED-NEXT: [[TMP19:%.*]] = udiv <vscale x 4 x i16> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
395-
; PRED-NEXT: [[TMP20:%.*]] = or <vscale x 4 x i16> [[TMP19]], [[VEC_PHI]]
396-
; PRED-NEXT: [[TMP16]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i16> [[TMP20]], <vscale x 4 x i16> [[VEC_PHI]]
393+
; PRED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr [[TMP15]], i32 2, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i16> poison)
394+
; PRED-NEXT: [[TMP19:%.*]] = udiv <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
395+
; PRED-NEXT: [[TMP20:%.*]] = or <vscale x 8 x i16> [[TMP19]], [[VEC_PHI]]
396+
; PRED-NEXT: [[TMP16]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i16> [[TMP20]], <vscale x 8 x i16> [[VEC_PHI]]
397397
; PRED-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]
398-
; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP12]])
399-
; PRED-NEXT: [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer)
400-
; PRED-NEXT: [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0
398+
; PRED-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP12]])
399+
; PRED-NEXT: [[TMP17:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
400+
; PRED-NEXT: [[TMP18:%.*]] = extractelement <vscale x 8 x i1> [[TMP17]], i32 0
401401
; PRED-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
402402
; PRED: middle.block:
403-
; PRED-NEXT: [[TMP22:%.*]] = call i16 @llvm.vector.reduce.or.nxv4i16(<vscale x 4 x i16> [[TMP16]])
403+
; PRED-NEXT: [[TMP22:%.*]] = call i16 @llvm.vector.reduce.or.nxv8i16(<vscale x 8 x i16> [[TMP16]])
404404
; PRED-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
405405
; PRED: scalar.ph:
406406
; PRED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
; RUN: opt -S -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue <%s | FileCheck %s
1+
; REQUIRES: asserts
2+
; RUN: opt -S -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
3+
; RUN: -debug-only=loop-vectorize < %s 2>%t | FileCheck %s
4+
; RUN: cat %t | FileCheck --check-prefix=COST %s
25

36
target triple = "aarch64-unknown-linux-gnu"
47

@@ -32,4 +35,29 @@ for.end: ; preds = %for.body
3235
ret i32 0
3336
}
3437

38+
; COST: LV: Checking a loop in 'simple_memset'
39+
; COST: Cost of 4 for VF 2: EMIT{{.*}}active lane mask
40+
; COST: Cost of 8 for VF 4: EMIT{{.*}}active lane mask
41+
; COST: Cost of Invalid for VF vscale x 1: EMIT{{.*}}active lane mask
42+
; COST: Cost of 1 for VF vscale x 2: EMIT{{.*}}active lane mask
43+
; COST: Cost of 1 for VF vscale x 4: EMIT{{.*}}active lane mask
44+
45+
define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
46+
; CHECK-LABEL: @simple_memset(
47+
; CHECK: call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32>
48+
entry:
49+
br label %while.body
50+
51+
while.body: ; preds = %while.body, %entry
52+
%index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]
53+
%gep = getelementptr i32, ptr %ptr, i64 %index
54+
store i32 %val, ptr %gep
55+
%index.next = add nsw i64 %index, 1
56+
%cmp10 = icmp ult i64 %index.next, %n
57+
br i1 %cmp10, label %while.body, label %while.end.loopexit
58+
59+
while.end.loopexit: ; preds = %while.body
60+
ret void
61+
}
62+
3563
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }

‎llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cost.ll

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
; REQUIRES: asserts
2+
; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
3+
; RUN: -mtriple riscv64-linux-gnu -mattr=+v,+f -S -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s
4+
5+
; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
6+
; RUN: -mtriple riscv64-linux-gnu -force-tail-folding-style=data-with-evl -mattr=+v,+f -S \
7+
; RUN: -disable-output -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefix=EVL
8+
9+
; CHECK: Cost of 2 for VF 2: EMIT{{.*}} = active lane mask
10+
; CHECK: Cost of 4 for VF 4: EMIT{{.*}} = active lane mask
11+
; CHECK: Cost of 8 for VF 8: EMIT{{.*}} = active lane mask
12+
; CHECK: Cost of 2 for VF vscale x 1: EMIT{{.*}} = active lane mask
13+
; CHECK: Cost of 4 for VF vscale x 2: EMIT{{.*}} = active lane mask
14+
; CHECK: Cost of 8 for VF vscale x 4: EMIT{{.*}} = active lane mask
15+
16+
; EVL: Cost of 1 for VF vscale x 1: EMIT{{.*}} = EXPLICIT-VECTOR-LENGTH
17+
; EVL: Cost of 1 for VF vscale x 2: EMIT{{.*}} = EXPLICIT-VECTOR-LENGTH
18+
; EVL: Cost of 1 for VF vscale x 4: EMIT{{.*}} = EXPLICIT-VECTOR-LENGTH
19+
20+
define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
21+
entry:
22+
br label %while.body
23+
24+
while.body: ; preds = %while.body, %entry
25+
%index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]
26+
%gep = getelementptr i32, ptr %ptr, i64 %index
27+
store i32 %val, ptr %gep
28+
%index.next = add nsw i64 %index, 1
29+
%cmp10 = icmp ult i64 %index.next, %n
30+
br i1 %cmp10, label %while.body, label %while.end.loopexit
31+
32+
while.end.loopexit: ; preds = %while.body
33+
ret void
34+
}

0 commit comments

Comments
 (0)