Skip to content

Commit 4eb9838

Browse files
fhahnfhahn
committed
[VPlan] Generalize VPValue::isDefinedOutsideLoopRegions.
Update isDefinedOutsideLoopRegions to check if a recipe is defined outside any region. Split off already approved #95842 now that this can be tested separately after landing VPlan-based LICM #107501
1 parent 1553714 commit 4eb9838

17 files changed

+186
-184
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -576,8 +576,9 @@ VPBasicBlock *VPBasicBlock::splitAt(iterator SplitAt) {
576576
return SplitBlock;
577577
}
578578

579-
VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() {
580-
VPRegionBlock *P = getParent();
579+
/// Return the enclosing loop region for region \p P. The templated version is
580+
/// used to support both const and non-const block arguments.
581+
template <typename T> static T *getEnclosingLoopRegionForRegion(T *P) {
581582
if (P && P->isReplicator()) {
582583
P = P->getParent();
583584
assert(!cast<VPRegionBlock>(P)->isReplicator() &&
@@ -586,6 +587,14 @@ VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() {
586587
return P;
587588
}
588589

590+
VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() {
591+
return getEnclosingLoopRegionForRegion(getParent());
592+
}
593+
594+
const VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() const {
595+
return getEnclosingLoopRegionForRegion(getParent());
596+
}
597+
589598
static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
590599
if (VPBB->empty()) {
591600
assert(
@@ -1458,6 +1467,11 @@ void VPlanIngredient::print(raw_ostream &O) const {
14581467

14591468
#endif
14601469

1470+
bool VPValue::isDefinedOutsideLoopRegions() const {
1471+
return !hasDefiningRecipe() ||
1472+
!getDefiningRecipe()->getParent()->getEnclosingLoopRegion();
1473+
}
1474+
14611475
void VPValue::replaceAllUsesWith(VPValue *New) {
14621476
replaceUsesWithIf(New, [](VPUser &, unsigned) { return true; });
14631477
}

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3199,6 +3199,7 @@ class VPBasicBlock : public VPBlockBase {
31993199
VPBasicBlock *splitAt(iterator SplitAt);
32003200

32013201
VPRegionBlock *getEnclosingLoopRegion();
3202+
const VPRegionBlock *getEnclosingLoopRegion() const;
32023203

32033204
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
32043205
/// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p

‎llvm/lib/Transforms/Vectorize/VPlanValue.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -180,10 +180,8 @@ class VPValue {
180180
return getUnderlyingValue();
181181
}
182182

183-
/// Returns true if the VPValue is defined outside any loop region, i.e. it
184-
/// is a live-in value.
185-
/// TODO: Also handle recipes defined in pre-header blocks.
186-
bool isDefinedOutsideLoopRegions() const { return !hasDefiningRecipe(); }
183+
/// Returns true if the VPValue is defined outside any loop region.
184+
bool isDefinedOutsideLoopRegions() const;
187185

188186
// Set \p Val as the underlying Value of this VPValue.
189187
void setUnderlyingValue(Value *Val) {

‎llvm/test/Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll

Lines changed: 49 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -58,30 +58,30 @@ define void @test_pr25490(i32 %n, ptr noalias nocapture %a, ptr noalias nocaptur
5858
; CHECK-NEXT: [[N_VEC5:%.*]] = and i64 [[TMP0]], 4294967288
5959
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
6060
; CHECK: vec.epilog.vector.body:
61-
; CHECK-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
62-
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX7]]
63-
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i8>, ptr [[TMP14]], align 1
64-
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX7]]
65-
; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i8>, ptr [[TMP15]], align 1
66-
; CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i8> [[WIDE_LOAD9]] to <8 x i16>
67-
; CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i8> [[WIDE_LOAD8]] to <8 x i16>
61+
; CHECK-NEXT: [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
62+
; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[C]], i64 [[INDEX6]]
63+
; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <8 x i8>, ptr [[TMP14]], align 1
64+
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX6]]
65+
; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i8>, ptr [[TMP15]], align 1
66+
; CHECK-NEXT: [[TMP16:%.*]] = zext <8 x i8> [[WIDE_LOAD8]] to <8 x i16>
67+
; CHECK-NEXT: [[TMP17:%.*]] = zext <8 x i8> [[WIDE_LOAD7]] to <8 x i16>
6868
; CHECK-NEXT: [[TMP18:%.*]] = mul nuw <8 x i16> [[TMP16]], [[TMP17]]
6969
; CHECK-NEXT: [[TMP19:%.*]] = lshr <8 x i16> [[TMP18]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
7070
; CHECK-NEXT: [[TMP20:%.*]] = trunc nuw <8 x i16> [[TMP19]] to <8 x i8>
7171
; CHECK-NEXT: store <8 x i8> [[TMP20]], ptr [[TMP15]], align 1
72-
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX7]]
73-
; CHECK-NEXT: [[WIDE_LOAD10:%.*]] = load <8 x i8>, ptr [[TMP21]], align 1
74-
; CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i8> [[WIDE_LOAD10]] to <8 x i16>
72+
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX6]]
73+
; CHECK-NEXT: [[WIDE_LOAD9:%.*]] = load <8 x i8>, ptr [[TMP21]], align 1
74+
; CHECK-NEXT: [[TMP22:%.*]] = zext <8 x i8> [[WIDE_LOAD9]] to <8 x i16>
7575
; CHECK-NEXT: [[TMP23:%.*]] = mul nuw <8 x i16> [[TMP22]], [[TMP17]]
7676
; CHECK-NEXT: [[TMP24:%.*]] = lshr <8 x i16> [[TMP23]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
7777
; CHECK-NEXT: [[TMP25:%.*]] = trunc nuw <8 x i16> [[TMP24]] to <8 x i8>
7878
; CHECK-NEXT: store <8 x i8> [[TMP25]], ptr [[TMP21]], align 1
79-
; CHECK-NEXT: [[INDEX_NEXT11]] = add nuw i64 [[INDEX7]], 8
80-
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC5]]
79+
; CHECK-NEXT: [[INDEX_NEXT10]] = add nuw i64 [[INDEX6]], 8
80+
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[N_VEC5]]
8181
; CHECK-NEXT: br i1 [[TMP26]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
8282
; CHECK: vec.epilog.middle.block:
83-
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N_VEC5]], [[TMP0]]
84-
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
83+
; CHECK-NEXT: [[CMP_N11:%.*]] = icmp eq i64 [[N_VEC5]], [[TMP0]]
84+
; CHECK-NEXT: br i1 [[CMP_N11]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[VEC_EPILOG_SCALAR_PH]]
8585
; CHECK: vec.epilog.scalar.ph:
8686
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
8787
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
@@ -165,15 +165,15 @@ define void @test_shrink_zext_in_preheader(ptr noalias %src, ptr noalias %dst, i
165165
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> zeroinitializer
166166
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i16> poison, i16 [[B]], i64 0
167167
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <16 x i32> zeroinitializer
168-
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
169-
; CHECK: vector.body:
170-
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
171168
; CHECK-NEXT: [[TMP5:%.*]] = mul <16 x i16> [[TMP2]], [[TMP4]]
172169
; CHECK-NEXT: [[TMP6:%.*]] = mul <16 x i16> [[TMP2]], [[TMP4]]
173170
; CHECK-NEXT: [[TMP7:%.*]] = lshr <16 x i16> [[TMP5]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
174171
; CHECK-NEXT: [[TMP8:%.*]] = lshr <16 x i16> [[TMP6]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
175172
; CHECK-NEXT: [[TMP9:%.*]] = trunc nuw <16 x i16> [[TMP7]] to <16 x i8>
176173
; CHECK-NEXT: [[TMP10:%.*]] = trunc nuw <16 x i16> [[TMP8]] to <16 x i8>
174+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
175+
; CHECK: vector.body:
176+
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
177177
; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[INDEX]] to i64
178178
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP11]]
179179
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 16
@@ -190,13 +190,13 @@ define void @test_shrink_zext_in_preheader(ptr noalias %src, ptr noalias %dst, i
190190
; CHECK-NEXT: [[TMP15:%.*]] = trunc i32 [[A]] to i16
191191
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i16> poison, i16 [[TMP15]], i64 0
192192
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x i16> poison, i16 [[B]], i64 0
193-
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
194-
; CHECK: vec.epilog.vector.body:
195-
; CHECK-NEXT: [[INDEX7:%.*]] = phi i32 [ 992, [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
196193
; CHECK-NEXT: [[TMP18:%.*]] = mul <8 x i16> [[TMP16]], [[TMP17]]
197194
; CHECK-NEXT: [[TMP19:%.*]] = lshr <8 x i16> [[TMP18]], <i16 8, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
198195
; CHECK-NEXT: [[TMP20:%.*]] = trunc <8 x i16> [[TMP19]] to <8 x i8>
199196
; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <8 x i8> [[TMP20]], <8 x i8> poison, <8 x i32> zeroinitializer
197+
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
198+
; CHECK: vec.epilog.vector.body:
199+
; CHECK-NEXT: [[INDEX7:%.*]] = phi i32 [ 992, [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
200200
; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[INDEX7]] to i64
201201
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP22]]
202202
; CHECK-NEXT: store <8 x i8> [[TMP21]], ptr [[TMP23]], align 1
@@ -244,41 +244,47 @@ define void @test_shrink_select(ptr noalias %src, ptr noalias %dst, i32 %A, i1 %
244244
; CHECK: vector.ph:
245245
; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[A]] to i16
246246
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i16> poison, i16 [[TMP0]], i64 0
247+
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> zeroinitializer
248+
; CHECK-NEXT: [[TMP3:%.*]] = mul <16 x i16> [[TMP2]], <i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99>
249+
; CHECK-NEXT: [[TMP4:%.*]] = mul <16 x i16> [[TMP2]], <i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99, i16 99>
250+
; CHECK-NEXT: [[TMP5:%.*]] = lshr <16 x i16> [[TMP3]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
251+
; CHECK-NEXT: [[TMP6:%.*]] = lshr <16 x i16> [[TMP4]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
252+
; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[C]], <16 x i16> [[TMP5]], <16 x i16> [[TMP3]]
253+
; CHECK-NEXT: [[TMP8:%.*]] = select i1 [[C]], <16 x i16> [[TMP6]], <16 x i16> [[TMP4]]
254+
; CHECK-NEXT: [[TMP9:%.*]] = trunc <16 x i16> [[TMP7]] to <16 x i8>
255+
; CHECK-NEXT: [[TMP10:%.*]] = trunc <16 x i16> [[TMP8]] to <16 x i8>
247256
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
248257
; CHECK: vector.body:
249258
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
250-
; CHECK-NEXT: [[TMP2:%.*]] = mul <16 x i16> [[TMP1]], <i16 99, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison>
251-
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> poison, <16 x i32> zeroinitializer
252-
; CHECK-NEXT: [[TMP4:%.*]] = lshr <16 x i16> [[TMP3]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
253-
; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[C]], <16 x i16> [[TMP4]], <16 x i16> [[TMP3]]
254-
; CHECK-NEXT: [[TMP6:%.*]] = trunc <16 x i16> [[TMP5]] to <16 x i8>
255-
; CHECK-NEXT: [[TMP7:%.*]] = sext i32 [[INDEX]] to i64
256-
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
257-
; CHECK-NEXT: store <16 x i8> [[TMP6]], ptr [[TMP8]], align 1
258-
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
259-
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
260-
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
259+
; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[INDEX]] to i64
260+
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP11]]
261+
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 16
262+
; CHECK-NEXT: store <16 x i8> [[TMP9]], ptr [[TMP12]], align 1
263+
; CHECK-NEXT: store <16 x i8> [[TMP10]], ptr [[TMP13]], align 1
264+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
265+
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
266+
; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
261267
; CHECK: middle.block:
262268
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
263269
; CHECK: vec.epilog.iter.check:
264270
; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
265271
; CHECK: vec.epilog.ph:
266-
; CHECK-NEXT: [[TMP10:%.*]] = trunc i32 [[A]] to i16
267-
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x i16> poison, i16 [[TMP10]], i64 0
272+
; CHECK-NEXT: [[TMP15:%.*]] = trunc i32 [[A]] to i16
273+
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i16> poison, i16 [[TMP15]], i64 0
274+
; CHECK-NEXT: [[TMP17:%.*]] = mul <8 x i16> [[TMP16]], <i16 99, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison>
275+
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i16> [[TMP17]], <8 x i16> poison, <8 x i32> zeroinitializer
276+
; CHECK-NEXT: [[TMP19:%.*]] = lshr <8 x i16> [[TMP18]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
277+
; CHECK-NEXT: [[TMP20:%.*]] = select i1 [[C]], <8 x i16> [[TMP19]], <8 x i16> [[TMP18]]
278+
; CHECK-NEXT: [[TMP21:%.*]] = trunc <8 x i16> [[TMP20]] to <8 x i8>
268279
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
269280
; CHECK: vec.epilog.vector.body:
270281
; CHECK-NEXT: [[INDEX3:%.*]] = phi i32 [ 992, [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
271-
; CHECK-NEXT: [[TMP12:%.*]] = mul <8 x i16> [[TMP11]], <i16 99, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison, i16 poison>
272-
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i16> [[TMP12]], <8 x i16> poison, <8 x i32> zeroinitializer
273-
; CHECK-NEXT: [[TMP14:%.*]] = lshr <8 x i16> [[TMP13]], <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
274-
; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[C]], <8 x i16> [[TMP14]], <8 x i16> [[TMP13]]
275-
; CHECK-NEXT: [[TMP16:%.*]] = trunc <8 x i16> [[TMP15]] to <8 x i8>
276-
; CHECK-NEXT: [[TMP17:%.*]] = sext i32 [[INDEX3]] to i64
277-
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP17]]
278-
; CHECK-NEXT: store <8 x i8> [[TMP16]], ptr [[TMP18]], align 1
282+
; CHECK-NEXT: [[TMP22:%.*]] = sext i32 [[INDEX3]] to i64
283+
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP22]]
284+
; CHECK-NEXT: store <8 x i8> [[TMP21]], ptr [[TMP23]], align 1
279285
; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i32 [[INDEX3]], 8
280-
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 1000
281-
; CHECK-NEXT: br i1 [[TMP19]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
286+
; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 1000
287+
; CHECK-NEXT: br i1 [[TMP24]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
282288
; CHECK: vec.epilog.middle.block:
283289
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
284290
; CHECK: vec.epilog.scalar.ph:

‎llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
2828
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4
2929
; CHECK-NEXT: [[TMP18:%.*]] = sdiv i64 [[M]], [[CONV6]]
3030
; CHECK-NEXT: [[TMP19:%.*]] = sdiv i64 [[M]], [[CONV6]]
31+
; CHECK-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP18]] to i32
32+
; CHECK-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP19]] to i32
33+
; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP18]], [[CONV61]]
34+
; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP19]], [[CONV61]]
35+
; CHECK-NEXT: [[TMP28:%.*]] = mul i32 [[X]], [[TMP20]]
36+
; CHECK-NEXT: [[TMP29:%.*]] = mul i32 [[X]], [[TMP21]]
3137
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
3238
; CHECK: [[VECTOR_BODY]]:
3339
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -37,16 +43,10 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
3743
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP14]], 0
3844
; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 1
3945
; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], [[TMP16]]
40-
; CHECK-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP18]] to i32
41-
; CHECK-NEXT: [[TMP21:%.*]] = trunc i64 [[TMP19]] to i32
42-
; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP18]], [[CONV61]]
43-
; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP19]], [[CONV61]]
4446
; CHECK-NEXT: [[TMP24:%.*]] = sub i64 [[TMP12]], [[TMP22]]
4547
; CHECK-NEXT: [[TMP25:%.*]] = sub i64 [[TMP17]], [[TMP23]]
4648
; CHECK-NEXT: [[TMP26:%.*]] = trunc i64 [[TMP24]] to i32
4749
; CHECK-NEXT: [[TMP27:%.*]] = trunc i64 [[TMP25]] to i32
48-
; CHECK-NEXT: [[TMP28:%.*]] = mul i32 [[X]], [[TMP20]]
49-
; CHECK-NEXT: [[TMP29:%.*]] = mul i32 [[X]], [[TMP21]]
5050
; CHECK-NEXT: [[TMP30:%.*]] = add i32 [[TMP28]], [[TMP26]]
5151
; CHECK-NEXT: [[TMP31:%.*]] = add i32 [[TMP29]], [[TMP27]]
5252
; CHECK-NEXT: [[TMP32:%.*]] = sext i32 [[TMP30]] to i64

‎llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,17 @@
1818
; Check that the extractvalue operands are actually free in vector code.
1919

2020
; FORCED: [[E1:%.+]] = extractvalue { i64, i64 } %sv, 0
21+
; FORCED-NEXT: [[E2:%.+]] = extractvalue { i64, i64 } %sv, 1
2122
; FORCED-NEXT: %broadcast.splatinsert = insertelement <2 x i64> poison, i64 [[E1]], i64 0
2223
; FORCED-NEXT: %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> poison, <2 x i32> zeroinitializer
23-
; FORCED-NEXT: [[E2:%.+]] = extractvalue { i64, i64 } %sv, 1
2424
; FORCED-NEXT: %broadcast.splatinsert1 = insertelement <2 x i64> poison, i64 [[E2]], i64 0
2525
; FORCED-NEXT: %broadcast.splat2 = shufflevector <2 x i64> %broadcast.splatinsert1, <2 x i64> poison, <2 x i32> zeroinitializer
26+
; FORCED-NEXT: [[ADD:%.+]] = add <2 x i64> %broadcast.splat, %broadcast.splat2
2627

2728
; FORCED-LABEL: vector.body: ; preds = %vector.body, %vector.ph
2829
; FORCED-NEXT: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
2930
; FORCED-NEXT: [[IV_0:%.]] = add i32 %index, 0
3031
; FORCED-NEXT: [[GEP:%.+]] = getelementptr i64, ptr %dst, i32 [[IV_0]]
31-
; FORCED-NEXT: [[ADD:%.+]] = add <2 x i64> %broadcast.splat, %broadcast.splat2
3232
; FORCED-NEXT: [[GEP2:%.+]] = getelementptr i64, ptr [[GEP]], i32 0
3333
; FORCED-NEXT: store <2 x i64> [[ADD]], ptr [[GEP2]], align 4
3434
; FORCED-NEXT: %index.next = add nuw i32 %index, 2
@@ -68,9 +68,9 @@ declare float @powf(float, float) readnone nounwind
6868
; FORCED-LABEL: define void @test_getVectorCallCost
6969

7070
; FORCED: [[E1:%.+]] = extractvalue { float, float } %sv, 0
71+
; FORCED-NEXT: [[E2:%.+]] = extractvalue { float, float } %sv, 1
7172
; FORCED-NEXT: %broadcast.splatinsert = insertelement <2 x float> poison, float [[E1]], i64 0
7273
; FORCED-NEXT: %broadcast.splat = shufflevector <2 x float> %broadcast.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
73-
; FORCED-NEXT: [[E2:%.+]] = extractvalue { float, float } %sv, 1
7474
; FORCED-NEXT: %broadcast.splatinsert1 = insertelement <2 x float> poison, float [[E2]], i64 0
7575
; FORCED-NEXT: %broadcast.splat2 = shufflevector <2 x float> %broadcast.splatinsert1, <2 x float> poison, <2 x i32> zeroinitializer
7676

0 commit comments

Comments
 (0)