Skip to content

Commit c04b9ba

Browse files
sparker-armsparker-arm
committed
[ARM][MVE] Clear MaskedInsts vector
In MVETailPredication, clear the vector before running on a new loop. Differential Revision: https://reviews.llvm.org/D73048
1 parent 45538b5 commit c04b9ba

File tree

2 files changed

+146
-0
lines changed

2 files changed

+146
-0
lines changed

‎llvm/lib/Target/ARM/MVETailPredication.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {
148148
if (skipLoop(L) || DisableTailPredication)
149149
return false;
150150

151+
MaskedInsts.clear();
151152
Function &F = *L->getHeader()->getParent();
152153
auto &TPC = getAnalysis<TargetPassConfig>();
153154
auto &TM = TPC.getTM<TargetMachine>();

‎llvm/test/CodeGen/Thumb2/LowOverheadLoops/clear-maskedinsts.ll

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2+
; RUN: opt -mtriple=thumbv8.1m.main -mattr=+mve.fp -mve-tail-predication -disable-mve-tail-predication=false %s -S -o - | FileCheck %s
3+
4+
define hidden i32 @_Z4loopPiPjiS0_i(i32* noalias nocapture readonly %s1, i32* noalias nocapture readonly %s2, i32 %x, i32* noalias nocapture %d, i32 %n) {
5+
; CHECK-LABEL: @_Z4loopPiPjiS0_i(
6+
; CHECK-NEXT: entry:
7+
; CHECK-NEXT: [[CMP63:%.*]] = icmp sgt i32 [[N:%.*]], 0
8+
; CHECK-NEXT: br i1 [[CMP63]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
9+
; CHECK: for.body.lr.ph:
10+
; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
11+
; CHECK-NEXT: [[N_RND_UP77:%.*]] = add nuw i32 [[N]], 3
12+
; CHECK-NEXT: [[N_VEC79:%.*]] = and i32 [[N_RND_UP77]], -4
13+
; CHECK-NEXT: [[TRIP_COUNT_MINUS_183:%.*]] = add nsw i32 [[N]], -1
14+
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N_VEC79]], -4
15+
; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 2
16+
; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 1
17+
; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i32 [[TMP1]], 1
18+
; CHECK-NEXT: br i1 [[TOBOOL]], label [[VECTOR_BODY75_PREHEADER:%.*]], label [[VECTOR_PH:%.*]]
19+
; CHECK: vector.body75.preheader:
20+
; CHECK-NEXT: call void @llvm.set.loop.iterations.i32(i32 [[TMP2]])
21+
; CHECK-NEXT: br label [[VECTOR_BODY75:%.*]]
22+
; CHECK: vector.ph:
23+
; CHECK-NEXT: [[BROADCAST_SPLATINSERT71:%.*]] = insertelement <4 x i32> undef, i32 [[X]], i32 0
24+
; CHECK-NEXT: [[BROADCAST_SPLAT72:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT71]], <4 x i32> undef, <4 x i32> zeroinitializer
25+
; CHECK-NEXT: call void @llvm.set.loop.iterations.i32(i32 [[TMP3]])
26+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
27+
; CHECK: vector.body:
28+
; CHECK-NEXT: [[LSR_IV9:%.*]] = phi i32* [ [[SCEVGEP10:%.*]], [[VECTOR_BODY]] ], [ [[D:%.*]], [[VECTOR_PH]] ]
29+
; CHECK-NEXT: [[TMP4:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
30+
; CHECK-NEXT: [[TMP5:%.*]] = phi i32 [ [[N]], [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
31+
; CHECK-NEXT: [[LSR_IV911:%.*]] = bitcast i32* [[LSR_IV9]] to <4 x i32>*
32+
; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP5]])
33+
; CHECK-NEXT: [[TMP7]] = sub i32 [[TMP5]], 4
34+
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[BROADCAST_SPLAT72]], <4 x i32>* [[LSR_IV911]], i32 4, <4 x i1> [[TMP6]])
35+
; CHECK-NEXT: [[SCEVGEP10]] = getelementptr i32, i32* [[LSR_IV9]], i32 4
36+
; CHECK-NEXT: [[TMP8]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP4]], i32 1)
37+
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0
38+
; CHECK-NEXT: br i1 [[TMP9]], label [[VECTOR_BODY]], label [[FOR_COND_CLEANUP]]
39+
; CHECK: vector.body75:
40+
; CHECK-NEXT: [[LSR_IV6:%.*]] = phi i32* [ [[S1:%.*]], [[VECTOR_BODY75_PREHEADER]] ], [ [[SCEVGEP7:%.*]], [[VECTOR_BODY75]] ]
41+
; CHECK-NEXT: [[LSR_IV3:%.*]] = phi i32* [ [[S2:%.*]], [[VECTOR_BODY75_PREHEADER]] ], [ [[SCEVGEP4:%.*]], [[VECTOR_BODY75]] ]
42+
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i32* [ [[D]], [[VECTOR_BODY75_PREHEADER]] ], [ [[SCEVGEP:%.*]], [[VECTOR_BODY75]] ]
43+
; CHECK-NEXT: [[INDEX80:%.*]] = phi i32 [ [[INDEX_NEXT81:%.*]], [[VECTOR_BODY75]] ], [ 0, [[VECTOR_BODY75_PREHEADER]] ]
44+
; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP2]], [[VECTOR_BODY75_PREHEADER]] ], [ [[TMP15:%.*]], [[VECTOR_BODY75]] ]
45+
; CHECK-NEXT: [[LSR_IV68:%.*]] = bitcast i32* [[LSR_IV6]] to <4 x i32>*
46+
; CHECK-NEXT: [[LSR_IV35:%.*]] = bitcast i32* [[LSR_IV3]] to <4 x i32>*
47+
; CHECK-NEXT: [[LSR_IV2:%.*]] = bitcast i32* [[LSR_IV]] to <4 x i32>*
48+
; CHECK-NEXT: [[BROADCAST_SPLATINSERT84:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX80]], i32 0
49+
; CHECK-NEXT: [[BROADCAST_SPLAT85:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT84]], <4 x i32> undef, <4 x i32> zeroinitializer
50+
; CHECK-NEXT: [[INDUCTION86:%.*]] = add <4 x i32> [[BROADCAST_SPLAT85]], <i32 0, i32 1, i32 2, i32 3>
51+
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> undef, i32 [[TRIP_COUNT_MINUS_183]], i32 0
52+
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <4 x i32> zeroinitializer
53+
; CHECK-NEXT: [[TMP13:%.*]] = icmp ule <4 x i32> [[INDUCTION86]], [[TMP12]]
54+
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV68]], i32 4, <4 x i1> [[TMP13]], <4 x i32> undef)
55+
; CHECK-NEXT: [[WIDE_MASKED_LOAD89:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[LSR_IV35]], i32 4, <4 x i1> [[TMP13]], <4 x i32> undef)
56+
; CHECK-NEXT: [[TMP14:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[WIDE_MASKED_LOAD89]], <4 x i32> [[WIDE_MASKED_LOAD]])
57+
; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[TMP14]], <4 x i32>* [[LSR_IV2]], i32 4, <4 x i1> [[TMP13]])
58+
; CHECK-NEXT: [[INDEX_NEXT81]] = add i32 [[INDEX80]], 4
59+
; CHECK-NEXT: [[SCEVGEP]] = getelementptr i32, i32* [[LSR_IV]], i32 4
60+
; CHECK-NEXT: [[SCEVGEP4]] = getelementptr i32, i32* [[LSR_IV3]], i32 4
61+
; CHECK-NEXT: [[SCEVGEP7]] = getelementptr i32, i32* [[LSR_IV6]], i32 4
62+
; CHECK-NEXT: [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 [[TMP10]], i32 1)
63+
; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0
64+
; CHECK-NEXT: br i1 [[TMP16]], label [[VECTOR_BODY75]], label [[FOR_COND_CLEANUP]]
65+
; CHECK: for.cond.cleanup:
66+
; CHECK-NEXT: ret i32 0
67+
;
68+
entry:
69+
%cmp63 = icmp sgt i32 %n, 0
70+
br i1 %cmp63, label %for.body.lr.ph, label %for.cond.cleanup
71+
72+
for.body.lr.ph: ; preds = %entry
73+
%tobool = icmp eq i32 %x, 0
74+
%n.rnd.up77 = add nuw i32 %n, 3
75+
%n.vec79 = and i32 %n.rnd.up77, -4
76+
%trip.count.minus.183 = add nsw i32 %n, -1
77+
%0 = add i32 %n.vec79, -4
78+
%1 = lshr i32 %0, 2
79+
%2 = add nuw nsw i32 %1, 1
80+
%3 = add nuw nsw i32 %1, 1
81+
br i1 %tobool, label %vector.body75.preheader, label %vector.ph
82+
83+
vector.body75.preheader: ; preds = %for.body.lr.ph
84+
call void @llvm.set.loop.iterations.i32(i32 %2)
85+
br label %vector.body75
86+
87+
vector.ph: ; preds = %for.body.lr.ph
88+
%broadcast.splatinsert71 = insertelement <4 x i32> undef, i32 %x, i32 0
89+
%broadcast.splat72 = shufflevector <4 x i32> %broadcast.splatinsert71, <4 x i32> undef, <4 x i32> zeroinitializer
90+
call void @llvm.set.loop.iterations.i32(i32 %3)
91+
br label %vector.body
92+
93+
vector.body: ; preds = %vector.body, %vector.ph
94+
%lsr.iv9 = phi i32* [ %scevgep10, %vector.body ], [ %d, %vector.ph ]
95+
%index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
96+
%4 = phi i32 [ %3, %vector.ph ], [ %8, %vector.body ]
97+
%lsr.iv911 = bitcast i32* %lsr.iv9 to <4 x i32>*
98+
%broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
99+
%broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
100+
%induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
101+
%5 = insertelement <4 x i32> undef, i32 %trip.count.minus.183, i32 0
102+
%6 = shufflevector <4 x i32> %5, <4 x i32> undef, <4 x i32> zeroinitializer
103+
%7 = icmp ule <4 x i32> %induction, %6
104+
call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %broadcast.splat72, <4 x i32>* %lsr.iv911, i32 4, <4 x i1> %7)
105+
%index.next = add i32 %index, 4
106+
%scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 4
107+
%8 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %4, i32 1)
108+
%9 = icmp ne i32 %8, 0
109+
br i1 %9, label %vector.body, label %for.cond.cleanup
110+
111+
vector.body75: ; preds = %vector.body75, %vector.body75.preheader
112+
%lsr.iv6 = phi i32* [ %s1, %vector.body75.preheader ], [ %scevgep7, %vector.body75 ]
113+
%lsr.iv3 = phi i32* [ %s2, %vector.body75.preheader ], [ %scevgep4, %vector.body75 ]
114+
%lsr.iv = phi i32* [ %d, %vector.body75.preheader ], [ %scevgep, %vector.body75 ]
115+
%index80 = phi i32 [ %index.next81, %vector.body75 ], [ 0, %vector.body75.preheader ]
116+
%10 = phi i32 [ %2, %vector.body75.preheader ], [ %15, %vector.body75 ]
117+
%lsr.iv68 = bitcast i32* %lsr.iv6 to <4 x i32>*
118+
%lsr.iv35 = bitcast i32* %lsr.iv3 to <4 x i32>*
119+
%lsr.iv2 = bitcast i32* %lsr.iv to <4 x i32>*
120+
%broadcast.splatinsert84 = insertelement <4 x i32> undef, i32 %index80, i32 0
121+
%broadcast.splat85 = shufflevector <4 x i32> %broadcast.splatinsert84, <4 x i32> undef, <4 x i32> zeroinitializer
122+
%induction86 = add <4 x i32> %broadcast.splat85, <i32 0, i32 1, i32 2, i32 3>
123+
%11 = insertelement <4 x i32> undef, i32 %trip.count.minus.183, i32 0
124+
%12 = shufflevector <4 x i32> %11, <4 x i32> undef, <4 x i32> zeroinitializer
125+
%13 = icmp ule <4 x i32> %induction86, %12
126+
%wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv68, i32 4, <4 x i1> %13, <4 x i32> undef)
127+
%wide.masked.load89 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv35, i32 4, <4 x i1> %13, <4 x i32> undef)
128+
%14 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %wide.masked.load89, <4 x i32> %wide.masked.load)
129+
call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %14, <4 x i32>* %lsr.iv2, i32 4, <4 x i1> %13)
130+
%index.next81 = add i32 %index80, 4
131+
%scevgep = getelementptr i32, i32* %lsr.iv, i32 4
132+
%scevgep4 = getelementptr i32, i32* %lsr.iv3, i32 4
133+
%scevgep7 = getelementptr i32, i32* %lsr.iv6, i32 4
134+
%15 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %10, i32 1)
135+
%16 = icmp ne i32 %15, 0
136+
br i1 %16, label %vector.body75, label %for.cond.cleanup
137+
138+
for.cond.cleanup: ; preds = %vector.body, %vector.body75, %entry
139+
ret i32 0
140+
}
141+
declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
142+
declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
143+
declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
144+
declare void @llvm.set.loop.iterations.i32(i32)
145+
declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)

0 commit comments

Comments
 (0)