Skip to content

Commit 469f6b9

Browse files
lukel97lukel97
authored
[DAGCombiner] Combine vp.strided.load with unit stride to vp.load (#66766)
This is the VP equivalent of #65674. We already combine MGATHER loads with unit stride to MLOAD, so this extends it for EXPERIMENTAL_VP_STRIDED_LOAD.
1 parent 9b5ef2b commit 469f6b9

File tree

3 files changed

+198
-36
lines changed

3 files changed

+198
-36
lines changed

‎llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -539,6 +539,7 @@ namespace {
539539
SDValue visitMSCATTER(SDNode *N);
540540
SDValue visitVPGATHER(SDNode *N);
541541
SDValue visitVPSCATTER(SDNode *N);
542+
SDValue visitVP_STRIDED_LOAD(SDNode *N);
542543
SDValue visitFP_TO_FP16(SDNode *N);
543544
SDValue visitFP16_TO_FP(SDNode *N);
544545
SDValue visitFP_TO_BF16(SDNode *N);
@@ -11959,6 +11960,22 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) {
1195911960
return SDValue();
1196011961
}
1196111962

11963+
SDValue DAGCombiner::visitVP_STRIDED_LOAD(SDNode *N) {
11964+
auto *SLD = cast<VPStridedLoadSDNode>(N);
11965+
EVT EltVT = SLD->getValueType(0).getVectorElementType();
11966+
// Combine strided loads with unit-stride to a regular VP load.
11967+
if (auto *CStride = dyn_cast<ConstantSDNode>(SLD->getStride());
11968+
CStride && CStride->getZExtValue() == EltVT.getStoreSize()) {
11969+
SDValue NewLd = DAG.getLoadVP(
11970+
SLD->getAddressingMode(), SLD->getExtensionType(), SLD->getValueType(0),
11971+
SDLoc(N), SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(),
11972+
SLD->getMask(), SLD->getVectorLength(), SLD->getMemoryVT(),
11973+
SLD->getMemOperand(), SLD->isExpandingLoad());
11974+
return CombineTo(N, NewLd, NewLd.getValue(1));
11975+
}
11976+
return SDValue();
11977+
}
11978+
1196211979
/// A vector select of 2 constant vectors can be simplified to math/logic to
1196311980
/// avoid a variable select instruction and possibly avoid constant loads.
1196411981
SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
@@ -25976,6 +25993,10 @@ SDValue DAGCombiner::visitVPOp(SDNode *N) {
2597625993
if (SDValue SD = visitVPSCATTER(N))
2597725994
return SD;
2597825995

25996+
if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD)
25997+
if (SDValue SD = visitVP_STRIDED_LOAD(N))
25998+
return SD;
25999+
2597926000
// VP operations in which all vector elements are disabled - either by
2598026001
// determining that the mask is all false or that the EVL is 0 - can be
2598126002
// eliminated.

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-vpload.ll

Lines changed: 87 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,16 @@ define <8 x i8> @strided_vpload_v8i8(ptr %ptr, i32 signext %stride, <8 x i1> %m,
9696
ret <8 x i8> %load
9797
}
9898

99+
define <8 x i8> @strided_vpload_v8i8_unit_stride(ptr %ptr, <8 x i1> %m, i32 zeroext %evl) {
100+
; CHECK-LABEL: strided_vpload_v8i8_unit_stride:
101+
; CHECK: # %bb.0:
102+
; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma
103+
; CHECK-NEXT: vle8.v v8, (a0), v0.t
104+
; CHECK-NEXT: ret
105+
%load = call <8 x i8> @llvm.experimental.vp.strided.load.v8i8.p0.i32(ptr %ptr, i32 1, <8 x i1> %m, i32 %evl)
106+
ret <8 x i8> %load
107+
}
108+
99109
declare <2 x i16> @llvm.experimental.vp.strided.load.v2i16.p0.i32(ptr, i32, <2 x i1>, i32)
100110

101111
define <2 x i16> @strided_vpload_v2i16(ptr %ptr, i32 signext %stride, <2 x i1> %m, i32 zeroext %evl) {
@@ -132,6 +142,16 @@ define <8 x i16> @strided_vpload_v8i16(ptr %ptr, i32 signext %stride, <8 x i1> %
132142
ret <8 x i16> %load
133143
}
134144

145+
define <8 x i16> @strided_vpload_v8i16_unit_stride(ptr %ptr, <8 x i1> %m, i32 zeroext %evl) {
146+
; CHECK-LABEL: strided_vpload_v8i16_unit_stride:
147+
; CHECK: # %bb.0:
148+
; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma
149+
; CHECK-NEXT: vle16.v v8, (a0), v0.t
150+
; CHECK-NEXT: ret
151+
%load = call <8 x i16> @llvm.experimental.vp.strided.load.v8i16.p0.i32(ptr %ptr, i32 2, <8 x i1> %m, i32 %evl)
152+
ret <8 x i16> %load
153+
}
154+
135155
define <8 x i16> @strided_vpload_v8i16_allones_mask(ptr %ptr, i32 signext %stride, i32 zeroext %evl) {
136156
; CHECK-LABEL: strided_vpload_v8i16_allones_mask:
137157
; CHECK: # %bb.0:
@@ -168,6 +188,16 @@ define <4 x i32> @strided_vpload_v4i32(ptr %ptr, i32 signext %stride, <4 x i1> %
168188
ret <4 x i32> %load
169189
}
170190

191+
define <4 x i32> @strided_vpload_v4i32_unit_stride(ptr %ptr, <4 x i1> %m, i32 zeroext %evl) {
192+
; CHECK-LABEL: strided_vpload_v4i32_unit_stride:
193+
; CHECK: # %bb.0:
194+
; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
195+
; CHECK-NEXT: vle32.v v8, (a0), v0.t
196+
; CHECK-NEXT: ret
197+
%load = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i32(ptr %ptr, i32 4, <4 x i1> %m, i32 %evl)
198+
ret <4 x i32> %load
199+
}
200+
171201
declare <8 x i32> @llvm.experimental.vp.strided.load.v8i32.p0.i32(ptr, i32, <8 x i1>, i32)
172202

173203
define <8 x i32> @strided_vpload_v8i32(ptr %ptr, i32 signext %stride, <8 x i1> %m, i32 zeroext %evl) {
@@ -204,6 +234,16 @@ define <2 x i64> @strided_vpload_v2i64(ptr %ptr, i32 signext %stride, <2 x i1> %
204234
ret <2 x i64> %load
205235
}
206236

237+
define <2 x i64> @strided_vpload_v2i64_unit_stride(ptr %ptr, <2 x i1> %m, i32 zeroext %evl) {
238+
; CHECK-LABEL: strided_vpload_v2i64_unit_stride:
239+
; CHECK: # %bb.0:
240+
; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma
241+
; CHECK-NEXT: vle64.v v8, (a0), v0.t
242+
; CHECK-NEXT: ret
243+
%load = call <2 x i64> @llvm.experimental.vp.strided.load.v2i64.p0.i32(ptr %ptr, i32 8, <2 x i1> %m, i32 %evl)
244+
ret <2 x i64> %load
245+
}
246+
207247
declare <4 x i64> @llvm.experimental.vp.strided.load.v4i64.p0.i32(ptr, i32, <4 x i1>, i32)
208248

209249
define <4 x i64> @strided_vpload_v4i64(ptr %ptr, i32 signext %stride, <4 x i1> %m, i32 zeroext %evl) {
@@ -288,6 +328,16 @@ define <8 x half> @strided_vpload_v8f16(ptr %ptr, i32 signext %stride, <8 x i1>
288328
ret <8 x half> %load
289329
}
290330

331+
define <8 x half> @strided_vpload_v8f16_unit_stride(ptr %ptr, <8 x i1> %m, i32 zeroext %evl) {
332+
; CHECK-LABEL: strided_vpload_v8f16_unit_stride:
333+
; CHECK: # %bb.0:
334+
; CHECK-NEXT: vsetvli zero, a1, e16, m1, ta, ma
335+
; CHECK-NEXT: vle16.v v8, (a0), v0.t
336+
; CHECK-NEXT: ret
337+
%load = call <8 x half> @llvm.experimental.vp.strided.load.v8f16.p0.i32(ptr %ptr, i32 2, <8 x i1> %m, i32 %evl)
338+
ret <8 x half> %load
339+
}
340+
291341
declare <2 x float> @llvm.experimental.vp.strided.load.v2f32.p0.i32(ptr, i32, <2 x i1>, i32)
292342

293343
define <2 x float> @strided_vpload_v2f32(ptr %ptr, i32 signext %stride, <2 x i1> %m, i32 zeroext %evl) {
@@ -312,6 +362,16 @@ define <4 x float> @strided_vpload_v4f32(ptr %ptr, i32 signext %stride, <4 x i1>
312362
ret <4 x float> %load
313363
}
314364

365+
define <4 x float> @strided_vpload_v4f32_unit_stride(ptr %ptr, <4 x i1> %m, i32 zeroext %evl) {
366+
; CHECK-LABEL: strided_vpload_v4f32_unit_stride:
367+
; CHECK: # %bb.0:
368+
; CHECK-NEXT: vsetvli zero, a1, e32, m1, ta, ma
369+
; CHECK-NEXT: vle32.v v8, (a0), v0.t
370+
; CHECK-NEXT: ret
371+
%load = call <4 x float> @llvm.experimental.vp.strided.load.v4f32.p0.i32(ptr %ptr, i32 4, <4 x i1> %m, i32 %evl)
372+
ret <4 x float> %load
373+
}
374+
315375
declare <8 x float> @llvm.experimental.vp.strided.load.v8f32.p0.i32(ptr, i32, <8 x i1>, i32)
316376

317377
define <8 x float> @strided_vpload_v8f32(ptr %ptr, i32 signext %stride, <8 x i1> %m, i32 zeroext %evl) {
@@ -348,6 +408,17 @@ define <2 x double> @strided_vpload_v2f64(ptr %ptr, i32 signext %stride, <2 x i1
348408
ret <2 x double> %load
349409
}
350410

411+
define <2 x double> @strided_vpload_v2f64_unit_stride(ptr %ptr, <2 x i1> %m, i32 zeroext %evl) {
412+
; CHECK-LABEL: strided_vpload_v2f64_unit_stride:
413+
; CHECK: # %bb.0:
414+
; CHECK-NEXT: vsetvli zero, a1, e64, m1, ta, ma
415+
; CHECK-NEXT: vle64.v v8, (a0), v0.t
416+
; CHECK-NEXT: ret
417+
%load = call <2 x double> @llvm.experimental.vp.strided.load.v2f64.p0.i32(ptr %ptr, i32 8, <2 x i1> %m, i32 %evl)
418+
ret <2 x double> %load
419+
}
420+
421+
351422
declare <4 x double> @llvm.experimental.vp.strided.load.v4f64.p0.i32(ptr, i32, <4 x i1>, i32)
352423

353424
define <4 x double> @strided_vpload_v4f64(ptr %ptr, i32 signext %stride, <4 x i1> %m, i32 zeroext %evl) {
@@ -416,10 +487,10 @@ define <32 x double> @strided_vpload_v32f64(ptr %ptr, i32 signext %stride, <32 x
416487
; CHECK-NEXT: li a4, 16
417488
; CHECK-NEXT: vmv1r.v v9, v0
418489
; CHECK-NEXT: mv a3, a2
419-
; CHECK-NEXT: bltu a2, a4, .LBB33_2
490+
; CHECK-NEXT: bltu a2, a4, .LBB40_2
420491
; CHECK-NEXT: # %bb.1:
421492
; CHECK-NEXT: li a3, 16
422-
; CHECK-NEXT: .LBB33_2:
493+
; CHECK-NEXT: .LBB40_2:
423494
; CHECK-NEXT: mul a4, a3, a1
424495
; CHECK-NEXT: add a4, a0, a4
425496
; CHECK-NEXT: addi a5, a2, -16
@@ -444,10 +515,10 @@ define <32 x double> @strided_vpload_v32f64_allones_mask(ptr %ptr, i32 signext %
444515
; CHECK: # %bb.0:
445516
; CHECK-NEXT: li a4, 16
446517
; CHECK-NEXT: mv a3, a2
447-
; CHECK-NEXT: bltu a2, a4, .LBB34_2
518+
; CHECK-NEXT: bltu a2, a4, .LBB41_2
448519
; CHECK-NEXT: # %bb.1:
449520
; CHECK-NEXT: li a3, 16
450-
; CHECK-NEXT: .LBB34_2:
521+
; CHECK-NEXT: .LBB41_2:
451522
; CHECK-NEXT: mul a4, a3, a1
452523
; CHECK-NEXT: add a4, a0, a4
453524
; CHECK-NEXT: addi a5, a2, -16
@@ -474,21 +545,21 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask
474545
; CHECK-RV32-NEXT: li a5, 32
475546
; CHECK-RV32-NEXT: vmv1r.v v8, v0
476547
; CHECK-RV32-NEXT: mv a3, a4
477-
; CHECK-RV32-NEXT: bltu a4, a5, .LBB35_2
548+
; CHECK-RV32-NEXT: bltu a4, a5, .LBB42_2
478549
; CHECK-RV32-NEXT: # %bb.1:
479550
; CHECK-RV32-NEXT: li a3, 32
480-
; CHECK-RV32-NEXT: .LBB35_2:
551+
; CHECK-RV32-NEXT: .LBB42_2:
481552
; CHECK-RV32-NEXT: mul a5, a3, a2
482553
; CHECK-RV32-NEXT: addi a6, a4, -32
483554
; CHECK-RV32-NEXT: sltu a4, a4, a6
484555
; CHECK-RV32-NEXT: addi a4, a4, -1
485556
; CHECK-RV32-NEXT: and a6, a4, a6
486557
; CHECK-RV32-NEXT: li a4, 16
487558
; CHECK-RV32-NEXT: add a5, a1, a5
488-
; CHECK-RV32-NEXT: bltu a6, a4, .LBB35_4
559+
; CHECK-RV32-NEXT: bltu a6, a4, .LBB42_4
489560
; CHECK-RV32-NEXT: # %bb.3:
490561
; CHECK-RV32-NEXT: li a6, 16
491-
; CHECK-RV32-NEXT: .LBB35_4:
562+
; CHECK-RV32-NEXT: .LBB42_4:
492563
; CHECK-RV32-NEXT: vsetivli zero, 4, e8, mf2, ta, ma
493564
; CHECK-RV32-NEXT: vslidedown.vi v0, v8, 4
494565
; CHECK-RV32-NEXT: vsetvli zero, a6, e64, m8, ta, ma
@@ -497,10 +568,10 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask
497568
; CHECK-RV32-NEXT: sltu a6, a3, a5
498569
; CHECK-RV32-NEXT: addi a6, a6, -1
499570
; CHECK-RV32-NEXT: and a5, a6, a5
500-
; CHECK-RV32-NEXT: bltu a3, a4, .LBB35_6
571+
; CHECK-RV32-NEXT: bltu a3, a4, .LBB42_6
501572
; CHECK-RV32-NEXT: # %bb.5:
502573
; CHECK-RV32-NEXT: li a3, 16
503-
; CHECK-RV32-NEXT: .LBB35_6:
574+
; CHECK-RV32-NEXT: .LBB42_6:
504575
; CHECK-RV32-NEXT: mul a4, a3, a2
505576
; CHECK-RV32-NEXT: add a4, a1, a4
506577
; CHECK-RV32-NEXT: vsetivli zero, 2, e8, mf4, ta, ma
@@ -524,21 +595,21 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask
524595
; CHECK-RV64-NEXT: li a5, 32
525596
; CHECK-RV64-NEXT: vmv1r.v v8, v0
526597
; CHECK-RV64-NEXT: mv a4, a3
527-
; CHECK-RV64-NEXT: bltu a3, a5, .LBB35_2
598+
; CHECK-RV64-NEXT: bltu a3, a5, .LBB42_2
528599
; CHECK-RV64-NEXT: # %bb.1:
529600
; CHECK-RV64-NEXT: li a4, 32
530-
; CHECK-RV64-NEXT: .LBB35_2:
601+
; CHECK-RV64-NEXT: .LBB42_2:
531602
; CHECK-RV64-NEXT: mul a5, a4, a2
532603
; CHECK-RV64-NEXT: addi a6, a3, -32
533604
; CHECK-RV64-NEXT: sltu a3, a3, a6
534605
; CHECK-RV64-NEXT: addi a3, a3, -1
535606
; CHECK-RV64-NEXT: and a6, a3, a6
536607
; CHECK-RV64-NEXT: li a3, 16
537608
; CHECK-RV64-NEXT: add a5, a1, a5
538-
; CHECK-RV64-NEXT: bltu a6, a3, .LBB35_4
609+
; CHECK-RV64-NEXT: bltu a6, a3, .LBB42_4
539610
; CHECK-RV64-NEXT: # %bb.3:
540611
; CHECK-RV64-NEXT: li a6, 16
541-
; CHECK-RV64-NEXT: .LBB35_4:
612+
; CHECK-RV64-NEXT: .LBB42_4:
542613
; CHECK-RV64-NEXT: vsetivli zero, 4, e8, mf2, ta, ma
543614
; CHECK-RV64-NEXT: vslidedown.vi v0, v8, 4
544615
; CHECK-RV64-NEXT: vsetvli zero, a6, e64, m8, ta, ma
@@ -547,10 +618,10 @@ define <33 x double> @strided_load_v33f64(ptr %ptr, i64 %stride, <33 x i1> %mask
547618
; CHECK-RV64-NEXT: sltu a6, a4, a5
548619
; CHECK-RV64-NEXT: addi a6, a6, -1
549620
; CHECK-RV64-NEXT: and a5, a6, a5
550-
; CHECK-RV64-NEXT: bltu a4, a3, .LBB35_6
621+
; CHECK-RV64-NEXT: bltu a4, a3, .LBB42_6
551622
; CHECK-RV64-NEXT: # %bb.5:
552623
; CHECK-RV64-NEXT: li a4, 16
553-
; CHECK-RV64-NEXT: .LBB35_6:
624+
; CHECK-RV64-NEXT: .LBB42_6:
554625
; CHECK-RV64-NEXT: mul a3, a4, a2
555626
; CHECK-RV64-NEXT: add a3, a1, a3
556627
; CHECK-RV64-NEXT: vsetivli zero, 2, e8, mf4, ta, ma

0 commit comments

Comments
 (0)