Skip to content

[RISCV] Prefer whole register loads and stores when VL=VLMAX #75531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24180,6 +24180,10 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) {

unsigned Index = Extract->getConstantOperandVal(1);
unsigned NumElts = VT.getVectorMinNumElements();
// A fixed length vector being extracted from a scalable vector
// may not be any *smaller* than the scalable one.
if (Index == 0 && NumElts >= Ld->getValueType(0).getVectorMinNumElements())
return SDValue();

// The definition of EXTRACT_SUBVECTOR states that the index must be a
// multiple of the minimum number of elements in the result type.
Expand Down
31 changes: 28 additions & 3 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9838,6 +9838,20 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
MVT XLenVT = Subtarget.getXLenVT();
MVT ContainerVT = getContainerForFixedLengthVector(VT);

// If we know the exact VLEN and our fixed length vector completely fills
// the container, use a whole register load instead.
const auto [MinVLMAX, MaxVLMAX] =
RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
SDValue NewLoad =
DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
Load->getPointerInfo(), Load->getOriginalAlign(),
Load->getMemOperand()->getFlags());
SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
}

SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget);

bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
Expand Down Expand Up @@ -9882,12 +9896,23 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,

MVT ContainerVT = getContainerForFixedLengthVector(VT);

SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
Subtarget);

SDValue NewValue =
convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);


// If we know the exact VLEN and our fixed length vector completely fills
// the container, use a whole register store instead.
const auto [MinVLMAX, MaxVLMAX] =
RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
getLMUL1VT(ContainerVT).bitsLE(ContainerVT))
return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
Store->getPointerInfo(), Store->getOriginalAlign(),
Store->getMemOperand()->getFlags());

SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
Subtarget);

bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
SDValue IntID = DAG.getTargetConstant(
IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, XLenVT);
Expand Down
177 changes: 118 additions & 59 deletions llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll
Original file line number Diff line number Diff line change
Expand Up @@ -63,93 +63,145 @@ define void @extract_v2i8_v8i8_6(ptr %x, ptr %y) {
}

define void @extract_v1i32_v8i32_4(ptr %x, ptr %y) {
; CHECK-LABEL: extract_v1i32_v8i32_4:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 4
; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT: vse32.v v8, (a1)
; CHECK-NEXT: ret
; CHECK-V-LABEL: extract_v1i32_v8i32_4:
; CHECK-V: # %bb.0:
; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-V-NEXT: vle32.v v8, (a0)
; CHECK-V-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; CHECK-V-NEXT: vslidedown.vi v8, v8, 4
; CHECK-V-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-V-NEXT: vse32.v v8, (a1)
; CHECK-V-NEXT: ret
;
; CHECK-KNOWNVLEN128-LABEL: extract_v1i32_v8i32_4:
; CHECK-KNOWNVLEN128: # %bb.0:
; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0)
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 4
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1)
; CHECK-KNOWNVLEN128-NEXT: ret
%a = load <8 x i32>, ptr %x
%c = call <1 x i32> @llvm.vector.extract.v1i32.v8i32(<8 x i32> %a, i64 4)
store <1 x i32> %c, ptr %y
ret void
}

define void @extract_v1i32_v8i32_5(ptr %x, ptr %y) {
; CHECK-LABEL: extract_v1i32_v8i32_5:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 5
; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-NEXT: vse32.v v8, (a1)
; CHECK-NEXT: ret
; CHECK-V-LABEL: extract_v1i32_v8i32_5:
; CHECK-V: # %bb.0:
; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-V-NEXT: vle32.v v8, (a0)
; CHECK-V-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; CHECK-V-NEXT: vslidedown.vi v8, v8, 5
; CHECK-V-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-V-NEXT: vse32.v v8, (a1)
; CHECK-V-NEXT: ret
;
; CHECK-KNOWNVLEN128-LABEL: extract_v1i32_v8i32_5:
; CHECK-KNOWNVLEN128: # %bb.0:
; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0)
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, m2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 5
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 1, e32, mf2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1)
; CHECK-KNOWNVLEN128-NEXT: ret
%a = load <8 x i32>, ptr %x
%c = call <1 x i32> @llvm.vector.extract.v1i32.v8i32(<8 x i32> %a, i64 5)
store <1 x i32> %c, ptr %y
ret void
}

define void @extract_v2i32_v8i32_0(ptr %x, ptr %y) {
; CHECK-LABEL: extract_v2i32_v8i32_0:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT: vse32.v v8, (a1)
; CHECK-NEXT: ret
; CHECK-V-LABEL: extract_v2i32_v8i32_0:
; CHECK-V: # %bb.0:
; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-V-NEXT: vle32.v v8, (a0)
; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-V-NEXT: vse32.v v8, (a1)
; CHECK-V-NEXT: ret
;
; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_v8i32_0:
; CHECK-KNOWNVLEN128: # %bb.0:
; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0)
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1)
; CHECK-KNOWNVLEN128-NEXT: ret
%a = load <8 x i32>, ptr %x
%c = call <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32> %a, i64 0)
store <2 x i32> %c, ptr %y
ret void
}

define void @extract_v2i32_v8i32_2(ptr %x, ptr %y) {
; CHECK-LABEL: extract_v2i32_v8i32_2:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 2, e32, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 2
; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT: vse32.v v8, (a1)
; CHECK-NEXT: ret
; CHECK-V-LABEL: extract_v2i32_v8i32_2:
; CHECK-V: # %bb.0:
; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-V-NEXT: vle32.v v8, (a0)
; CHECK-V-NEXT: vsetivli zero, 2, e32, m1, ta, ma
; CHECK-V-NEXT: vslidedown.vi v8, v8, 2
; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-V-NEXT: vse32.v v8, (a1)
; CHECK-V-NEXT: ret
;
; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_v8i32_2:
; CHECK-KNOWNVLEN128: # %bb.0:
; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0)
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, m1, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 2
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1)
; CHECK-KNOWNVLEN128-NEXT: ret
%a = load <8 x i32>, ptr %x
%c = call <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32> %a, i64 2)
store <2 x i32> %c, ptr %y
ret void
}

define void @extract_v2i32_v8i32_4(ptr %x, ptr %y) {
; CHECK-LABEL: extract_v2i32_v8i32_4:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 2, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 4
; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT: vse32.v v8, (a1)
; CHECK-NEXT: ret
; CHECK-V-LABEL: extract_v2i32_v8i32_4:
; CHECK-V: # %bb.0:
; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-V-NEXT: vle32.v v8, (a0)
; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, ta, ma
; CHECK-V-NEXT: vslidedown.vi v8, v8, 4
; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-V-NEXT: vse32.v v8, (a1)
; CHECK-V-NEXT: ret
;
; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_v8i32_4:
; CHECK-KNOWNVLEN128: # %bb.0:
; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0)
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, m2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 4
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1)
; CHECK-KNOWNVLEN128-NEXT: ret
%a = load <8 x i32>, ptr %x
%c = call <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32> %a, i64 4)
store <2 x i32> %c, ptr %y
ret void
}

define void @extract_v2i32_v8i32_6(ptr %x, ptr %y) {
; CHECK-LABEL: extract_v2i32_v8i32_6:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 2, e32, m2, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 6
; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-NEXT: vse32.v v8, (a1)
; CHECK-NEXT: ret
; CHECK-V-LABEL: extract_v2i32_v8i32_6:
; CHECK-V: # %bb.0:
; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-V-NEXT: vle32.v v8, (a0)
; CHECK-V-NEXT: vsetivli zero, 2, e32, m2, ta, ma
; CHECK-V-NEXT: vslidedown.vi v8, v8, 6
; CHECK-V-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-V-NEXT: vse32.v v8, (a1)
; CHECK-V-NEXT: ret
;
; CHECK-KNOWNVLEN128-LABEL: extract_v2i32_v8i32_6:
; CHECK-KNOWNVLEN128: # %bb.0:
; CHECK-KNOWNVLEN128-NEXT: vl2re32.v v8, (a0)
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, m2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 6
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vse32.v v8, (a1)
; CHECK-KNOWNVLEN128-NEXT: ret
%a = load <8 x i32>, ptr %x
%c = call <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32> %a, i64 6)
store <2 x i32> %c, ptr %y
Expand Down Expand Up @@ -271,13 +323,20 @@ define void @extract_v2i8_nxv2i8_6(<vscale x 2 x i8> %x, ptr %y) {
}

define void @extract_v8i32_nxv16i32_8(<vscale x 16 x i32> %x, ptr %y) {
; CHECK-LABEL: extract_v8i32_nxv16i32_8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 8, e32, m4, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v8, 8
; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-NEXT: vse32.v v8, (a0)
; CHECK-NEXT: ret
; CHECK-V-LABEL: extract_v8i32_nxv16i32_8:
; CHECK-V: # %bb.0:
; CHECK-V-NEXT: vsetivli zero, 8, e32, m4, ta, ma
; CHECK-V-NEXT: vslidedown.vi v8, v8, 8
; CHECK-V-NEXT: vsetivli zero, 8, e32, m2, ta, ma
; CHECK-V-NEXT: vse32.v v8, (a0)
; CHECK-V-NEXT: ret
;
; CHECK-KNOWNVLEN128-LABEL: extract_v8i32_nxv16i32_8:
; CHECK-KNOWNVLEN128: # %bb.0:
; CHECK-KNOWNVLEN128-NEXT: vsetivli zero, 8, e32, m4, ta, ma
; CHECK-KNOWNVLEN128-NEXT: vslidedown.vi v8, v8, 8
; CHECK-KNOWNVLEN128-NEXT: vs2r.v v8, (a0)
; CHECK-KNOWNVLEN128-NEXT: ret
%c = call <8 x i32> @llvm.vector.extract.v8i32.nxv16i32(<vscale x 16 x i32> %x, i64 8)
store <8 x i32> %c, ptr %y
ret void
Expand Down
12 changes: 6 additions & 6 deletions llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract.ll
Original file line number Diff line number Diff line change
Expand Up @@ -1141,10 +1141,10 @@ define float @extractelt_fdiv_v4f32(<4 x float> %x) {
define i32 @extractelt_v16i32_idx7_exact_vlen(ptr %x) nounwind vscale_range(2,2) {
; CHECK-LABEL: extractelt_v16i32_idx7_exact_vlen:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: addi a0, a0, 16
; CHECK-NEXT: vl1re32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v9, 3
; CHECK-NEXT: vslidedown.vi v8, v8, 3
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%a = load <16 x i32>, ptr %x
Expand All @@ -1155,10 +1155,10 @@ define i32 @extractelt_v16i32_idx7_exact_vlen(ptr %x) nounwind vscale_range(2,2)
define i32 @extractelt_v16i32_idx15_exact_vlen(ptr %x) nounwind vscale_range(2,2) {
; CHECK-LABEL: extractelt_v16i32_idx15_exact_vlen:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e32, m4, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: addi a0, a0, 48
; CHECK-NEXT: vl1re32.v v8, (a0)
; CHECK-NEXT: vsetivli zero, 1, e32, m1, ta, ma
; CHECK-NEXT: vslidedown.vi v8, v11, 3
; CHECK-NEXT: vslidedown.vi v8, v8, 3
; CHECK-NEXT: vmv.x.s a0, v8
; CHECK-NEXT: ret
%a = load <16 x i32>, ptr %x
Expand Down
15 changes: 5 additions & 10 deletions llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
Original file line number Diff line number Diff line change
Expand Up @@ -140,8 +140,7 @@ define <6 x i1> @load_v6i1(ptr %p) {
define <4 x i32> @exact_vlen_i32_m1(ptr %p) vscale_range(2,2) {
; CHECK-LABEL: exact_vlen_i32_m1:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 4, e32, m1, ta, ma
; CHECK-NEXT: vle32.v v8, (a0)
; CHECK-NEXT: vl1re32.v v8, (a0)
; CHECK-NEXT: ret
%v = load <4 x i32>, ptr %p
ret <4 x i32> %v
Expand All @@ -150,8 +149,7 @@ define <4 x i32> @exact_vlen_i32_m1(ptr %p) vscale_range(2,2) {
define <16 x i8> @exact_vlen_i8_m1(ptr %p) vscale_range(2,2) {
; CHECK-LABEL: exact_vlen_i8_m1:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vl1r.v v8, (a0)
; CHECK-NEXT: ret
%v = load <16 x i8>, ptr %p
ret <16 x i8> %v
Expand All @@ -160,8 +158,7 @@ define <16 x i8> @exact_vlen_i8_m1(ptr %p) vscale_range(2,2) {
define <32 x i8> @exact_vlen_i8_m2(ptr %p) vscale_range(2,2) {
; CHECK-LABEL: exact_vlen_i8_m2:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a1, zero, e8, m2, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vl2r.v v8, (a0)
; CHECK-NEXT: ret
%v = load <32 x i8>, ptr %p
ret <32 x i8> %v
Expand All @@ -170,8 +167,7 @@ define <32 x i8> @exact_vlen_i8_m2(ptr %p) vscale_range(2,2) {
define <128 x i8> @exact_vlen_i8_m8(ptr %p) vscale_range(2,2) {
; CHECK-LABEL: exact_vlen_i8_m8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetvli a1, zero, e8, m8, ta, ma
; CHECK-NEXT: vle8.v v8, (a0)
; CHECK-NEXT: vl8r.v v8, (a0)
; CHECK-NEXT: ret
%v = load <128 x i8>, ptr %p
ret <128 x i8> %v
Expand All @@ -180,8 +176,7 @@ define <128 x i8> @exact_vlen_i8_m8(ptr %p) vscale_range(2,2) {
define <16 x i64> @exact_vlen_i64_m8(ptr %p) vscale_range(2,2) {
; CHECK-LABEL: exact_vlen_i64_m8:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 16, e64, m8, ta, ma
; CHECK-NEXT: vle64.v v8, (a0)
; CHECK-NEXT: vl8re64.v v8, (a0)
; CHECK-NEXT: ret
%v = load <16 x i64>, ptr %p
ret <16 x i64> %v
Expand Down
Loading