@@ -1745,13 +1745,6 @@ bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1745
1745
Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1746
1746
MachineMemOperand::MOVolatile;
1747
1747
return true;
1748
- case Intrinsic::riscv_seg2_load:
1749
- case Intrinsic::riscv_seg3_load:
1750
- case Intrinsic::riscv_seg4_load:
1751
- case Intrinsic::riscv_seg5_load:
1752
- case Intrinsic::riscv_seg6_load:
1753
- case Intrinsic::riscv_seg7_load:
1754
- case Intrinsic::riscv_seg8_load:
1755
1748
case Intrinsic::riscv_seg2_load_mask:
1756
1749
case Intrinsic::riscv_seg3_load_mask:
1757
1750
case Intrinsic::riscv_seg4_load_mask:
@@ -1761,17 +1754,6 @@ bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1761
1754
case Intrinsic::riscv_seg8_load_mask:
1762
1755
return SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
1763
1756
/*IsUnitStrided*/ false, /*UsePtrVal*/ true);
1764
- case Intrinsic::riscv_seg2_store:
1765
- case Intrinsic::riscv_seg3_store:
1766
- case Intrinsic::riscv_seg4_store:
1767
- case Intrinsic::riscv_seg5_store:
1768
- case Intrinsic::riscv_seg6_store:
1769
- case Intrinsic::riscv_seg7_store:
1770
- case Intrinsic::riscv_seg8_store:
1771
- // Operands are (vec, ..., vec, ptr, vl)
1772
- return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1773
- /*IsStore*/ true,
1774
- /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
1775
1757
case Intrinsic::riscv_seg2_store_mask:
1776
1758
case Intrinsic::riscv_seg3_store_mask:
1777
1759
case Intrinsic::riscv_seg4_store_mask:
@@ -10573,13 +10555,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
10573
10555
switch (IntNo) {
10574
10556
default:
10575
10557
break;
10576
- case Intrinsic::riscv_seg2_load:
10577
- case Intrinsic::riscv_seg3_load:
10578
- case Intrinsic::riscv_seg4_load:
10579
- case Intrinsic::riscv_seg5_load:
10580
- case Intrinsic::riscv_seg6_load:
10581
- case Intrinsic::riscv_seg7_load:
10582
- case Intrinsic::riscv_seg8_load:
10583
10558
case Intrinsic::riscv_seg2_load_mask:
10584
10559
case Intrinsic::riscv_seg3_load_mask:
10585
10560
case Intrinsic::riscv_seg4_load_mask:
@@ -10602,18 +10577,13 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
10602
10577
ContainerVT.getScalarSizeInBits();
10603
10578
EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
10604
10579
10605
- // Masked: (pointer, mask, vl)
10606
- // Non-masked: (pointer, vl)
10607
- bool IsMasked = Op.getNumOperands() > 4;
10580
+ // Operands: (chain, int_id, pointer, mask, vl)
10608
10581
SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
10609
- SDValue Mask =
10610
- IsMasked ? Op.getOperand(3) : getAllOnesMask(ContainerVT, VL, DL, DAG);
10582
+ SDValue Mask = Op.getOperand(3);
10611
10583
MVT MaskVT = Mask.getSimpleValueType();
10612
- if (MaskVT.isFixedLengthVector()) {
10613
- MVT MaskContainerVT =
10614
- ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10615
- Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
10616
- }
10584
+ MVT MaskContainerVT =
10585
+ ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10586
+ Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
10617
10587
10618
10588
SDValue IntID = DAG.getTargetConstant(VlsegInts[NF - 2], DL, XLenVT);
10619
10589
auto *Load = cast<MemIntrinsicSDNode>(Op);
@@ -10681,13 +10651,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
10681
10651
switch (IntNo) {
10682
10652
default:
10683
10653
break;
10684
- case Intrinsic::riscv_seg2_store:
10685
- case Intrinsic::riscv_seg3_store:
10686
- case Intrinsic::riscv_seg4_store:
10687
- case Intrinsic::riscv_seg5_store:
10688
- case Intrinsic::riscv_seg6_store:
10689
- case Intrinsic::riscv_seg7_store:
10690
- case Intrinsic::riscv_seg8_store:
10691
10654
case Intrinsic::riscv_seg2_store_mask:
10692
10655
case Intrinsic::riscv_seg3_store_mask:
10693
10656
case Intrinsic::riscv_seg4_store_mask:
@@ -10702,24 +10665,8 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
10702
10665
Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
10703
10666
Intrinsic::riscv_vsseg8_mask};
10704
10667
10705
- bool IsMasked = false;
10706
- switch (IntNo) {
10707
- case Intrinsic::riscv_seg2_store_mask:
10708
- case Intrinsic::riscv_seg3_store_mask:
10709
- case Intrinsic::riscv_seg4_store_mask:
10710
- case Intrinsic::riscv_seg5_store_mask:
10711
- case Intrinsic::riscv_seg6_store_mask:
10712
- case Intrinsic::riscv_seg7_store_mask:
10713
- case Intrinsic::riscv_seg8_store_mask:
10714
- IsMasked = true;
10715
- break;
10716
- default:
10717
- break;
10718
- }
10719
-
10720
- // Non-masked: (chain, int_id, vec*, ptr, vl)
10721
- // Masked: (chain, int_id, vec*, ptr, mask, vl)
10722
- unsigned NF = Op->getNumOperands() - (IsMasked ? 5 : 4);
10668
+ // Operands: (chain, int_id, vec*, ptr, mask, vl)
10669
+ unsigned NF = Op->getNumOperands() - 5;
10723
10670
assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
10724
10671
MVT XLenVT = Subtarget.getXLenVT();
10725
10672
MVT VT = Op->getOperand(2).getSimpleValueType();
@@ -10729,14 +10676,11 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
10729
10676
EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
10730
10677
10731
10678
SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
10732
- SDValue Mask = IsMasked ? Op.getOperand(Op.getNumOperands() - 2)
10733
- : getAllOnesMask(ContainerVT, VL, DL, DAG);
10679
+ SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
10734
10680
MVT MaskVT = Mask.getSimpleValueType();
10735
- if (MaskVT.isFixedLengthVector()) {
10736
- MVT MaskContainerVT =
10737
- ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10738
- Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
10739
- }
10681
+ MVT MaskContainerVT =
10682
+ ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10683
+ Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
10740
10684
10741
10685
SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
10742
10686
SDValue Ptr = Op->getOperand(NF + 2);
@@ -23781,10 +23725,10 @@ bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
23781
23725
}
23782
23726
23783
23727
static const Intrinsic::ID FixedVlsegIntrIds[] = {
23784
- Intrinsic::riscv_seg2_load , Intrinsic::riscv_seg3_load ,
23785
- Intrinsic::riscv_seg4_load , Intrinsic::riscv_seg5_load ,
23786
- Intrinsic::riscv_seg6_load , Intrinsic::riscv_seg7_load ,
23787
- Intrinsic::riscv_seg8_load };
23728
+ Intrinsic::riscv_seg2_load_mask , Intrinsic::riscv_seg3_load_mask ,
23729
+ Intrinsic::riscv_seg4_load_mask , Intrinsic::riscv_seg5_load_mask ,
23730
+ Intrinsic::riscv_seg6_load_mask , Intrinsic::riscv_seg7_load_mask ,
23731
+ Intrinsic::riscv_seg8_load_mask };
23788
23732
23789
23733
/// Lower an interleaved load into a vlsegN intrinsic.
23790
23734
///
@@ -23835,10 +23779,10 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
23835
23779
};
23836
23780
23837
23781
Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
23838
-
23839
- CallInst *VlsegN = Builder.CreateIntrinsic(
23840
- FixedVlsegIntrIds[Factor - 2], {VTy, LI->getPointerOperandType() , XLenTy},
23841
- {LI->getPointerOperand(), VL});
23782
+ Value *Mask = Builder.getAllOnesMask(VTy->getElementCount());
23783
+ CallInst *VlsegN =
23784
+ Builder.CreateIntrinsic( FixedVlsegIntrIds[Factor - 2], {VTy, XLenTy},
23785
+ {LI->getPointerOperand(), Mask , VL});
23842
23786
23843
23787
for (unsigned i = 0; i < Shuffles.size(); i++) {
23844
23788
Value *SubVec = Builder.CreateExtractValue(VlsegN, Indices[i]);
@@ -23849,10 +23793,10 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
23849
23793
}
23850
23794
23851
23795
static const Intrinsic::ID FixedVssegIntrIds[] = {
23852
- Intrinsic::riscv_seg2_store , Intrinsic::riscv_seg3_store ,
23853
- Intrinsic::riscv_seg4_store , Intrinsic::riscv_seg5_store ,
23854
- Intrinsic::riscv_seg6_store , Intrinsic::riscv_seg7_store ,
23855
- Intrinsic::riscv_seg8_store };
23796
+ Intrinsic::riscv_seg2_store_mask , Intrinsic::riscv_seg3_store_mask ,
23797
+ Intrinsic::riscv_seg4_store_mask , Intrinsic::riscv_seg5_store_mask ,
23798
+ Intrinsic::riscv_seg6_store_mask , Intrinsic::riscv_seg7_store_mask ,
23799
+ Intrinsic::riscv_seg8_store_mask };
23856
23800
23857
23801
/// Lower an interleaved store into a vssegN intrinsic.
23858
23802
///
@@ -23912,8 +23856,7 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
23912
23856
}
23913
23857
23914
23858
Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
23915
- SI->getModule(), FixedVssegIntrIds[Factor - 2],
23916
- {VTy, SI->getPointerOperandType(), XLenTy});
23859
+ SI->getModule(), FixedVssegIntrIds[Factor - 2], {VTy, XLenTy});
23917
23860
23918
23861
SmallVector<Value *, 10> Ops;
23919
23862
SmallVector<int, 16> NewShuffleMask;
@@ -23933,7 +23876,8 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
23933
23876
// potentially under larger LMULs) because we checked that the fixed vector
23934
23877
// type fits in isLegalInterleavedAccessType
23935
23878
Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
23936
- Ops.append({SI->getPointerOperand(), VL});
23879
+ Value *StoreMask = Builder.getAllOnesMask(VTy->getElementCount());
23880
+ Ops.append({SI->getPointerOperand(), StoreMask, VL});
23937
23881
23938
23882
Builder.CreateCall(VssegNFunc, Ops);
23939
23883
@@ -23962,10 +23906,10 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
23962
23906
23963
23907
if (auto *FVTy = dyn_cast<FixedVectorType>(ResVTy)) {
23964
23908
Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
23909
+ Value *Mask = Builder.getAllOnesMask(FVTy->getElementCount());
23965
23910
Return =
23966
- Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2],
23967
- {ResVTy, LI->getPointerOperandType(), XLenTy},
23968
- {LI->getPointerOperand(), VL});
23911
+ Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2], {ResVTy, XLenTy},
23912
+ {LI->getPointerOperand(), Mask, VL});
23969
23913
} else {
23970
23914
static const Intrinsic::ID IntrIds[] = {
23971
23915
Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
@@ -24029,12 +23973,12 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
24029
23973
24030
23974
if (auto *FVTy = dyn_cast<FixedVectorType>(InVTy)) {
24031
23975
Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
24032
- SI->getModule(), FixedVssegIntrIds[Factor - 2],
24033
- {InVTy, SI->getPointerOperandType(), XLenTy});
23976
+ SI->getModule(), FixedVssegIntrIds[Factor - 2], {InVTy, XLenTy});
24034
23977
24035
23978
SmallVector<Value *, 10> Ops(InterleaveValues);
24036
23979
Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
24037
- Ops.append({SI->getPointerOperand(), VL});
23980
+ Value *Mask = Builder.getAllOnesMask(FVTy->getElementCount());
23981
+ Ops.append({SI->getPointerOperand(), Mask, VL});
24038
23982
24039
23983
Builder.CreateCall(VssegNFunc, Ops);
24040
23984
} else {
@@ -24156,15 +24100,9 @@ bool RISCVTargetLowering::lowerInterleavedVPLoad(
24156
24100
24157
24101
Value *Return = nullptr;
24158
24102
if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
24159
- static const Intrinsic::ID FixedMaskedVlsegIntrIds[] = {
24160
- Intrinsic::riscv_seg2_load_mask, Intrinsic::riscv_seg3_load_mask,
24161
- Intrinsic::riscv_seg4_load_mask, Intrinsic::riscv_seg5_load_mask,
24162
- Intrinsic::riscv_seg6_load_mask, Intrinsic::riscv_seg7_load_mask,
24163
- Intrinsic::riscv_seg8_load_mask};
24164
-
24165
- Return = Builder.CreateIntrinsic(FixedMaskedVlsegIntrIds[Factor - 2],
24166
- {FVTy, XLenTy},
24167
- {Load->getArgOperand(0), Mask, EVL});
24103
+ Return =
24104
+ Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2], {FVTy, XLenTy},
24105
+ {Load->getArgOperand(0), Mask, EVL});
24168
24106
} else {
24169
24107
static const Intrinsic::ID IntrMaskIds[] = {
24170
24108
Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
@@ -24276,15 +24214,9 @@ bool RISCVTargetLowering::lowerInterleavedVPStore(
24276
24214
XLenTy);
24277
24215
24278
24216
if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
24279
- static const Intrinsic::ID FixedMaskedVssegIntrIds[] = {
24280
- Intrinsic::riscv_seg2_store_mask, Intrinsic::riscv_seg3_store_mask,
24281
- Intrinsic::riscv_seg4_store_mask, Intrinsic::riscv_seg5_store_mask,
24282
- Intrinsic::riscv_seg6_store_mask, Intrinsic::riscv_seg7_store_mask,
24283
- Intrinsic::riscv_seg8_store_mask};
24284
-
24285
24217
SmallVector<Value *, 8> Operands(InterleaveOperands);
24286
24218
Operands.append({Store->getArgOperand(1), Mask, EVL});
24287
- Builder.CreateIntrinsic(FixedMaskedVssegIntrIds [Factor - 2], {FVTy, XLenTy},
24219
+ Builder.CreateIntrinsic(FixedVssegIntrIds [Factor - 2], {FVTy, XLenTy},
24288
24220
Operands);
24289
24221
return true;
24290
24222
}
0 commit comments