Skip to content

Commit 6425f2d

Browse files
alexey-bataevalexey-bataev
authored
[SLP]Improve minbitwidth analysis for trun'ed gather nodes.
If the gather node is trunc'ed, better to trunc scalars and then gather them rather than gather and then trunc. Trunc for scalars is free in most cases. Reviewers: RKSimon Reviewed By: RKSimon Pull Request: #99072
1 parent 440fffa commit 6425f2d

File tree

6 files changed

+62
-47
lines changed

6 files changed

+62
-47
lines changed

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15502,8 +15502,21 @@ void BoUpSLP::computeMinimumValueSizes() {
1550215502
auto ComputeMaxBitWidth = [&](const TreeEntry &E, bool IsTopRoot,
1550315503
bool IsProfitableToDemoteRoot, unsigned Opcode,
1550415504
unsigned Limit, bool IsTruncRoot,
15505-
bool IsSignedCmp) {
15505+
bool IsSignedCmp) -> unsigned {
1550615506
ToDemote.clear();
15507+
// Check if the root is trunc and the next node is gather/buildvector, then
15508+
// keep trunc in scalars, which is free in most cases.
15509+
if (E.isGather() && IsTruncRoot && E.UserTreeIndices.size() == 1 &&
15510+
E.Idx > (IsStoreOrInsertElt ? 2 : 1)) {
15511+
ToDemote.push_back(E.Idx);
15512+
const TreeEntry *UserTE = E.UserTreeIndices.back().UserTE;
15513+
auto It = MinBWs.find(UserTE);
15514+
if (It != MinBWs.end())
15515+
return It->second.first;
15516+
return DL->getTypeSizeInBits(
15517+
E.UserTreeIndices.back().UserTE->Scalars.front()->getType());
15518+
}
15519+
1550715520
unsigned VF = E.getVectorFactor();
1550815521
auto *TreeRootIT = dyn_cast<IntegerType>(E.Scalars.front()->getType());
1550915522
if (!TreeRootIT || !Opcode)

‎llvm/test/Transforms/SLPVectorizer/X86/int-bitcast-minbitwidth.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ define void @t(i64 %v) {
55
; CHECK-LABEL: define void @t(
66
; CHECK-SAME: i64 [[V:%.*]]) {
77
; CHECK-NEXT: entry:
8-
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[V]], i32 0
9-
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
10-
; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i16>
8+
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[V]] to i16
9+
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i32 0
10+
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> poison, <4 x i32> zeroinitializer
1111
; CHECK-NEXT: [[TMP3:%.*]] = mul <4 x i16> [[TMP2]], <i16 2, i16 3, i16 6, i16 5>
1212
; CHECK-NEXT: [[TMP4:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP3]])
1313
; CHECK-NEXT: [[TMP5:%.*]] = sext i16 [[TMP4]] to i32

‎llvm/test/Transforms/SLPVectorizer/X86/minbitwidth-transformed-operand.ll

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,16 @@ define void @test(i64 %d.promoted.i) {
55
; CHECK-LABEL: define void @test(
66
; CHECK-SAME: i64 [[D_PROMOTED_I:%.*]]) {
77
; CHECK-NEXT: entry:
8-
; CHECK-NEXT: [[AND_1_I:%.*]] = and i64 0, [[D_PROMOTED_I]]
9-
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i64> <i64 poison, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, i64 [[AND_1_I]], i32 0
10-
; CHECK-NEXT: [[TMP1:%.*]] = trunc <8 x i64> [[TMP0]] to <8 x i1>
11-
; CHECK-NEXT: [[TMP2:%.*]] = mul <8 x i1> [[TMP1]], zeroinitializer
12-
; CHECK-NEXT: [[AND_1_I_1:%.*]] = and i64 0, 0
13-
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i64> <i64 poison, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, i64 [[AND_1_I_1]], i32 0
14-
; CHECK-NEXT: [[TMP4:%.*]] = trunc <8 x i64> [[TMP3]] to <8 x i1>
15-
; CHECK-NEXT: [[TMP5:%.*]] = mul <8 x i1> [[TMP4]], zeroinitializer
16-
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
17-
; CHECK-NEXT: [[TMP7:%.*]] = zext i1 [[TMP6]] to i32
18-
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]])
8+
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 0, i32 0
9+
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[D_PROMOTED_I]], i32 0
10+
; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i64> [[TMP0]], [[TMP1]]
11+
; CHECK-NEXT: [[TMP3:%.*]] = trunc <2 x i64> [[TMP2]] to <2 x i1>
12+
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i1> [[TMP3]], <2 x i1> poison, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
13+
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i1> <i1 poison, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <16 x i1> [[TMP4]], <16 x i32> <i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
14+
; CHECK-NEXT: [[TMP6:%.*]] = mul <16 x i1> [[TMP5]], zeroinitializer
15+
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP6]])
1916
; CHECK-NEXT: [[TMP9:%.*]] = zext i1 [[TMP8]] to i32
20-
; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 [[TMP7]], [[TMP9]]
21-
; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[OP_RDX]], 0
17+
; CHECK-NEXT: [[TMP10:%.*]] = and i32 [[TMP9]], 0
2218
; CHECK-NEXT: store i32 [[TMP10]], ptr null, align 4
2319
; CHECK-NEXT: ret void
2420
;

‎llvm/test/Transforms/SLPVectorizer/X86/resched.ll

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -11,26 +11,31 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv()
1111
; CHECK: if.then22.i:
1212
; CHECK-NEXT: [[SUB_I:%.*]] = add nsw i32 undef, -1
1313
; CHECK-NEXT: [[CONV31_I:%.*]] = and i32 undef, [[SUB_I]]
14-
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0
15-
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <4 x i32> zeroinitializer
16-
; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[TMP1]], <i32 1, i32 2, i32 3, i32 4>
17-
; CHECK-NEXT: [[SHR_4_I_I:%.*]] = lshr i32 [[CONV31_I]], 5
18-
; CHECK-NEXT: [[SHR_5_I_I:%.*]] = lshr i32 [[CONV31_I]], 6
19-
; CHECK-NEXT: [[SHR_6_I_I:%.*]] = lshr i32 [[CONV31_I]], 7
20-
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0
21-
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> zeroinitializer
22-
; CHECK-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[TMP4]], <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
23-
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <16 x i32> poison, i32 [[SUB_I]], i32 0
24-
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
25-
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i32> [[TMP6]], <16 x i32> [[TMP7]], <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
26-
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[SHR_4_I_I]], i32 5
27-
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[SHR_5_I_I]], i32 6
28-
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[SHR_6_I_I]], i32 7
29-
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
30-
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP11]], <16 x i32> [[TMP12]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
31-
; CHECK-NEXT: [[TMP14:%.*]] = trunc <16 x i32> [[TMP13]] to <16 x i8>
32-
; CHECK-NEXT: [[TMP15:%.*]] = and <16 x i8> [[TMP14]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
33-
; CHECK-NEXT: store <16 x i8> [[TMP15]], ptr undef, align 1
14+
; CHECK-NEXT: [[SHR_I_I:%.*]] = lshr i32 [[CONV31_I]], 1
15+
; CHECK-NEXT: [[SHR_1_I_I:%.*]] = lshr i32 [[CONV31_I]], 2
16+
; CHECK-NEXT: [[SHR_2_I_I:%.*]] = lshr i32 [[CONV31_I]], 3
17+
; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[SUB_I]] to i8
18+
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <16 x i8> poison, i8 [[TMP0]], i32 0
19+
; CHECK-NEXT: [[TMP2:%.*]] = trunc i32 [[SHR_I_I]] to i8
20+
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[TMP2]], i32 1
21+
; CHECK-NEXT: [[TMP4:%.*]] = trunc i32 [[SHR_1_I_I]] to i8
22+
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[TMP4]], i32 2
23+
; CHECK-NEXT: [[TMP6:%.*]] = trunc i32 [[SHR_2_I_I]] to i8
24+
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <16 x i8> [[TMP5]], i8 [[TMP6]], i32 3
25+
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> poison, i32 [[CONV31_I]], i32 0
26+
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> poison, <4 x i32> zeroinitializer
27+
; CHECK-NEXT: [[TMP10:%.*]] = lshr <4 x i32> [[TMP9]], <i32 4, i32 5, i32 6, i32 7>
28+
; CHECK-NEXT: [[TMP11:%.*]] = trunc <4 x i32> [[TMP10]] to <4 x i8>
29+
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i8> [[TMP11]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
30+
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP12]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
31+
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <8 x i32> poison, i32 [[CONV31_I]], i32 0
32+
; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP14]], <8 x i32> poison, <8 x i32> zeroinitializer
33+
; CHECK-NEXT: [[TMP16:%.*]] = lshr <8 x i32> [[TMP15]], <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
34+
; CHECK-NEXT: [[TMP17:%.*]] = trunc <8 x i32> [[TMP16]] to <8 x i8>
35+
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i8> [[TMP17]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
36+
; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <16 x i8> [[TMP13]], <16 x i8> [[TMP18]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
37+
; CHECK-NEXT: [[TMP20:%.*]] = and <16 x i8> [[TMP19]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
38+
; CHECK-NEXT: store <16 x i8> [[TMP20]], ptr undef, align 1
3439
; CHECK-NEXT: unreachable
3540
; CHECK: if.end50.i:
3641
; CHECK-NEXT: ret void

‎llvm/test/Transforms/SLPVectorizer/X86/shuffle-multivector.ll

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2-
; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-160 | FileCheck %s
2+
; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-163 | FileCheck %s
33

44
define void @test1(i128 %p0, i128 %p1, i128 %p2, i128 %p3, <4 x i128> %vec) {
55
; CHECK-LABEL: @test1(
@@ -14,13 +14,14 @@ define void @test1(i128 %p0, i128 %p1, i128 %p2, i128 %p3, <4 x i128> %vec) {
1414
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
1515
; CHECK-NEXT: [[T5:%.*]] = trunc i128 [[P1]] to i32
1616
; CHECK-NEXT: [[TMP8:%.*]] = sdiv <4 x i32> [[TMP3]], [[TMP7]]
17-
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i128> [[TMP1]], <2 x i128> [[TMP5]], <4 x i32> <i32 poison, i32 0, i32 3, i32 2>
18-
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i128> [[VEC:%.*]], <4 x i128> [[TMP9]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
19-
; CHECK-NEXT: [[TMP11:%.*]] = trunc <4 x i128> [[TMP10]] to <4 x i32>
20-
; CHECK-NEXT: [[TMP12:%.*]] = sdiv <4 x i32> [[TMP8]], [[TMP11]]
17+
; CHECK-NEXT: [[TMP9:%.*]] = trunc <4 x i128> [[VEC:%.*]] to <4 x i32>
18+
; CHECK-NEXT: [[TMP10:%.*]] = trunc <4 x i128> [[VEC]] to <4 x i32>
19+
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP6]], <4 x i32> <i32 poison, i32 0, i32 3, i32 2>
20+
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP10]], <4 x i32> [[TMP11]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
21+
; CHECK-NEXT: [[TMP13:%.*]] = sdiv <4 x i32> [[TMP8]], [[TMP12]]
2122
; CHECK-NEXT: br label [[BB:%.*]]
2223
; CHECK: bb:
23-
; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP12]], [[ENTRY:%.*]] ]
24+
; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP13]], [[ENTRY:%.*]] ]
2425
; CHECK-NEXT: ret void
2526
;
2627
entry:

‎llvm/test/Transforms/SLPVectorizer/orig-btiwidth-les-projected.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ define i32 @test(i4 %0) {
55
; CHECK-LABEL: define i32 @test(
66
; CHECK-SAME: i4 [[TMP0:%.*]]) {
77
; CHECK-NEXT: entry:
8-
; CHECK-NEXT: [[TMP1:%.*]] = trunc i8 0 to i4
9-
; CHECK-NEXT: [[TMP2:%.*]] = trunc i8 0 to i4
10-
; CHECK-NEXT: [[ADD_R:%.*]] = or i4 [[TMP1]], [[TMP0]]
11-
; CHECK-NEXT: [[ADD_R14:%.*]] = or i4 0, [[TMP2]]
8+
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i4> <i4 poison, i4 0>, i4 [[TMP0]], i32 0
9+
; CHECK-NEXT: [[TMP2:%.*]] = or <2 x i4> zeroinitializer, [[TMP1]]
10+
; CHECK-NEXT: [[ADD_R:%.*]] = extractelement <2 x i4> [[TMP2]], i32 0
11+
; CHECK-NEXT: [[ADD_R14:%.*]] = extractelement <2 x i4> [[TMP2]], i32 1
1212
; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i4 [[ADD_R]], [[ADD_R14]]
1313
; CHECK-NEXT: ret i32 0
1414
;

0 commit comments

Comments
 (0)