Skip to content

Commit f0b3b6d

Browse files
RKSimonRKSimon
committed
[DAG] isConstantIntBuildVectorOrConstantInt - peek through bitcasts (#112710) (REAPPLIED)
Alter both isConstantIntBuildVectorOrConstantInt + isConstantFPBuildVectorOrConstantFP to return a bool instead of the underlying SDNode, and adjust usage to account for this. Update isConstantIntBuildVectorOrConstantInt to peek though bitcasts when attempting to find a constant, in particular this improves canonicalization of constants to the RHS on commutable instructions. X86 is the beneficiary here as it often bitcasts rematerializable 0/-1 vector constants as vXi32 and bitcasts to the requested type Minor cleanup that helps with #107423 Reapplied after regression fix ba1255d
1 parent 94cddcf commit f0b3b6d

18 files changed

+570
-650
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAG.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2301,10 +2301,11 @@ class SelectionDAG {
23012301
Align getEVTAlign(EVT MemoryVT) const;
23022302

23032303
/// Test whether the given value is a constant int or similar node.
2304-
SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N) const;
2304+
bool isConstantIntBuildVectorOrConstantInt(SDValue N,
2305+
bool AllowOpaques = true) const;
23052306

23062307
/// Test whether the given value is a constant FP or similar node.
2307-
SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) const ;
2308+
bool isConstantFPBuildVectorOrConstantFP(SDValue N) const;
23082309

23092310
/// \returns true if \p N is any kind of constant or build_vector of
23102311
/// constants, int or float. If a vector, it may not necessarily be a splat.

‎llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1205,13 +1205,13 @@ SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
12051205
SDValue N00 = N0.getOperand(0);
12061206
SDValue N01 = N0.getOperand(1);
12071207

1208-
if (DAG.isConstantIntBuildVectorOrConstantInt(peekThroughBitcasts(N01))) {
1208+
if (DAG.isConstantIntBuildVectorOrConstantInt(N01)) {
12091209
SDNodeFlags NewFlags;
12101210
if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() &&
12111211
Flags.hasNoUnsignedWrap())
12121212
NewFlags.setNoUnsignedWrap(true);
12131213

1214-
if (DAG.isConstantIntBuildVectorOrConstantInt(peekThroughBitcasts(N1))) {
1214+
if (DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
12151215
// Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
12161216
if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1}))
12171217
return DAG.getNode(Opc, DL, VT, N00, OpNode, NewFlags);
@@ -9932,10 +9932,10 @@ SDValue DAGCombiner::visitRotate(SDNode *N) {
99329932
// fold (rot* (rot* x, c2), c1)
99339933
// -> (rot* x, ((c1 % bitsize) +- (c2 % bitsize) + bitsize) % bitsize)
99349934
if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
9935-
SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
9936-
SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
9937-
if (C1 && C2 && C1->getValueType(0) == C2->getValueType(0)) {
9938-
EVT ShiftVT = C1->getValueType(0);
9935+
bool C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
9936+
bool C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
9937+
if (C1 && C2 && N1.getValueType() == N0.getOperand(1).getValueType()) {
9938+
EVT ShiftVT = N1.getValueType();
99399939
bool SameSide = (N->getOpcode() == NextOp);
99409940
unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
99419941
SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
@@ -16807,8 +16807,8 @@ SDValue DAGCombiner::visitVP_FADD(SDNode *N) {
1680716807
SDValue DAGCombiner::visitFADD(SDNode *N) {
1680816808
SDValue N0 = N->getOperand(0);
1680916809
SDValue N1 = N->getOperand(1);
16810-
SDNode *N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N0);
16811-
SDNode *N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N1);
16810+
bool N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N0);
16811+
bool N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N1);
1681216812
EVT VT = N->getValueType(0);
1681316813
SDLoc DL(N);
1681416814
const TargetOptions &Options = DAG.getTarget().Options;
@@ -16905,10 +16905,8 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
1690516905
// of rounding steps.
1690616906
if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) {
1690716907
if (N0.getOpcode() == ISD::FMUL) {
16908-
SDNode *CFP00 =
16909-
DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0));
16910-
SDNode *CFP01 =
16911-
DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1));
16908+
bool CFP00 = DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0));
16909+
bool CFP01 = DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1));
1691216910

1691316911
// (fadd (fmul x, c), x) -> (fmul x, c+1)
1691416912
if (CFP01 && !CFP00 && N0.getOperand(0) == N1) {
@@ -16928,10 +16926,8 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
1692816926
}
1692916927

1693016928
if (N1.getOpcode() == ISD::FMUL) {
16931-
SDNode *CFP10 =
16932-
DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(0));
16933-
SDNode *CFP11 =
16934-
DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(1));
16929+
bool CFP10 = DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(0));
16930+
bool CFP11 = DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(1));
1693516931

1693616932
// (fadd x, (fmul x, c)) -> (fmul x, c+1)
1693716933
if (CFP11 && !CFP10 && N1.getOperand(0) == N0) {
@@ -16951,8 +16947,7 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
1695116947
}
1695216948

1695316949
if (N0.getOpcode() == ISD::FADD) {
16954-
SDNode *CFP00 =
16955-
DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0));
16950+
bool CFP00 = DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0));
1695616951
// (fadd (fadd x, x), x) -> (fmul x, 3.0)
1695716952
if (!CFP00 && N0.getOperand(0) == N0.getOperand(1) &&
1695816953
(N0.getOperand(0) == N1)) {
@@ -16962,8 +16957,7 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
1696216957
}
1696316958

1696416959
if (N1.getOpcode() == ISD::FADD) {
16965-
SDNode *CFP10 =
16966-
DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(0));
16960+
bool CFP10 = DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(0));
1696716961
// (fadd x, (fadd x, x)) -> (fmul x, 3.0)
1696816962
if (!CFP10 && N1.getOperand(0) == N1.getOperand(1) &&
1696916963
N1.getOperand(0) == N0) {

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7000,10 +7000,10 @@ void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
70007000

70017001
// Canonicalize:
70027002
// binop(const, nonconst) -> binop(nonconst, const)
7003-
SDNode *N1C = isConstantIntBuildVectorOrConstantInt(N1);
7004-
SDNode *N2C = isConstantIntBuildVectorOrConstantInt(N2);
7005-
SDNode *N1CFP = isConstantFPBuildVectorOrConstantFP(N1);
7006-
SDNode *N2CFP = isConstantFPBuildVectorOrConstantFP(N2);
7003+
bool N1C = isConstantIntBuildVectorOrConstantInt(N1);
7004+
bool N2C = isConstantIntBuildVectorOrConstantInt(N2);
7005+
bool N1CFP = isConstantFPBuildVectorOrConstantFP(N1);
7006+
bool N2CFP = isConstantFPBuildVectorOrConstantFP(N2);
70077007
if ((N1C && !N2C) || (N1CFP && !N2CFP))
70087008
std::swap(N1, N2);
70097009

@@ -13200,39 +13200,44 @@ bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
1320013200
return true;
1320113201
}
1320213202

13203-
// Returns the SDNode if it is a constant integer BuildVector
13204-
// or constant integer.
13205-
SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) const {
13206-
if (isa<ConstantSDNode>(N))
13207-
return N.getNode();
13203+
// Returns true if it is a constant integer BuildVector or constant integer,
13204+
// possibly hidden by a bitcast.
13205+
bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
13206+
SDValue N, bool AllowOpaques) const {
13207+
N = peekThroughBitcasts(N);
13208+
13209+
if (auto *C = dyn_cast<ConstantSDNode>(N))
13210+
return AllowOpaques || !C->isOpaque();
13211+
1320813212
if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
13209-
return N.getNode();
13213+
return true;
13214+
1321013215
// Treat a GlobalAddress supporting constant offset folding as a
1321113216
// constant integer.
13212-
if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
13217+
if (auto *GA = dyn_cast<GlobalAddressSDNode>(N))
1321313218
if (GA->getOpcode() == ISD::GlobalAddress &&
1321413219
TLI->isOffsetFoldingLegal(GA))
13215-
return GA;
13220+
return true;
13221+
1321613222
if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
1321713223
isa<ConstantSDNode>(N.getOperand(0)))
13218-
return N.getNode();
13219-
return nullptr;
13224+
return true;
13225+
return false;
1322013226
}
1322113227

13222-
// Returns the SDNode if it is a constant float BuildVector
13223-
// or constant float.
13224-
SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
13228+
// Returns true if it is a constant float BuildVector or constant float.
13229+
bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
1322513230
if (isa<ConstantFPSDNode>(N))
13226-
return N.getNode();
13231+
return true;
1322713232

1322813233
if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
13229-
return N.getNode();
13234+
return true;
1323013235

1323113236
if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
1323213237
isa<ConstantFPSDNode>(N.getOperand(0)))
13233-
return N.getNode();
13238+
return true;
1323413239

13235-
return nullptr;
13240+
return false;
1323613241
}
1323713242

1323813243
std::optional<bool> SelectionDAG::isBoolConstant(SDValue N,

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20760,7 +20760,7 @@ static SDValue performSubAddMULCombine(SDNode *N, SelectionDAG &DAG) {
2076020760

2076120761
if (!Add.hasOneUse())
2076220762
return SDValue();
20763-
if (DAG.isConstantIntBuildVectorOrConstantInt(peekThroughBitcasts(X)))
20763+
if (DAG.isConstantIntBuildVectorOrConstantInt(X))
2076420764
return SDValue();
2076520765

2076620766
SDValue M1 = Add.getOperand(0);

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -56546,14 +56546,9 @@ static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
5654656546
SDValue Op1 = N->getOperand(1);
5654756547
SDLoc DL(N);
5654856548

56549-
// TODO: Add NoOpaque handling to isConstantIntBuildVectorOrConstantInt.
5655056549
auto IsNonOpaqueConstant = [&](SDValue Op) {
56551-
if (SDNode *C = DAG.isConstantIntBuildVectorOrConstantInt(Op)) {
56552-
if (auto *Cst = dyn_cast<ConstantSDNode>(C))
56553-
return !Cst->isOpaque();
56554-
return true;
56555-
}
56556-
return false;
56550+
return DAG.isConstantIntBuildVectorOrConstantInt(Op,
56551+
/*AllowOpaques*/ false);
5655756552
};
5655856553

5655956554
// X86 can't encode an immediate LHS of a sub. See if we can push the

‎llvm/test/CodeGen/X86/avx2-arith.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ define <32 x i8> @mul_v32i8(<32 x i8> %i, <32 x i8> %j) nounwind readnone {
122122
; CHECK-LABEL: mul_v32i8:
123123
; CHECK: # %bb.0:
124124
; CHECK-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
125-
; CHECK-NEXT: vpand %ymm1, %ymm2, %ymm3
125+
; CHECK-NEXT: vpand %ymm2, %ymm1, %ymm3
126126
; CHECK-NEXT: vpmaddubsw %ymm3, %ymm0, %ymm3
127127
; CHECK-NEXT: vpand %ymm2, %ymm3, %ymm3
128128
; CHECK-NEXT: vpandn %ymm1, %ymm2, %ymm1

‎llvm/test/CodeGen/X86/combine-sra.ll

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -725,12 +725,11 @@ define <4 x i64> @combine_vec4i64_ashr_clamped(<4 x i64> %x, <4 x i64> %y) {
725725
; SSE41: # %bb.0:
726726
; SSE41-NEXT: movdqa %xmm0, %xmm4
727727
; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456]
728-
; SSE41-NEXT: movdqa %xmm3, %xmm0
729-
; SSE41-NEXT: pxor %xmm7, %xmm0
728+
; SSE41-NEXT: movdqa %xmm3, %xmm6
729+
; SSE41-NEXT: pxor %xmm7, %xmm6
730730
; SSE41-NEXT: movdqa {{.*#+}} xmm8 = [9223372039002259519,9223372039002259519]
731-
; SSE41-NEXT: movdqa %xmm8, %xmm6
732-
; SSE41-NEXT: pcmpeqd %xmm0, %xmm6
733-
; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
731+
; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm6[0,0,2,2]
732+
; SSE41-NEXT: pcmpeqd %xmm8, %xmm6
734733
; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483711,2147483711,2147483711,2147483711]
735734
; SSE41-NEXT: movdqa %xmm5, %xmm0
736735
; SSE41-NEXT: pcmpgtd %xmm9, %xmm0

0 commit comments

Comments
 (0)