Skip to content

Commit bc349ce

Browse files
author
Thorsten Schütt
authored
[GlobalIsel] combine insert vector element (#89363)
preliminary steps poison symbols
1 parent 9145514 commit bc349ce

File tree

6 files changed

+277
-3
lines changed

6 files changed

+277
-3
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -853,6 +853,9 @@ class CombinerHelper {
853853
bool matchExtractVectorElementWithDifferentIndices(const MachineOperand &MO,
854854
BuildFnTy &MatchInfo);
855855

856+
/// Combine insert vector element OOB.
857+
bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo);
858+
856859
private:
857860
/// Checks for legality of an indexed variant of \p LdSt.
858861
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;

‎llvm/include/llvm/CodeGen/GlobalISel/Utils.h

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -559,5 +559,31 @@ void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI);
559559
/// having only floating-point operands.
560560
bool isPreISelGenericFloatingPointOpcode(unsigned Opc);
561561

562+
/// Returns true if \p Reg can create undef or poison from non-undef &
563+
/// non-poison operands. \p ConsiderFlagsAndMetadata controls whether poison
564+
/// producing flags and metadata on the instruction are considered. This can be
565+
/// used to see if the instruction could still introduce undef or poison even
566+
/// without poison generating flags and metadata which might be on the
567+
/// instruction.
568+
bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
569+
bool ConsiderFlagsAndMetadata = true);
570+
571+
/// Returns true if \p Reg can create poison from non-poison operands.
572+
bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
573+
bool ConsiderFlagsAndMetadata = true);
574+
575+
/// Returns true if \p Reg cannot be poison and undef.
576+
bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
577+
const MachineRegisterInfo &MRI,
578+
unsigned Depth = 0);
579+
580+
/// Returns true if \p Reg cannot be poison, but may be undef.
581+
bool isGuaranteedNotToBePoison(Register Reg, const MachineRegisterInfo &MRI,
582+
unsigned Depth = 0);
583+
584+
/// Returns true if \p Reg cannot be undef, but may be poison.
585+
bool isGuaranteedNotToBeUndef(Register Reg, const MachineRegisterInfo &MRI,
586+
unsigned Depth = 0);
587+
562588
} // End namespace llvm.
563589
#endif

‎llvm/include/llvm/Target/GlobalISel/Combine.td

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1525,11 +1525,39 @@ def combine_shuffle_concat : GICombineRule<
15251525
[{ return Helper.matchCombineShuffleConcat(*${root}, ${matchinfo}); }]),
15261526
(apply [{ Helper.applyCombineShuffleConcat(*${root}, ${matchinfo}); }])>;
15271527

1528-
// match_extract_of_element must be the first!
1528+
def insert_vector_element_idx_undef : GICombineRule<
1529+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1530+
(match (G_IMPLICIT_DEF $idx),
1531+
(G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
1532+
(apply (G_IMPLICIT_DEF $root))>;
1533+
1534+
def insert_vector_element_elt_undef : GICombineRule<
1535+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1536+
(match (G_IMPLICIT_DEF $elt),
1537+
(G_INSERT_VECTOR_ELT $root, $src, $elt, $idx),
1538+
[{ return isGuaranteedNotToBePoison(${src}.getReg(), MRI); }]),
1539+
(apply (GIReplaceReg $root, $src))>;
1540+
1541+
def insert_vector_element_extract_vector_element : GICombineRule<
1542+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1543+
(match (G_EXTRACT_VECTOR_ELT $elt, $src, $idx),
1544+
(G_INSERT_VECTOR_ELT $root, $src, $elt, $idx)),
1545+
(apply (GIReplaceReg $root, $src))>;
1546+
1547+
def insert_vector_elt_oob : GICombineRule<
1548+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1549+
(match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
1550+
[{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]),
1551+
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
1552+
1553+
// match_extract_of_element and insert_vector_elt_oob must be the first!
15291554
def vector_ops_combines: GICombineGroup<[
15301555
match_extract_of_element_undef_vector,
15311556
match_extract_of_element_undef_index,
1557+
insert_vector_element_idx_undef,
1558+
insert_vector_element_elt_undef,
15321559
match_extract_of_element,
1560+
insert_vector_elt_oob,
15331561
extract_vector_element_not_const,
15341562
extract_vector_element_different_indices,
15351563
extract_vector_element_build_vector2,
@@ -1553,7 +1581,8 @@ extract_vector_element_build_vector_trunc5,
15531581
extract_vector_element_build_vector_trunc6,
15541582
extract_vector_element_build_vector_trunc7,
15551583
extract_vector_element_build_vector_trunc8,
1556-
extract_vector_element_freeze
1584+
extract_vector_element_freeze,
1585+
insert_vector_element_extract_vector_element
15571586
]>;
15581587

15591588
// FIXME: These should use the custom predicate feature once it lands.

‎llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ bool CombinerHelper::matchExtractVectorElement(MachineInstr &MI,
7777

7878
// Fold extractVectorElement(Vector, TOOLARGE) -> undef
7979
if (IndexC && VectorTy.isFixedVector() &&
80-
IndexC->getZExtValue() >= VectorTy.getNumElements() &&
80+
IndexC->uge(VectorTy.getNumElements()) &&
8181
isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
8282
// For fixed-length vectors, it's invalid to extract out-of-range elements.
8383
MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); };
@@ -324,3 +324,26 @@ bool CombinerHelper::matchExtractVectorElementWithBuildVectorTrunc(
324324

325325
return true;
326326
}
327+
328+
bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI,
329+
BuildFnTy &MatchInfo) {
330+
GInsertVectorElement *Insert = cast<GInsertVectorElement>(&MI);
331+
332+
Register Dst = Insert->getReg(0);
333+
LLT DstTy = MRI.getType(Dst);
334+
Register Index = Insert->getIndexReg();
335+
336+
if (!DstTy.isFixedVector())
337+
return false;
338+
339+
std::optional<ValueAndVReg> MaybeIndex =
340+
getIConstantVRegValWithLookThrough(Index, MRI);
341+
342+
if (MaybeIndex && MaybeIndex->Value.uge(DstTy.getNumElements()) &&
343+
isLegalOrBeforeLegalizer({TargetOpcode::G_IMPLICIT_DEF, {DstTy}})) {
344+
MatchInfo = [=](MachineIRBuilder &B) { B.buildUndef(Dst); };
345+
return true;
346+
}
347+
348+
return false;
349+
}

‎llvm/lib/CodeGen/GlobalISel/Utils.cpp

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "llvm/CodeGen/GlobalISel/Utils.h"
1313
#include "llvm/ADT/APFloat.h"
1414
#include "llvm/ADT/APInt.h"
15+
#include "llvm/Analysis/ValueTracking.h"
1516
#include "llvm/CodeGen/CodeGenCommonISel.h"
1617
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
1718
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
@@ -28,6 +29,7 @@
2829
#include "llvm/CodeGen/StackProtector.h"
2930
#include "llvm/CodeGen/TargetInstrInfo.h"
3031
#include "llvm/CodeGen/TargetLowering.h"
32+
#include "llvm/CodeGen/TargetOpcodes.h"
3133
#include "llvm/CodeGen/TargetPassConfig.h"
3234
#include "llvm/CodeGen/TargetRegisterInfo.h"
3335
#include "llvm/IR/Constants.h"
@@ -1709,3 +1711,84 @@ bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
17091711
return false;
17101712
}
17111713
}
1714+
1715+
namespace {
1716+
enum class UndefPoisonKind {
1717+
PoisonOnly = (1 << 0),
1718+
UndefOnly = (1 << 1),
1719+
UndefOrPoison = PoisonOnly | UndefOnly,
1720+
};
1721+
}
1722+
1723+
[[maybe_unused]] static bool includesPoison(UndefPoisonKind Kind) {
1724+
return (unsigned(Kind) & unsigned(UndefPoisonKind::PoisonOnly)) != 0;
1725+
}
1726+
1727+
[[maybe_unused]] static bool includesUndef(UndefPoisonKind Kind) {
1728+
return (unsigned(Kind) & unsigned(UndefPoisonKind::UndefOnly)) != 0;
1729+
}
1730+
1731+
static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1732+
bool ConsiderFlagsAndMetadata,
1733+
UndefPoisonKind Kind) {
1734+
MachineInstr *RegDef = MRI.getVRegDef(Reg);
1735+
1736+
switch (RegDef->getOpcode()) {
1737+
case TargetOpcode::G_FREEZE:
1738+
return false;
1739+
default:
1740+
return true;
1741+
}
1742+
}
1743+
1744+
static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
1745+
const MachineRegisterInfo &MRI,
1746+
unsigned Depth,
1747+
UndefPoisonKind Kind) {
1748+
if (Depth >= MaxAnalysisRecursionDepth)
1749+
return false;
1750+
1751+
MachineInstr *RegDef = MRI.getVRegDef(Reg);
1752+
1753+
switch (RegDef->getOpcode()) {
1754+
case TargetOpcode::G_FREEZE:
1755+
return true;
1756+
case TargetOpcode::G_IMPLICIT_DEF:
1757+
return !includesUndef(Kind);
1758+
default:
1759+
return false;
1760+
}
1761+
}
1762+
1763+
bool llvm::canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
1764+
bool ConsiderFlagsAndMetadata) {
1765+
return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1766+
UndefPoisonKind::UndefOrPoison);
1767+
}
1768+
1769+
bool canCreatePoison(Register Reg, const MachineRegisterInfo &MRI,
1770+
bool ConsiderFlagsAndMetadata = true) {
1771+
return ::canCreateUndefOrPoison(Reg, MRI, ConsiderFlagsAndMetadata,
1772+
UndefPoisonKind::PoisonOnly);
1773+
}
1774+
1775+
bool llvm::isGuaranteedNotToBeUndefOrPoison(Register Reg,
1776+
const MachineRegisterInfo &MRI,
1777+
unsigned Depth) {
1778+
return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1779+
UndefPoisonKind::UndefOrPoison);
1780+
}
1781+
1782+
bool llvm::isGuaranteedNotToBePoison(Register Reg,
1783+
const MachineRegisterInfo &MRI,
1784+
unsigned Depth) {
1785+
return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1786+
UndefPoisonKind::PoisonOnly);
1787+
}
1788+
1789+
bool llvm::isGuaranteedNotToBeUndef(Register Reg,
1790+
const MachineRegisterInfo &MRI,
1791+
unsigned Depth) {
1792+
return ::isGuaranteedNotToBeUndefOrPoison(Reg, MRI, Depth,
1793+
UndefPoisonKind::UndefOnly);
1794+
}

‎llvm/test/CodeGen/AArch64/GlobalISel/combine-insert-vec-elt.mir

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,3 +201,113 @@ body: |
201201
RET_ReallyLR
202202
203203
...
204+
---
205+
name: test_idx_undef
206+
body: |
207+
bb.1:
208+
liveins: $x0
209+
; CHECK-LABEL: name: test_idx_undef
210+
; CHECK: liveins: $x0
211+
; CHECK-NEXT: {{ $}}
212+
; CHECK-NEXT: RET_ReallyLR
213+
%3:_(s8) = G_CONSTANT i8 127
214+
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
215+
%4:_(s8) = G_CONSTANT i8 -128
216+
%5:_(s64) = G_IMPLICIT_DEF
217+
%0:_(p0) = COPY $x0
218+
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
219+
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
220+
RET_ReallyLR
221+
222+
...
223+
---
224+
name: test_elt_undef
225+
body: |
226+
bb.1:
227+
liveins: $x0
228+
; CHECK-LABEL: name: test_elt_undef
229+
; CHECK: liveins: $x0
230+
; CHECK-NEXT: {{ $}}
231+
; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127
232+
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s8) = G_IMPLICIT_DEF
233+
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
234+
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[DEF]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8)
235+
; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>))
236+
; CHECK-NEXT: RET_ReallyLR
237+
%3:_(s8) = G_CONSTANT i8 127
238+
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
239+
%4:_(s8) = G_IMPLICIT_DEF
240+
%5:_(s64) = G_CONSTANT i64 3
241+
%0:_(p0) = COPY $x0
242+
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
243+
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
244+
RET_ReallyLR
245+
246+
...
247+
---
248+
name: test_elt_undef_with_freeze
249+
body: |
250+
bb.1:
251+
liveins: $x0
252+
; CHECK-LABEL: name: test_elt_undef_with_freeze
253+
; CHECK: liveins: $x0
254+
; CHECK-NEXT: {{ $}}
255+
; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127
256+
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8)
257+
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
258+
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<32 x s8>) = G_FREEZE [[BUILD_VECTOR]]
259+
; CHECK-NEXT: G_STORE [[FREEZE]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>))
260+
; CHECK-NEXT: RET_ReallyLR
261+
%3:_(s8) = G_CONSTANT i8 127
262+
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
263+
%4:_(s8) = G_IMPLICIT_DEF
264+
%5:_(s64) = G_CONSTANT i64 3
265+
%0:_(p0) = COPY $x0
266+
%9:_(<32 x s8>) = G_FREEZE %2
267+
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %9, %4(s8), %5(s64)
268+
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
269+
RET_ReallyLR
270+
271+
...
272+
---
273+
name: test_insert_extract
274+
body: |
275+
bb.1:
276+
liveins: $x0
277+
; CHECK-LABEL: name: test_insert_extract
278+
; CHECK: liveins: $x0
279+
; CHECK-NEXT: {{ $}}
280+
; CHECK-NEXT: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 127
281+
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
282+
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s8>) = G_BUILD_VECTOR [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8), [[C]](s8)
283+
; CHECK-NEXT: G_STORE [[BUILD_VECTOR]](<32 x s8>), [[COPY]](p0) :: (store (<32 x s8>))
284+
; CHECK-NEXT: RET_ReallyLR
285+
%3:_(s8) = G_CONSTANT i8 127
286+
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
287+
%5:_(s64) = G_CONSTANT i64 3
288+
%4:_(s8) = G_EXTRACT_VECTOR_ELT %2, %5
289+
%0:_(p0) = COPY $x0
290+
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
291+
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
292+
RET_ReallyLR
293+
294+
...
295+
---
296+
name: test_idx_oob
297+
body: |
298+
bb.1:
299+
liveins: $x0
300+
; CHECK-LABEL: name: test_idx_oob
301+
; CHECK: liveins: $x0
302+
; CHECK-NEXT: {{ $}}
303+
; CHECK-NEXT: RET_ReallyLR
304+
%3:_(s8) = G_CONSTANT i8 127
305+
%2:_(<32 x s8>) = G_BUILD_VECTOR %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8), %3(s8)
306+
%4:_(s8) = G_CONSTANT i8 -128
307+
%5:_(s64) = G_CONSTANT i64 1024
308+
%0:_(p0) = COPY $x0
309+
%1:_(<32 x s8>) = G_INSERT_VECTOR_ELT %2, %4(s8), %5(s64)
310+
G_STORE %1(<32 x s8>), %0(p0) :: (store (<32 x s8>))
311+
RET_ReallyLR
312+
313+
...

0 commit comments

Comments
 (0)