Skip to content

Commit 36c2311

Browse files
committed
[LoopVectorize] Use new getUniqueLatchExitBlock routine
With PR #88385 I am introducing support for vectorising more loops with early exits that don't require a scalar epilogue. As such, if a loop doesn't have a unique exit block it will not automatically imply we require a scalar epilogue. Also, in all places in the code today where we use the variable LoopExitBlock we actually mean the exit block from the latch. Therefore, it seemed reasonable to add a new getUniqueLatchExitBlock that allows the caller to determine the exit block taken from the latch and use this instead of getUniqueExitBlock. I also renamed LoopExitBlock to be LatchExitBlock. I feel this not only better reflects how the variable is used today, but also prepares the code for PR #88385. While doing this I also noticed that one of the comments in requiresScalarEpilogue is wrong when we require a scalar epilogue, i.e. when we're not exiting from the latch block. This doesn't always imply we have multiple exits, e.g. see the test in Transforms/LoopVectorize/unroll_nonlatch.ll where the latch unconditionally branches back to the only exiting block.
1 parent 44cfbef commit 36c2311

File tree

3 files changed

+27
-13
lines changed

3 files changed

+27
-13
lines changed

llvm/include/llvm/Support/GenericLoopInfo.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,10 @@ template <class BlockT, class LoopT> class LoopBase {
294294
/// Otherwise return null.
295295
BlockT *getUniqueExitBlock() const;
296296

297+
/// Return the unique exit block for the latch, or null if there are multiple
298+
/// different exit blocks.
299+
BlockT *getUniqueLatchExitBlock() const;
300+
297301
/// Return true if this loop does not have any exit blocks.
298302
bool hasNoExitBlocks() const;
299303

llvm/include/llvm/Support/GenericLoopInfoImpl.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,16 @@ BlockT *LoopBase<BlockT, LoopT>::getUniqueExitBlock() const {
159159
return getExitBlockHelper(this, true).first;
160160
}
161161

162+
template <class BlockT, class LoopT>
163+
BlockT *LoopBase<BlockT, LoopT>::getUniqueLatchExitBlock() const {
164+
const BlockT *Latch = getLoopLatch();
165+
assert(Latch && "Latch block must exists");
166+
SmallVector<BlockT *, 4> ExitBlocks;
167+
getUniqueExitBlocksHelper(this, ExitBlocks,
168+
[Latch](const BlockT *BB) { return BB == Latch; });
169+
return ExitBlocks.size() == 1 ? ExitBlocks[0] : nullptr;
170+
}
171+
162172
/// getExitEdges - Return all pairs of (_inside_block_,_outside_block_).
163173
template <class BlockT, class LoopT>
164174
void LoopBase<BlockT, LoopT>::getExitEdges(

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -652,9 +652,8 @@ class InnerLoopVectorizer {
652652
/// Middle Block between the vector and the scalar.
653653
BasicBlock *LoopMiddleBlock;
654654

655-
/// The unique ExitBlock of the scalar loop if one exists. Note that
656-
/// there can be multiple exiting edges reaching this block.
657-
BasicBlock *LoopExitBlock;
655+
/// The exit block from the loop latch, if one exists.
656+
BasicBlock *LatchExitBlock;
658657

659658
/// The scalar loop body.
660659
BasicBlock *LoopScalarBody;
@@ -1390,6 +1389,8 @@ class LoopVectorizationCostModel {
13901389

13911390
/// Returns true if we're required to use a scalar epilogue for at least
13921391
/// the final iteration of the original loop.
1392+
/// TODO: It would be good to cache the result and avoid recalculating,
1393+
/// which will also avoid reprinting the same debug message many times.
13931394
bool requiresScalarEpilogue(bool IsVectorizing) const {
13941395
if (!isScalarEpilogueAllowed()) {
13951396
LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
@@ -1398,8 +1399,8 @@ class LoopVectorizationCostModel {
13981399
// If we might exit from anywhere but the latch, must run the exiting
13991400
// iteration in scalar form.
14001401
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
1401-
LLVM_DEBUG(
1402-
dbgs() << "LV: Loop requires scalar epilogue: multiple exits\n");
1402+
LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
1403+
"from latch block\n");
14031404
return true;
14041405
}
14051406
if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
@@ -2043,8 +2044,7 @@ class GeneratedRTChecks {
20432044
/// adjusts the branches to branch to the vector preheader or \p Bypass,
20442045
/// depending on the generated condition.
20452046
BasicBlock *emitSCEVChecks(BasicBlock *Bypass,
2046-
BasicBlock *LoopVectorPreHeader,
2047-
BasicBlock *LoopExitBlock) {
2047+
BasicBlock *LoopVectorPreHeader) {
20482048
if (!SCEVCheckCond)
20492049
return nullptr;
20502050

@@ -2516,7 +2516,7 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
25162516

25172517
BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
25182518
BasicBlock *const SCEVCheckBlock =
2519-
RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader, LoopExitBlock);
2519+
RTChecks.emitSCEVChecks(Bypass, LoopVectorPreHeader);
25202520
if (!SCEVCheckBlock)
25212521
return nullptr;
25222522

@@ -2533,7 +2533,7 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
25332533
// If there is an epilogue which must run, there's no edge from the
25342534
// middle block to exit blocks and thus no need to update the immediate
25352535
// dominator of the exit blocks.
2536-
DT->changeImmediateDominator(LoopExitBlock, SCEVCheckBlock);
2536+
DT->changeImmediateDominator(LatchExitBlock, SCEVCheckBlock);
25372537
}
25382538

25392539
LoopBypassBlocks.push_back(SCEVCheckBlock);
@@ -2581,9 +2581,9 @@ void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
25812581
LoopScalarBody = OrigLoop->getHeader();
25822582
LoopVectorPreHeader = OrigLoop->getLoopPreheader();
25832583
assert(LoopVectorPreHeader && "Invalid loop structure");
2584-
LoopExitBlock = OrigLoop->getUniqueExitBlock(); // may be nullptr
2585-
assert((LoopExitBlock || Cost->requiresScalarEpilogue(VF.isVector())) &&
2586-
"multiple exit loop without required epilogue?");
2584+
LatchExitBlock = OrigLoop->getUniqueLatchExitBlock();
2585+
assert((LatchExitBlock || Cost->requiresScalarEpilogue(VF.isVector())) &&
2586+
"Expected loop without exit from latch to require an epilogue");
25872587

25882588
LoopMiddleBlock =
25892589
SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
@@ -7788,7 +7788,7 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton(
77887788
// If there is an epilogue which must run, there's no edge from the
77897789
// middle block to exit blocks and thus no need to update the immediate
77907790
// dominator of the exit blocks.
7791-
DT->changeImmediateDominator(LoopExitBlock,
7791+
DT->changeImmediateDominator(LatchExitBlock,
77927792
EPI.EpilogueIterationCountCheck);
77937793

77947794
// Keep track of bypass blocks, as they feed start values to the induction and

0 commit comments

Comments
 (0)