1- // ===- DivRemPairs.cpp - Hoist/decompose division and remainder -*- C++ -* -===//
1+ // ===- DivRemPairs.cpp - Hoist/[dr]ecompose division and remainder ------- -===//
22//
33// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
44// See https://llvm.org/LICENSE.txt for license information.
55// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
66//
77// ===----------------------------------------------------------------------===//
88//
9- // This pass hoists and/or decomposes integer division and remainder
9+ // This pass hoists and/or decomposes/recomposes integer division and remainder
1010// instructions to enable CFG improvements and better codegen.
1111//
1212// ===----------------------------------------------------------------------===//
1919#include " llvm/Analysis/TargetTransformInfo.h"
2020#include " llvm/IR/Dominators.h"
2121#include " llvm/IR/Function.h"
22+ #include " llvm/IR/PatternMatch.h"
2223#include " llvm/Pass.h"
2324#include " llvm/Support/DebugCounter.h"
2425#include " llvm/Transforms/Scalar.h"
2526#include " llvm/Transforms/Utils/BypassSlowDivision.h"
2627
2728using namespace llvm ;
29+ using namespace llvm ::PatternMatch;
2830
2931#define DEBUG_TYPE " div-rem-pairs"
3032STATISTIC (NumPairs, " Number of div/rem pairs"
33+ STATISTIC (NumRecomposed, " Number of instructions recomposed"
3134STATISTIC (NumHoisted, " Number of instructions hoisted"
3235STATISTIC (NumDecomposed, " Number of instructions decomposed"
3336DEBUG_COUNTER (DRPCounter, " div-rem-pairs-transform"
3437 " Controls transformations in div-rem-pairs pass"
3538
39+ namespace {
40+ struct ExpandedMatch {
41+ DivRemMapKey Key;
42+ Instruction *Value;
43+ };
44+ } // namespace
45+
46+ // / See if we can match: (which is the form we expand into)
47+ // / X - ((X ?/ Y) * Y)
48+ // / which is equivalent to:
49+ // / X ?% Y
50+ static llvm::Optional<ExpandedMatch> matchExpandedRem (Instruction &I) {
51+ Value *Dividend, *XroundedDownToMultipleOfY;
52+ if (!match (&I, m_Sub (m_Value (Dividend), m_Value (XroundedDownToMultipleOfY))))
53+ return llvm::None;
54+
55+ Value *Divisor;
56+ Instruction *Div;
57+ // Look for ((X / Y) * Y)
58+ if (!match (
59+ XroundedDownToMultipleOfY,
60+ m_c_Mul (m_CombineAnd (m_IDiv (m_Specific (Dividend), m_Value (Divisor)),
61+ m_Instruction (Div)),
62+ m_Deferred (Divisor))))
63+ return llvm::None;
64+
65+ ExpandedMatch M;
66+ M.Key .SignedOp = Div->getOpcode () == Instruction::SDiv;
67+ M.Key .Dividend = Dividend;
68+ M.Key .Divisor = Divisor;
69+ M.Value = &I;
70+ return M;
71+ }
72+
3673// / A thin wrapper to store two values that we matched as div-rem pair.
3774// / We want this extra indirection to avoid dealing with RAUW'ing the map keys.
3875struct DivRemPairWorklistEntry {
@@ -62,6 +99,16 @@ struct DivRemPairWorklistEntry {
6299 // / In this pair, what are the divident and divisor?
63100 Value *getDividend () const { return DivInst->getOperand (0 ); }
64101 Value *getDivisor () const { return DivInst->getOperand (1 ); }
102+
103+ bool isRemExpanded () const {
104+ switch (RemInst->getOpcode ()) {
105+ case Instruction::SRem:
106+ case Instruction::URem:
107+ return false ; // single 'rem' instruction - unexpanded form.
108+ default :
109+ return true ; // anything else means we have remainder in expanded form.
110+ }
111+ }
65112};
66113using DivRemWorklistTy = SmallVector<DivRemPairWorklistEntry,
4 >;
67114
@@ -87,6 +134,8 @@ static DivRemWorklistTy getWorklist(Function &F) {
87134 RemMap[DivRemMapKey (true , I.getOperand (0 ), I.getOperand (1 ))] = &I;
88135 else if (I.getOpcode () == Instruction::URem)
89136 RemMap[DivRemMapKey (false , I.getOperand (0 ), I.getOperand (1 ))] = &I;
137+ else if (auto Match = matchExpandedRem (I))
138+ RemMap[Match->Key ] = Match->Value ;
90139 }
91140 }
92141
@@ -137,22 +186,61 @@ static bool optimizeDivRem(Function &F, const TargetTransformInfo &TTI,
137186
138187 // Process each entry in the worklist.
139188 for (DivRemPairWorklistEntry &E : Worklist) {
189+ if (!DebugCounter::shouldExecute (DRPCounter))
190+ continue ;
191+
140192 bool HasDivRemOp = TTI.hasDivRemOp (E.getType (), E.isSigned ());
141193
142194 auto &DivInst = E.DivInst ;
143195 auto &RemInst = E.RemInst ;
144196
197+ const bool RemOriginallyWasInExpandedForm = E.isRemExpanded ();
198+
199+ if (HasDivRemOp && E.isRemExpanded ()) {
200+ // The target supports div+rem but the rem is expanded.
201+ // We should recompose it first.
202+ Value *X = E.getDividend ();
203+ Value *Y = E.getDivisor ();
204+ Instruction *RealRem = E.isSigned () ? BinaryOperator::CreateSRem (X, Y)
205+ : BinaryOperator::CreateURem (X, Y);
206+ // Note that we place it right next to the original expanded instruction,
207+ // and letting further handling to move it if needed.
208+ RealRem->setName (RemInst->getName () + " .recomposed"
209+ RealRem->insertAfter (RemInst);
210+ Instruction *OrigRemInst = RemInst;
211+ // Update AssertingVH<> with new instruction so it doesn't assert.
212+ RemInst = RealRem;
213+ // And replace the original instruction with the new one.
214+ OrigRemInst->replaceAllUsesWith (RealRem);
215+ OrigRemInst->eraseFromParent ();
216+ NumRecomposed++;
217+ // Note that we have left ((X / Y) * Y) around.
218+ // If it had other uses we could rewrite it as X - X % Y
219+ }
220+
221+ assert ((!E.isRemExpanded () || !HasDivRemOp) &&
222+ " *If* the target supports div-rem, then by now the RemInst *is* "
223+ " Instruction::[US]Rem."
224+
145225 // If the target supports div+rem and the instructions are in the same block
146226 // already, there's nothing to do. The backend should handle this. If the
147227 // target does not support div+rem, then we will decompose the rem.
148228 if (HasDivRemOp && RemInst->getParent () == DivInst->getParent ())
149229 continue ;
150230
151231 bool DivDominates = DT.dominates (DivInst, RemInst);
152- if (!DivDominates && !DT.dominates (RemInst, DivInst))
232+ if (!DivDominates && !DT.dominates (RemInst, DivInst)) {
233+ // We have matching div-rem pair, but they are in two different blocks,
234+ // neither of which dominates one another.
235+ assert (!RemOriginallyWasInExpandedForm &&
236+ " Won't happen for expanded-form rem."
237+ // FIXME: We could hoist both ops to the common predecessor block?
153238 continue ;
239+ }
154240
155- if (!DebugCounter::shouldExecute (DRPCounter))
241+ // The target does not have a single div/rem operation,
242+ // and the rem is already in expanded form. Nothing to do.
243+ if (!HasDivRemOp && E.isRemExpanded ())
156244 continue ;
157245
158246 if (HasDivRemOp) {
@@ -164,9 +252,15 @@ static bool optimizeDivRem(Function &F, const TargetTransformInfo &TTI,
164252 DivInst->moveAfter (RemInst);
165253 NumHoisted++;
166254 } else {
167- // The target does not have a single div/rem operation. Decompose the
168- // remainder calculation as:
255+ // The target does not have a single div/rem operation,
256+ // and the rem is *not* in a already-expanded form.
257+ // Decompose the remainder calculation as:
169258 // X % Y --> X - ((X / Y) * Y).
259+
260+ assert (!RemOriginallyWasInExpandedForm &&
261+ " We should not be expanding if the rem was in expanded form to "
262+ " begin with."
263+
170264 Value *X = E.getDividend ();
171265 Value *Y = E.getDivisor ();
172266 Instruction *Mul = BinaryOperator::CreateMul (DivInst, Y);
0 commit comments