@@ -540,10 +540,6 @@ class InnerLoopVectorizer {
540540protected:
541541 friend class LoopVectorizationPlanner;
542542
543- /// Iteratively sink the scalarized operands of a predicated instruction into
544- /// the block that was created for it.
545- void sinkScalarOperands(Instruction *PredInst);
546-
547543 /// Returns (and creates if needed) the trip count of the widened loop.
548544 Value *getOrCreateVectorTripCount(BasicBlock *InsertBlock);
549545
@@ -628,9 +624,6 @@ class InnerLoopVectorizer {
628624 /// A list of all bypass blocks. The first block is the entry of the loop.
629625 SmallVector<BasicBlock *, 4> LoopBypassBlocks;
630626
631- /// Store instructions that were predicated.
632- SmallVector<Instruction *, 4> PredicatedInstructions;
633-
634627 /// Trip count of the original loop.
635628 Value *TripCount = nullptr;
636629
@@ -2382,17 +2375,13 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
23822375 if (auto *II = dyn_cast<AssumeInst>(Cloned))
23832376 AC->registerAssumption(II);
23842377
2385- // End if-block.
2386- VPRegionBlock *Parent = RepRecipe->getParent()->getParent();
2387- bool IfPredicateInstr = Parent ? Parent->isReplicator() : false;
23882378 assert(
2389- (Parent || !RepRecipe->getParent()->getPlan()->getVectorLoopRegion() ||
2379+ (RepRecipe->getParent()->getParent() ||
2380+ !RepRecipe->getParent()->getPlan()->getVectorLoopRegion() ||
23902381 all_of(RepRecipe->operands(),
23912382 [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); })) &&
23922383 "Expected a recipe is either within a region or all of its operands "
23932384 "are defined outside the vectorized region.");
2394- if (IfPredicateInstr)
2395- PredicatedInstructions.push_back(Cloned);
23962385}
23972386
23982387Value *
@@ -2866,9 +2855,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
28662855 if (!State.Plan->getVectorLoopRegion())
28672856 return;
28682857
2869- for (Instruction *PI : PredicatedInstructions)
2870- sinkScalarOperands(&*PI);
2871-
28722858 VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion();
28732859 VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock();
28742860 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
@@ -2894,82 +2880,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
28942880 VF.getKnownMinValue() * UF);
28952881}
28962882
2897- void InnerLoopVectorizer::sinkScalarOperands(Instruction *PredInst) {
2898- // The basic block and loop containing the predicated instruction.
2899- auto *PredBB = PredInst->getParent();
2900- auto *VectorLoop = LI->getLoopFor(PredBB);
2901-
2902- // Initialize a worklist with the operands of the predicated instruction.
2903- SetVector<Value *> Worklist(PredInst->op_begin(), PredInst->op_end());
2904-
2905- // Holds instructions that we need to analyze again. An instruction may be
2906- // reanalyzed if we don't yet know if we can sink it or not.
2907- SmallVector<Instruction *, 8> InstsToReanalyze;
2908-
2909- // Returns true if a given use occurs in the predicated block. Phi nodes use
2910- // their operands in their corresponding predecessor blocks.
2911- auto IsBlockOfUsePredicated = [&](Use &U) -> bool {
2912- auto *I = cast<Instruction>(U.getUser());
2913- BasicBlock *BB = I->getParent();
2914- if (auto *Phi = dyn_cast<PHINode>(I))
2915- BB = Phi->getIncomingBlock(
2916- PHINode::getIncomingValueNumForOperand(U.getOperandNo()));
2917- return BB == PredBB;
2918- };
2919-
2920- // Iteratively sink the scalarized operands of the predicated instruction
2921- // into the block we created for it. When an instruction is sunk, it's
2922- // operands are then added to the worklist. The algorithm ends after one pass
2923- // through the worklist doesn't sink a single instruction.
2924- bool Changed;
2925- do {
2926- // Add the instructions that need to be reanalyzed to the worklist, and
2927- // reset the changed indicator.
2928- Worklist.insert_range(InstsToReanalyze);
2929- InstsToReanalyze.clear();
2930- Changed = false;
2931-
2932- while (!Worklist.empty()) {
2933- auto *I = dyn_cast<Instruction>(Worklist.pop_back_val());
2934-
2935- // We can't sink an instruction if it is a phi node, is not in the loop,
2936- // may have side effects or may read from memory.
2937- // TODO: Could do more granular checking to allow sinking
2938- // a load past non-store instructions.
2939- if (!I || isa<PHINode>(I) || !VectorLoop->contains(I) ||
2940- I->mayHaveSideEffects() || I->mayReadFromMemory())
2941- continue;
2942-
2943- // If the instruction is already in PredBB, check if we can sink its
2944- // operands. In that case, VPlan's sinkScalarOperands() succeeded in
2945- // sinking the scalar instruction I, hence it appears in PredBB; but it
2946- // may have failed to sink I's operands (recursively), which we try
2947- // (again) here.
2948- if (I->getParent() == PredBB) {
2949- Worklist.insert_range(I->operands());
2950- continue;
2951- }
2952-
2953- // It's legal to sink the instruction if all its uses occur in the
2954- // predicated block. Otherwise, there's nothing to do yet, and we may
2955- // need to reanalyze the instruction.
2956- if (!llvm::all_of(I->uses(), IsBlockOfUsePredicated)) {
2957- InstsToReanalyze.push_back(I);
2958- continue;
2959- }
2960-
2961- // Move the instruction to the beginning of the predicated block, and add
2962- // it's operands to the worklist.
2963- I->moveBefore(PredBB->getFirstInsertionPt());
2964- Worklist.insert_range(I->operands());
2965-
2966- // The sinking may have enabled other instructions to be sunk, so we will
2967- // need to iterate.
2968- Changed = true;
2969- }
2970- } while (Changed);
2971- }
2972-
29732883void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
29742884 auto Iter = vp_depth_first_deep(Plan.getEntry());
29752885 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
0 commit comments