@@ -3830,16 +3830,18 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
3830
3830
LoopExitInstDef = Def;
3831
3831
}
3832
3832
3833
+ VectorParts RdxParts (UF);
3834
+ for (unsigned Part = 0 ; Part < UF; ++Part)
3835
+ RdxParts[Part] = State.get (LoopExitInstDef, Part);
3836
+
3833
3837
// If the vector reduction can be performed in a smaller type, we truncate
3834
3838
// then extend the loop exit value to enable InstCombine to evaluate the
3835
3839
// entire expression in the smaller type.
3836
3840
if (VF.isVector () && PhiTy != RdxDesc.getRecurrenceType ()) {
3837
3841
assert (!PhiR->isInLoop () && " Unexpected truncated inloop reduction!" );
3838
3842
Type *RdxVecTy = VectorType::get (RdxDesc.getRecurrenceType (), VF);
3839
3843
Builder.SetInsertPoint (VectorLoopLatch->getTerminator ());
3840
- VectorParts RdxParts (UF);
3841
3844
for (unsigned Part = 0 ; Part < UF; ++Part) {
3842
- RdxParts[Part] = State.get (LoopExitInstDef, Part);
3843
3845
Value *Trunc = Builder.CreateTrunc (RdxParts[Part], RdxVecTy);
3844
3846
Value *Extnd = RdxDesc.isSigned () ? Builder.CreateSExt (Trunc, VecTy)
3845
3847
: Builder.CreateZExt (Trunc, VecTy);
@@ -3851,14 +3853,12 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
3851
3853
}
3852
3854
Builder.SetInsertPoint (LoopMiddleBlock,
3853
3855
LoopMiddleBlock->getFirstInsertionPt ());
3854
- for (unsigned Part = 0 ; Part < UF; ++Part) {
3856
+ for (unsigned Part = 0 ; Part < UF; ++Part)
3855
3857
RdxParts[Part] = Builder.CreateTrunc (RdxParts[Part], RdxVecTy);
3856
- State.reset (LoopExitInstDef, RdxParts[Part], Part);
3857
- }
3858
3858
}
3859
3859
3860
3860
// Reduce all of the unrolled parts into a single vector.
3861
- Value *ReducedPartRdx = State. get (LoopExitInstDef, 0 ) ;
3861
+ Value *ReducedPartRdx = RdxParts[ 0 ] ;
3862
3862
unsigned Op = RecurrenceDescriptor::getOpcode (RK);
3863
3863
3864
3864
// The middle block terminator has already been assigned a DebugLoc here (the
@@ -3870,13 +3870,13 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
3870
3870
// accidentally cause an extra step back into the loop while debugging.
3871
3871
State.setDebugLocFrom (LoopMiddleBlock->getTerminator ()->getDebugLoc ());
3872
3872
if (PhiR->isOrdered ())
3873
- ReducedPartRdx = State. get (LoopExitInstDef, UF - 1 ) ;
3873
+ ReducedPartRdx = RdxParts[ UF - 1 ] ;
3874
3874
else {
3875
3875
// Floating-point operations should have some FMF to enable the reduction.
3876
3876
IRBuilderBase::FastMathFlagGuard FMFG (Builder);
3877
3877
Builder.setFastMathFlags (RdxDesc.getFastMathFlags ());
3878
3878
for (unsigned Part = 1 ; Part < UF; ++Part) {
3879
- Value *RdxPart = State. get (LoopExitInstDef, Part) ;
3879
+ Value *RdxPart = RdxParts[ Part] ;
3880
3880
if (Op != Instruction::ICmp && Op != Instruction::FCmp)
3881
3881
ReducedPartRdx = Builder.CreateBinOp (
3882
3882
(Instruction::BinaryOps)Op, RdxPart, ReducedPartRdx, " bin.rdx" );
0 commit comments