@@ -2296,87 +2296,13 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
2296
2296
}
2297
2297
2298
2298
int LoopVectorizationLegality::isConsecutivePtr (Value *Ptr) {
2299
- assert (Ptr->getType ()->isPointerTy () && " Unexpected non-ptr" );
2300
- auto *SE = PSE.getSE ();
2301
- // Make sure that the pointer does not point to structs.
2302
- if (Ptr->getType ()->getPointerElementType ()->isAggregateType ())
2303
- return 0 ;
2304
-
2305
- // If this value is a pointer induction variable, we know it is consecutive.
2306
- PHINode *Phi = dyn_cast_or_null<PHINode>(Ptr);
2307
- if (Phi && Inductions.count (Phi)) {
2308
- InductionDescriptor II = Inductions[Phi];
2309
- return II.getConsecutiveDirection ();
2310
- }
2311
-
2312
- GetElementPtrInst *Gep = getGEPInstruction (Ptr);
2313
- if (!Gep)
2314
- return 0 ;
2315
-
2316
- unsigned NumOperands = Gep->getNumOperands ();
2317
- Value *GpPtr = Gep->getPointerOperand ();
2318
- // If this GEP value is a consecutive pointer induction variable and all of
2319
- // the indices are constant, then we know it is consecutive.
2320
- Phi = dyn_cast<PHINode>(GpPtr);
2321
- if (Phi && Inductions.count (Phi)) {
2322
-
2323
- // Make sure that the pointer does not point to structs.
2324
- PointerType *GepPtrType = cast<PointerType>(GpPtr->getType ());
2325
- if (GepPtrType->getElementType ()->isAggregateType ())
2326
- return 0 ;
2327
-
2328
- // Make sure that all of the index operands are loop invariant.
2329
- for (unsigned i = 1 ; i < NumOperands; ++i)
2330
- if (!SE->isLoopInvariant (PSE.getSCEV (Gep->getOperand (i)), TheLoop))
2331
- return 0 ;
2332
2299
2333
- InductionDescriptor II = Inductions[Phi];
2334
- return II.getConsecutiveDirection ();
2335
- }
2336
-
2337
- unsigned InductionOperand = getGEPInductionOperand (Gep);
2338
-
2339
- // Check that all of the gep indices are uniform except for our induction
2340
- // operand.
2341
- for (unsigned i = 0 ; i != NumOperands; ++i)
2342
- if (i != InductionOperand &&
2343
- !SE->isLoopInvariant (PSE.getSCEV (Gep->getOperand (i)), TheLoop))
2344
- return 0 ;
2345
-
2346
- // We can emit wide load/stores only if the last non-zero index is the
2347
- // induction variable.
2348
- const SCEV *Last = nullptr ;
2349
- if (!getSymbolicStrides () || !getSymbolicStrides ()->count (Gep))
2350
- Last = PSE.getSCEV (Gep->getOperand (InductionOperand));
2351
- else {
2352
- // Because of the multiplication by a stride we can have a s/zext cast.
2353
- // We are going to replace this stride by 1 so the cast is safe to ignore.
2354
- //
2355
- // %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
2356
- // %0 = trunc i64 %indvars.iv to i32
2357
- // %mul = mul i32 %0, %Stride1
2358
- // %idxprom = zext i32 %mul to i64 << Safe cast.
2359
- // %arrayidx = getelementptr inbounds i32* %B, i64 %idxprom
2360
- //
2361
- Last = replaceSymbolicStrideSCEV (PSE, *getSymbolicStrides (),
2362
- Gep->getOperand (InductionOperand), Gep);
2363
- if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(Last))
2364
- Last =
2365
- (C->getSCEVType () == scSignExtend || C->getSCEVType () == scZeroExtend)
2366
- ? C->getOperand ()
2367
- : Last;
2368
- }
2369
- if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Last)) {
2370
- const SCEV *Step = AR->getStepRecurrence (*SE);
2371
-
2372
- // The memory is consecutive because the last index is consecutive
2373
- // and all other indices are loop invariant.
2374
- if (Step->isOne ())
2375
- return 1 ;
2376
- if (Step->isAllOnesValue ())
2377
- return -1 ;
2378
- }
2300
+ const ValueToValueMap &Strides = getSymbolicStrides () ? *getSymbolicStrides () :
2301
+ ValueToValueMap ();
2379
2302
2303
+ int Stride = getPtrStride (PSE, Ptr, TheLoop, Strides, true , false );
2304
+ if (Stride == 1 || Stride == -1 )
2305
+ return Stride;
2380
2306
return 0 ;
2381
2307
}
2382
2308
@@ -2813,8 +2739,6 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
2813
2739
Ptr = Builder.Insert (Gep2);
2814
2740
2815
2741
} else { // No GEP
2816
- // Use the induction element ptr.
2817
- assert (isa<PHINode>(Ptr) && " Invalid induction ptr" );
2818
2742
setDebugLocFromInst (Builder, Ptr);
2819
2743
Ptr = getScalarValue (Ptr, 0 , 0 );
2820
2744
}
0 commit comments