|
13 | 13 | #include "llvm/Analysis/Loads.h"
|
14 | 14 | #include "llvm/Analysis/AliasAnalysis.h"
|
15 | 15 | #include "llvm/Analysis/AssumeBundleQueries.h"
|
16 |
| -#include "llvm/Analysis/LoopAccessAnalysis.h" |
17 | 16 | #include "llvm/Analysis/LoopInfo.h"
|
18 | 17 | #include "llvm/Analysis/MemoryBuiltins.h"
|
19 | 18 | #include "llvm/Analysis/MemoryLocation.h"
|
@@ -276,88 +275,84 @@ static bool AreEquivalentAddressValues(const Value *A, const Value *B) {
|
276 | 275 | bool llvm::isDereferenceableAndAlignedInLoop(
|
277 | 276 | LoadInst *LI, Loop *L, ScalarEvolution &SE, DominatorTree &DT,
|
278 | 277 | AssumptionCache *AC, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
|
279 |
| - const Align Alignment = LI->getAlign(); |
280 | 278 | auto &DL = LI->getDataLayout();
|
281 | 279 | Value *Ptr = LI->getPointerOperand();
|
| 280 | + |
282 | 281 | APInt EltSize(DL.getIndexTypeSizeInBits(Ptr->getType()),
|
283 | 282 | DL.getTypeStoreSize(LI->getType()).getFixedValue());
|
| 283 | + const Align Alignment = LI->getAlign(); |
| 284 | + |
| 285 | + Instruction *HeaderFirstNonPHI = L->getHeader()->getFirstNonPHI(); |
284 | 286 |
|
285 | 287 | // If given a uniform (i.e. non-varying) address, see if we can prove the
|
286 | 288 | // access is safe within the loop w/o needing predication.
|
287 | 289 | if (L->isLoopInvariant(Ptr))
|
288 |
| - return isDereferenceableAndAlignedPointer( |
289 |
| - Ptr, Alignment, EltSize, DL, L->getHeader()->getFirstNonPHI(), AC, &DT); |
290 |
| - |
291 |
| - const SCEV *PtrScev = SE.getSCEV(Ptr); |
292 |
| - auto *AddRec = dyn_cast<SCEVAddRecExpr>(PtrScev); |
| 290 | + return isDereferenceableAndAlignedPointer(Ptr, Alignment, EltSize, DL, |
| 291 | + HeaderFirstNonPHI, AC, &DT); |
293 | 292 |
|
294 |
| - // Check to see if we have a repeating access pattern and it's possible |
295 |
| - // to prove all accesses are well aligned. |
| 293 | + // Otherwise, check to see if we have a repeating access pattern where we can |
| 294 | + // prove that all accesses are well aligned and dereferenceable. |
| 295 | + auto *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Ptr)); |
296 | 296 | if (!AddRec || AddRec->getLoop() != L || !AddRec->isAffine())
|
297 | 297 | return false;
|
298 |
| - |
299 | 298 | auto* Step = dyn_cast<SCEVConstant>(AddRec->getStepRecurrence(SE));
|
300 | 299 | if (!Step)
|
301 | 300 | return false;
|
302 | 301 |
|
303 |
| - // For the moment, restrict ourselves to the case where the access size is a |
304 |
| - // multiple of the requested alignment and the base is aligned. |
305 |
| - // TODO: generalize if a case found which warrants |
306 |
| - if (EltSize.urem(Alignment.value()) != 0) |
| 302 | + auto TC = SE.getSmallConstantMaxTripCount(L, Predicates); |
| 303 | + if (!TC) |
307 | 304 | return false;
|
308 | 305 |
|
309 | 306 | // TODO: Handle overlapping accesses.
|
310 |
| - if (EltSize.ugt(Step->getAPInt().abs())) |
311 |
| - return false; |
312 |
| - |
313 |
| - const SCEV *MaxBECount = |
314 |
| - SE.getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates); |
315 |
| - if (isa<SCEVCouldNotCompute>(MaxBECount)) |
316 |
| - return false; |
317 |
| - |
318 |
| - const auto &[AccessStart, AccessEnd] = getStartAndEndForAccess( |
319 |
| - L, PtrScev, LI->getType(), MaxBECount, &SE, nullptr); |
320 |
| - if (isa<SCEVCouldNotCompute>(AccessStart) || |
321 |
| - isa<SCEVCouldNotCompute>(AccessEnd)) |
| 307 | + // We should be computing AccessSize as (TC - 1) * Step + EltSize. |
| 308 | + if (EltSize.sgt(Step->getAPInt())) |
322 | 309 | return false;
|
323 | 310 |
|
324 |
| - // Try to get the access size. |
325 |
| - const SCEV *PtrDiff = SE.getMinusSCEV(AccessEnd, AccessStart); |
326 |
| - APInt MaxPtrDiff = SE.getUnsignedRangeMax(PtrDiff); |
| 311 | + // Compute the total access size for access patterns with unit stride and |
| 312 | + // patterns with gaps. For patterns with unit stride, Step and EltSize are the |
| 313 | + // same. |
| 314 | + // For patterns with gaps (i.e. non unit stride), we are |
| 315 | + // accessing EltSize bytes at every Step. |
| 316 | + APInt AccessSize = TC * Step->getAPInt(); |
327 | 317 |
|
| 318 | + assert(SE.isLoopInvariant(AddRec->getStart(), L) && |
| 319 | + "implied by addrec definition"); |
328 | 320 | Value *Base = nullptr;
|
329 |
| - APInt AccessSize; |
330 |
| - if (const SCEVUnknown *NewBase = dyn_cast<SCEVUnknown>(AccessStart)) { |
331 |
| - Base = NewBase->getValue(); |
332 |
| - AccessSize = MaxPtrDiff; |
333 |
| - } else if (auto *MinAdd = dyn_cast<SCEVAddExpr>(AccessStart)) { |
334 |
| - if (MinAdd->getNumOperands() != 2) |
335 |
| - return false; |
336 |
| - |
337 |
| - const auto *Offset = dyn_cast<SCEVConstant>(MinAdd->getOperand(0)); |
338 |
| - const auto *NewBase = dyn_cast<SCEVUnknown>(MinAdd->getOperand(1)); |
339 |
| - if (!Offset || !NewBase) |
340 |
| - return false; |
341 |
| - |
342 |
| - // The following code below assumes the offset is unsigned, but GEP |
343 |
| - // offsets are treated as signed so we can end up with a signed value |
344 |
| - // here too. For example, suppose the initial PHI value is (i8 255), |
345 |
| - // the offset will be treated as (i8 -1) and sign-extended to (i64 -1). |
346 |
| - if (Offset->getAPInt().isNegative()) |
347 |
| - return false; |
| 321 | + if (auto *StartS = dyn_cast<SCEVUnknown>(AddRec->getStart())) { |
| 322 | + Base = StartS->getValue(); |
| 323 | + } else if (auto *StartS = dyn_cast<SCEVAddExpr>(AddRec->getStart())) { |
| 324 | + // Handle (NewBase + offset) as start value. |
| 325 | + const auto *Offset = dyn_cast<SCEVConstant>(StartS->getOperand(0)); |
| 326 | + const auto *NewBase = dyn_cast<SCEVUnknown>(StartS->getOperand(1)); |
| 327 | + if (StartS->getNumOperands() == 2 && Offset && NewBase) { |
| 328 | + // The following code below assumes the offset is unsigned, but GEP |
| 329 | + // offsets are treated as signed so we can end up with a signed value |
| 330 | + // here too. For example, suppose the initial PHI value is (i8 255), |
| 331 | + // the offset will be treated as (i8 -1) and sign-extended to (i64 -1). |
| 332 | + if (Offset->getAPInt().isNegative()) |
| 333 | + return false; |
348 | 334 |
|
349 |
| - // For the moment, restrict ourselves to the case where the offset is a |
350 |
| - // multiple of the requested alignment and the base is aligned. |
351 |
| - // TODO: generalize if a case found which warrants |
352 |
| - if (Offset->getAPInt().urem(Alignment.value()) != 0) |
353 |
| - return false; |
| 335 | + // For the moment, restrict ourselves to the case where the offset is a |
| 336 | + // multiple of the requested alignment and the base is aligned. |
| 337 | + // TODO: generalize if a case found which warrants |
| 338 | + if (Offset->getAPInt().urem(Alignment.value()) != 0) |
| 339 | + return false; |
| 340 | + Base = NewBase->getValue(); |
| 341 | + bool Overflow = false; |
| 342 | + AccessSize = AccessSize.uadd_ov(Offset->getAPInt(), Overflow); |
| 343 | + if (Overflow) |
| 344 | + return false; |
| 345 | + } |
| 346 | + } |
354 | 347 |
|
355 |
| - AccessSize = MaxPtrDiff + Offset->getAPInt(); |
356 |
| - Base = NewBase->getValue(); |
357 |
| - } else |
| 348 | + if (!Base) |
358 | 349 | return false;
|
359 | 350 |
|
360 |
| - Instruction *HeaderFirstNonPHI = L->getHeader()->getFirstNonPHI(); |
| 351 | + // For the moment, restrict ourselves to the case where the access size is a |
| 352 | + // multiple of the requested alignment and the base is aligned. |
| 353 | + // TODO: generalize if a case found which warrants |
| 354 | + if (EltSize.urem(Alignment.value()) != 0) |
| 355 | + return false; |
361 | 356 | return isDereferenceableAndAlignedPointer(Base, Alignment, AccessSize, DL,
|
362 | 357 | HeaderFirstNonPHI, AC, &DT);
|
363 | 358 | }
|
|
0 commit comments