Rebase, fix up tests, address comments:

* Check AddRec Loop vs. current
* Remove stray struct token declaration
* Add comment on Opcode field of VPHistogramRecipe
* Use Ctx.Types.inferScalarType when computing cost
* Check for a LiveIn value if there isn't an underlying value for costing
* Tolerate nulls for both underlying and LiveIn, just in case.

Author: huntergr-arm

llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

@@ -1119,8 +1119,9 @@ static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
   if (!match(HIdx, m_ZExtOrSExtOrSelf(m_Load(m_Value(VPtrVal)))))
     return false;
 
-  if (!isa<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal)) ||
-      TheLoop->isLoopInvariant(VPtrVal))
+  // Make sure the index address varies in this loop, not an outer loop.
+  const auto *AR = dyn_cast<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal));
+  if (!AR || AR->getLoop() != TheLoop)
     return false;
 
   // Ensure we'll have the same mask by checking that all parts of the histogram

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -69,7 +69,6 @@ class LoopVectorizationCostModel;
 class LoopVersioning;
 
 struct VPCostContext;
-struct HistogramInfo;
 
 namespace Intrinsic {
 typedef unsigned ID;
@@ -1698,6 +1697,7 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
 /// of intrinsics. The only update operations currently supported are
 /// 'add' and 'sub' where the other term is loop-invariant.
 class VPHistogramRecipe : public VPRecipeBase {
+  /// Opcode of the update operation, currently either add or sub.
   unsigned Opcode;
 
 public:

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -1090,20 +1090,24 @@ InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
   //        whether we can use base + vec-of-smaller-indices or just
   //        vec-of-pointers.
   assert(VF.isVector() && "Invalid VF for histogram cost");
-  Value *Address = getOperand(0)->getUnderlyingValue();
-  Value *IncAmt = getOperand(1)->getUnderlyingValue();
-  Type *IncTy = IncAmt->getType();
+  Type *AddressTy = Ctx.Types.inferScalarType(getOperand(0));
+  VPValue *IncAmt = getOperand(1);
+  Type *IncTy = Ctx.Types.inferScalarType(IncAmt);
   VectorType *VTy = VectorType::get(IncTy, VF);
 
   // Assume that a non-constant update value (or a constant != 1) requires
   // a multiply, and add that into the cost.
+  Value *RHS = IncAmt->getUnderlyingValue();
+  // The underlying value may be null, check for a live-in if so.
+  if (!RHS && IncAmt->isLiveIn())
+    RHS = IncAmt->getLiveInIRValue();
   InstructionCost MulCost = TTI::TCC_Free;
-  ConstantInt *RHS = dyn_cast<ConstantInt>(IncAmt);
-  if (!RHS || RHS->getZExtValue() != 1)
+  ConstantInt *CI = dyn_cast_if_present<ConstantInt>(RHS);
+  if (!CI || CI->getZExtValue() != 1)
     MulCost = Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy);
 
   // Find the cost of the histogram operation itself.
-  Type *PtrTy = VectorType::get(Address->getType(), VF);
+  Type *PtrTy = VectorType::get(AddressTy, VF);
   Type *MaskTy = VectorType::get(Type::getInt1Ty(Ctx.LLVMCtx), VF);
   IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
                               Type::getVoidTy(Ctx.LLVMCtx),

llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll

@@ -72,8 +72,7 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vscale.i32()
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP9:%.*]] = shl nuw nsw i32 [[TMP8]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP9:%.*]] = trunc nuw nsw i64 [[TMP6]] to i32
 ; NORMAL_DEP_LIMIT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
 ; NORMAL_DEP_LIMIT-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NORMAL_DEP_LIMIT-NEXT:    br label [[FOR_BODY:%.*]]

llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll

@@ -732,8 +732,7 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc nuw nsw i64 [[TMP8]] to i32
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]