ScalarEvolution.cpp

Go to the documentation of this file.

//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the implementation of the scalar evolution analysis
// engine, which is used primarily to analyze expressions involving induction
// variables in loops.
//
// There are several aspects to this library.  First is the representation of
// scalar expressions, which are represented as subclasses of the SCEV class.
// These classes are used to represent certain types of subexpressions that we
// can handle. We only create one SCEV of a particular shape, so
// pointer-comparisons for equality are legal.
//
// One important aspect of the SCEV objects is that they are never cyclic, even
// if there is a cycle in the dataflow for an expression (ie, a PHI node).  If
// the PHI node is one of the idioms that we can represent (e.g., a polynomial
// recurrence) then we represent it directly as a recurrence node, otherwise we
// represent it as a SCEVUnknown node.
//
// In addition to being able to represent expressions of various types, we also
// have folders that are used to build the *canonical* representation for a
// particular expression.  These folders are capable of using a variety of
// rewrite rules to simplify the expressions.
//
// Once the folders are defined, we can implement the more interesting
// higher-level code, such as the code that recognizes PHI nodes of various
// types, computes the execution count of a loop, etc.
//
// TODO: We should use these routines and value representations to implement
// dependence analysis!
//
//===----------------------------------------------------------------------===//
//
// There are several good references for the techniques used in this analysis.
//
//  Chains of recurrences -- a method to expedite the evaluation
//  of closed-form functions
//  Olaf Bachmann, Paul S. Wang, Eugene V. Zima
//
//  On computational properties of chains of recurrences
//  Eugene V. Zima
//
//  Symbolic Evaluation of Chains of Recurrences for Loop Optimization
//  Robert A. van Engelen
//
//  Efficient Symbolic Analysis for Optimizing Compilers
//  Robert A. van Engelen
//
//  Using the chains of recurrences algebra for data dependence testing and
//  induction variable substitution
//  MS Thesis, Johnie Birch
//
//===----------------------------------------------------------------------===//
 
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/FoldingSet.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/Verifier.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InterleavedRange.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
#include <climits>
#include <cstdint>
#include <cstdlib>
#include <map>
#include <memory>
#include <numeric>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>
 
using namespace llvm;
using namespace PatternMatch;
using namespace SCEVPatternMatch;
 
#define DEBUG_TYPE "scalar-evolution"
 
STATISTIC(NumExitCountsComputed,
          "Number of loop exits with predictable exit counts");
STATISTIC(NumExitCountsNotComputed,
          "Number of loop exits without predictable exit counts");
STATISTIC(NumBruteForceTripCountsComputed,
          "Number of loops with trip counts computed by force");
 
#ifdef EXPENSIVE_CHECKS
bool llvm::VerifySCEV = true;
#else
bool llvm::VerifySCEV = false;
#endif
 
static cl::opt<unsigned>
    MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
                            cl::desc("Maximum number of iterations SCEV will "
                                     "symbolically execute a constant "
                                     "derived loop"),
                            cl::init(100));
 
static cl::opt<bool, true> VerifySCEVOpt(
    "verify-scev", cl::Hidden, cl::location(VerifySCEV),
    cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
static cl::opt<bool> VerifySCEVStrict(
    "verify-scev-strict", cl::Hidden,
    cl::desc("Enable stricter verification with -verify-scev is passed"));
 
static cl::opt<bool> VerifyIR(
    "scev-verify-ir", cl::Hidden,
    cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
    cl::init(false));
 
static cl::opt<unsigned> MulOpsInlineThreshold(
    "scev-mulops-inline-threshold", cl::Hidden,
    cl::desc("Threshold for inlining multiplication operands into a SCEV"),
    cl::init(32));
 
static cl::opt<unsigned> AddOpsInlineThreshold(
    "scev-addops-inline-threshold", cl::Hidden,
    cl::desc("Threshold for inlining addition operands into a SCEV"),
    cl::init(500));
 
static cl::opt<unsigned> MaxSCEVCompareDepth(
    "scalar-evolution-max-scev-compare-depth", cl::Hidden,
    cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
    cl::init(32));
 
static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
    "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
    cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
    cl::init(2));
 
static cl::opt<unsigned> MaxValueCompareDepth(
    "scalar-evolution-max-value-compare-depth", cl::Hidden,
    cl::desc("Maximum depth of recursive value complexity comparisons"),
    cl::init(2));
 
static cl::opt<unsigned>
    MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
                  cl::desc("Maximum depth of recursive arithmetics"),
                  cl::init(32));
 
static cl::opt<unsigned> MaxConstantEvolvingDepth(
    "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
    cl::desc("Maximum depth of recursive constant evolving"), cl::init(32));
 
static cl::opt<unsigned>
    MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
                 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
                 cl::init(8));
 
static cl::opt<unsigned>
    MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
                  cl::desc("Max coefficients in AddRec during evolving"),
                  cl::init(8));
 
static cl::opt<unsigned>
    HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
                  cl::desc("Size of the expression which is considered huge"),
                  cl::init(4096));
 
static cl::opt<unsigned> RangeIterThreshold(
    "scev-range-iter-threshold", cl::Hidden,
    cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
    cl::init(32));
 
static cl::opt<unsigned> MaxLoopGuardCollectionDepth(
    "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
    cl::desc("Maximum depth for recursive loop guard collection"), cl::init(1));
 
static cl::opt<bool>
ClassifyExpressions("scalar-evolution-classify-expressions",
    cl::Hidden, cl::init(true),
    cl::desc("When printing analysis, include information on every instruction"));
 
static cl::opt<bool> UseExpensiveRangeSharpening(
    "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
    cl::init(false),
    cl::desc("Use more powerful methods of sharpening expression ranges. May "
             "be costly in terms of compile time"));
 
static cl::opt<bool>
    EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
                            cl::desc("Handle <= and >= in finite loops"),
                            cl::init(true));
 
static cl::opt<bool> UseContextForNoWrapFlagInference(
    "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
    cl::desc("Infer nuw/nsw flags using context where suitable"),
    cl::init(true));
 
//===----------------------------------------------------------------------===//
//                           SCEV class definitions
//===----------------------------------------------------------------------===//
 
void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
  // Leaf nodes are always their own canonical.
  switch (getSCEVType()) {
  case scConstant:
  case scVScale:
  case scUnknown:
    CanonicalSCEV = this;
    return;
  default:
    break;
  }
 
  // For all other expressions, check whether any immediate operand has a
  // different canonical. Since operands are always created before their parent,
  // their canonical pointers are already set — no recursion needed.
  bool Changed = false;
  SmallVector<SCEVUse, 4> CanonOps;
  for (SCEVUse Op : operands()) {
    CanonOps.push_back(Op->getCanonical());
    Changed |= CanonOps.back() != Op;
  }
 
  if (!Changed) {
    CanonicalSCEV = this;
    return;
  }
 
  // Rebuild the expression from the canonical operands, stripping use flags.
  CanonicalSCEV = SE.getWithOperands(this, CanonOps);
}
 
//===----------------------------------------------------------------------===//
// Implementation of the SCEV class.
//
 
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void SCEV::dump() const {
  print(dbgs());
  dbgs() << '\n';
}
#endif
 
void SCEV::print(raw_ostream &OS) const {
  switch (getSCEVType()) {
  case scConstant:
    cast<SCEVConstant>(this)->getValue()->printAsOperand(OS, false);
    return;
  case scVScale:
    OS << "vscale";
    return;
  case scPtrToAddr: {
    const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(this);
    SCEVUse Op = PtrCast->getOperand();
    OS << "(ptrtoaddr " << *Op->getType() << " " << Op << " to "
       << *PtrCast->getType() << ")";
    return;
  }
  case scTruncate: {
    const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(this);
    SCEVUse Op = Trunc->getOperand();
    OS << "(trunc " << *Op->getType() << " " << Op << " to "
       << *Trunc->getType() << ")";
    return;
  }
  case scZeroExtend: {
    const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(this);
    SCEVUse Op = ZExt->getOperand();
    OS << "(zext " << *Op->getType() << " " << Op << " to " << *ZExt->getType()
       << ")";
    return;
  }
  case scSignExtend: {
    const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(this);
    SCEVUse Op = SExt->getOperand();
    OS << "(sext " << *Op->getType() << " " << Op << " to " << *SExt->getType()
       << ")";
    return;
  }
  case scAddRecExpr: {
    const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(this);
    OS << "{" << AR->getOperand(0);
    for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
      OS << ",+," << AR->getOperand(i);
    OS << "}<";
    if (AR->hasNoUnsignedWrap())
      OS << "nuw><";
    if (AR->hasNoSignedWrap())
      OS << "nsw><";
    if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
        !AR->hasNoSignedWrap())
      OS << "nw><";
    AR->getLoop()->getHeader()->printAsOperand(OS, /*PrintType=*/false);
    OS << ">";
    return;
  }
  case scAddExpr:
  case scMulExpr:
  case scUMaxExpr:
  case scSMaxExpr:
  case scUMinExpr:
  case scSMinExpr:
  case scSequentialUMinExpr: {
    const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(this);
    const char *OpStr = nullptr;
    switch (NAry->getSCEVType()) {
    case scAddExpr: OpStr = " + "; break;
    case scMulExpr: OpStr = " * "; break;
    case scUMaxExpr: OpStr = " umax "; break;
    case scSMaxExpr: OpStr = " smax "; break;
    case scUMinExpr:
      OpStr = " umin ";
      break;
    case scSMinExpr:
      OpStr = " smin ";
      break;
    case scSequentialUMinExpr:
      OpStr = " umin_seq ";
      break;
    default:
      llvm_unreachable("There are no other nary expression types.");
    }
    OS << "(" << llvm::interleaved(NAry->operands(), OpStr) << ")";
    switch (NAry->getSCEVType()) {
    case scAddExpr:
    case scMulExpr:
      if (NAry->hasNoUnsignedWrap())
        OS << "<nuw>";
      if (NAry->hasNoSignedWrap())
        OS << "<nsw>";
      break;
    default:
      // Nothing to print for other nary expressions.
      break;
    }
    return;
  }
  case scUDivExpr: {
    const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(this);
    OS << "(" << UDiv->getLHS() << " /u " << UDiv->getRHS() << ")";
    return;
  }
  case scUnknown:
    cast<SCEVUnknown>(this)->getValue()->printAsOperand(OS, false);
    return;
  case scCouldNotCompute:
    OS << "***COULDNOTCOMPUTE***";
    return;
  }
  llvm_unreachable("Unknown SCEV kind!");
}
 
ArrayRef<SCEVUse> SCEV::operands() const {
  switch (getSCEVType()) {
  case scConstant:
  case scVScale:
  case scUnknown:
    return {};
  case scPtrToAddr:
  case scTruncate:
  case scZeroExtend:
  case scSignExtend:
    return cast<SCEVCastExpr>(this)->operands();
  case scAddRecExpr:
  case scAddExpr:
  case scMulExpr:
  case scUMaxExpr:
  case scSMaxExpr:
  case scUMinExpr:
  case scSMinExpr:
  case scSequentialUMinExpr:
    return cast<SCEVNAryExpr>(this)->operands();
  case scUDivExpr:
    return cast<SCEVUDivExpr>(this)->operands();
  case scCouldNotCompute:
    llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
  }
  llvm_unreachable("Unknown SCEV kind!");
}
 
bool SCEV::isZero() const { return match(this, m_scev_Zero()); }
 
bool SCEV::isOne() const { return match(this, m_scev_One()); }
 
bool SCEV::isAllOnesValue() const { return match(this, m_scev_AllOnes()); }
 
bool SCEV::isNonConstantNegative() const {
  const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(this);
  if (!Mul) return false;
 
  // If there is a constant factor, it will be first.
  const SCEVConstant *SC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
  if (!SC) return false;
 
  // Return true if the value is negative, this matches things like (-42 * V).
  return SC->getAPInt().isNegative();
}
 
SCEVCouldNotCompute::SCEVCouldNotCompute()
    : SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0, nullptr) {}
 
bool SCEVCouldNotCompute::classof(const SCEV *S) {
  return S->getSCEVType() == scCouldNotCompute;
}
 
const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
  auto &Entry = ConstantSCEVs[V];
  if (Entry)
    return Entry;
 
  FoldingSetNodeID ID;
  ID.AddInteger(scConstant);
  ID.AddPointer(V);
  FoldingSetInsertToken Token;
  if (SCEVConstant *S =
          static_cast<SCEVConstant *>(UniqueSCEVs.lookup(ID, Token)))
    return Entry = S;
  SCEVConstant *S =
      new (SCEVAllocator) SCEVConstant(ID.Intern(SCEVAllocator), V);
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  return Entry = S;
}
 
const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
  return getConstant(ConstantInt::get(getContext(), Val));
}
 
const SCEV *
ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
  IntegerType *ITy = cast<IntegerType>(getEffectiveSCEVType(Ty));
  // TODO: Avoid implicit trunc?
  // See https://github.com/llvm/llvm-project/issues/112510.
  return getConstant(
      ConstantInt::get(ITy, V, isSigned, /*ImplicitTrunc=*/true));
}
 
const SCEV *ScalarEvolution::getVScale(Type *Ty) {
  FoldingSetNodeID ID;
  ID.AddInteger(scVScale);
  ID.AddPointer(Ty);
  FoldingSetInsertToken Token;
  if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
    return S;
  SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(SCEVAllocator), Ty);
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  return S;
}
 
const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
                                             SCEV::NoWrapFlags Flags) {
  const SCEV *Res = getConstant(Ty, EC.getKnownMinValue());
  if (EC.isScalable())
    Res = getMulExpr(Res, getVScale(Ty), Flags);
  return Res;
}
 
SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
                           SCEVUse op, Type *ty)
    : SCEV(ID, SCEVTy, computeExpressionSize(op), ty), Op(op) {}
 
SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
                                     const SCEV *Op, Type *ITy)
    : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
  assert(getOperand()->getType()->isPointerTy() && getType()->isIntegerTy() &&
         "Must be a non-bit-width-changing pointer-to-integer cast!");
}
 
SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
                                           SCEVTypes SCEVTy, SCEVUse op,
                                           Type *ty)
    : SCEVCastExpr(ID, SCEVTy, op, ty) {}
 
SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
                                   Type *ty)
    : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
  assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
         "Cannot truncate non-integer value!");
}
 
SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
                                       Type *ty)
    : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
  assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
         "Cannot zero extend non-integer value!");
}
 
SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
                                       Type *ty)
    : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
  assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
         "Cannot sign extend non-integer value!");
}
 
void SCEVUnknown::deleted() {
  // Clear this SCEVUnknown from various maps.
  SE->forgetMemoizedResults({this});
 
  // Remove this SCEVUnknown from the uniquing map.
  SE->UniqueSCEVs.erase(this);
 
  // Release the value.
  setValPtr(nullptr);
}
 
void SCEVUnknown::allUsesReplacedWith(Value *New) {
  // Clear this SCEVUnknown from various maps.
  SE->forgetMemoizedResults({this});
 
  // Remove this SCEVUnknown from the uniquing map.
  SE->UniqueSCEVs.erase(this);
 
  // Replace the value pointer in case someone is still using this SCEVUnknown.
  setValPtr(New);
}
 
//===----------------------------------------------------------------------===//
//                               SCEV Utilities
//===----------------------------------------------------------------------===//
 
/// Compare the two values \p LV and \p RV in terms of their "complexity" where
/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
/// operands in SCEV expressions.
static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
                                  Value *RV, unsigned Depth) {
  if (Depth > MaxValueCompareDepth)
    return 0;
 
  // Order pointer values after integer values. This helps SCEVExpander form
  // GEPs.
  bool LIsPointer = LV->getType()->isPointerTy(),
       RIsPointer = RV->getType()->isPointerTy();
  if (LIsPointer != RIsPointer)
    return (int)LIsPointer - (int)RIsPointer;
 
  // Compare getValueID values.
  unsigned LID = LV->getValueID(), RID = RV->getValueID();
  if (LID != RID)
    return (int)LID - (int)RID;
 
  // Sort arguments by their position.
  if (const auto *LA = dyn_cast<Argument>(LV)) {
    const auto *RA = cast<Argument>(RV);
    unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
    return (int)LArgNo - (int)RArgNo;
  }
 
  if (const auto *LGV = dyn_cast<GlobalValue>(LV)) {
    const auto *RGV = cast<GlobalValue>(RV);
 
    if (auto L = LGV->getLinkage() - RGV->getLinkage())
      return L;
 
    const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
      auto LT = GV->getLinkage();
      return !(GlobalValue::isPrivateLinkage(LT) ||
               GlobalValue::isInternalLinkage(LT));
    };
 
    // Use the names to distinguish the two values, but only if the
    // names are semantically important.
    if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
      return LGV->getName().compare(RGV->getName());
  }
 
  // For instructions, compare their loop depth, and their operand count.  This
  // is pretty loose.
  if (const auto *LInst = dyn_cast<Instruction>(LV)) {
    const auto *RInst = cast<Instruction>(RV);
 
    // Compare loop depths.
    const BasicBlock *LParent = LInst->getParent(),
                     *RParent = RInst->getParent();
    if (LParent != RParent) {
      unsigned LDepth = LI->getLoopDepth(LParent),
               RDepth = LI->getLoopDepth(RParent);
      if (LDepth != RDepth)
        return (int)LDepth - (int)RDepth;
    }
 
    // Compare the number of operands.
    unsigned LNumOps = LInst->getNumOperands(),
             RNumOps = RInst->getNumOperands();
    if (LNumOps != RNumOps)
      return (int)LNumOps - (int)RNumOps;
 
    for (unsigned Idx : seq(LNumOps)) {
      int Result = CompareValueComplexity(LI, LInst->getOperand(Idx),
                                          RInst->getOperand(Idx), Depth + 1);
      if (Result != 0)
        return Result;
    }
  }
 
  return 0;
}
 
// Return negative, zero, or positive, if LHS is less than, equal to, or greater
// than RHS, respectively. A three-way result allows recursive comparisons to be
// more efficient.
// If the max analysis depth was reached, return std::nullopt, assuming we do
// not know if they are equivalent for sure.
static std::optional<int>
CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
                      const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
  // Fast-path: SCEVs are uniqued so we can do a quick equality check.
  if (LHS == RHS)
    return 0;
 
  // Primarily, sort the SCEVs by their getSCEVType().
  SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
  if (LType != RType)
    return (int)LType - (int)RType;
 
  if (Depth > MaxSCEVCompareDepth)
    return std::nullopt;
 
  // Aside from the getSCEVType() ordering, the particular ordering
  // isn't very important except that it's beneficial to be consistent,
  // so that (a + b) and (b + a) don't end up as different expressions.
  switch (LType) {
  case scUnknown: {
    const SCEVUnknown *LU = cast<SCEVUnknown>(LHS);
    const SCEVUnknown *RU = cast<SCEVUnknown>(RHS);
 
    int X =
        CompareValueComplexity(LI, LU->getValue(), RU->getValue(), Depth + 1);
    return X;
  }
 
  case scConstant: {
    const SCEVConstant *LC = cast<SCEVConstant>(LHS);
    const SCEVConstant *RC = cast<SCEVConstant>(RHS);
 
    // Compare constant values.
    const APInt &LA = LC->getAPInt();
    const APInt &RA = RC->getAPInt();
    unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
    if (LBitWidth != RBitWidth)
      return (int)LBitWidth - (int)RBitWidth;
    return LA.ult(RA) ? -1 : 1;
  }
 
  case scVScale: {
    const auto *LTy = cast<IntegerType>(cast<SCEVVScale>(LHS)->getType());
    const auto *RTy = cast<IntegerType>(cast<SCEVVScale>(RHS)->getType());
    return LTy->getBitWidth() - RTy->getBitWidth();
  }
 
  case scAddRecExpr: {
    const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(LHS);
    const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(RHS);
 
    // There is always a dominance between two recs that are used by one SCEV,
    // so we can safely sort recs by loop header dominance. We require such
    // order in getAddExpr.
    const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
    if (LLoop != RLoop) {
      const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
      assert(LHead != RHead && "Two loops share the same header?");
      if (DT.dominates(LHead, RHead))
        return 1;
      assert(DT.dominates(RHead, LHead) &&
             "No dominance between recurrences used by one SCEV?");
      return -1;
    }
 
    [[fallthrough]];
  }
 
  case scTruncate:
  case scZeroExtend:
  case scSignExtend:
  case scPtrToAddr:
  case scAddExpr:
  case scMulExpr:
  case scUDivExpr:
  case scSMaxExpr:
  case scUMaxExpr:
  case scSMinExpr:
  case scUMinExpr:
  case scSequentialUMinExpr: {
    ArrayRef<SCEVUse> LOps = LHS->operands();
    ArrayRef<SCEVUse> ROps = RHS->operands();
 
    // Lexicographically compare n-ary-like expressions.
    unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
    if (LNumOps != RNumOps)
      return (int)LNumOps - (int)RNumOps;
 
    for (unsigned i = 0; i != LNumOps; ++i) {
      auto X = CompareSCEVComplexity(LI, LOps[i].getPointer(),
                                     ROps[i].getPointer(), DT, Depth + 1);
      if (X != 0)
        return X;
    }
    return 0;
  }
 
  case scCouldNotCompute:
    llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
  }
  llvm_unreachable("Unknown SCEV kind!");
}
 
/// Given a list of SCEV objects, order them by their complexity, and group
/// objects of the same complexity together by value.  When this routine is
/// finished, we know that any duplicates in the vector are consecutive and that
/// complexity is monotonically increasing.
///
/// Note that we go take special precautions to ensure that we get deterministic
/// results from this routine.  In other words, we don't want the results of
/// this to depend on where the addresses of various SCEV objects happened to
/// land in memory.
static void GroupByComplexity(SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
                              DominatorTree &DT) {
  if (Ops.size() < 2) return;  // Noop
 
  // Whether LHS has provably less complexity than RHS.
  auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
    auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
    return Complexity && *Complexity < 0;
  };
  if (Ops.size() == 2) {
    // This is the common case, which also happens to be trivially simple.
    // Special case it.
    SCEVUse &LHS = Ops[0], &RHS = Ops[1];
    if (IsLessComplex(RHS, LHS))
      std::swap(LHS, RHS);
    return;
  }
 
  // Do the rough sort by complexity.
  llvm::stable_sort(
      Ops, [&](SCEVUse LHS, SCEVUse RHS) { return IsLessComplex(LHS, RHS); });
 
  // Now that we are sorted by complexity, group elements of the same
  // complexity.  Note that this is, at worst, N^2, but the vector is likely to
  // be extremely short in practice.  Note that we take this approach because we
  // do not want to depend on the addresses of the objects we are grouping.
  for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
    const SCEV *S = Ops[i];
    unsigned Complexity = S->getSCEVType();
 
    // If there are any objects of the same complexity and same value as this
    // one, group them.
    for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
      if (Ops[j] == S) { // Found a duplicate.
        // Move it to immediately after i'th element.
        std::swap(Ops[i+1], Ops[j]);
        ++i;   // no need to rescan it.
        if (i == e-2) return;  // Done!
      }
    }
  }
}
 
/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
/// least HugeExprThreshold nodes).
static bool hasHugeExpression(ArrayRef<SCEVUse> Ops) {
  return any_of(Ops, [](const SCEV *S) {
    return S->getExpressionSize() >= HugeExprThreshold;
  });
}
 
/// Performs a number of common optimizations on the passed \p Ops. If the
/// whole expression reduces down to a single operand, it will be returned.
///
/// The following optimizations are performed:
///  * Fold constants using the \p Fold function.
///  * Remove identity constants satisfying \p IsIdentity.
///  * If a constant satisfies \p IsAbsorber, return it.
///  * Sort operands by complexity.
template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
static const SCEV *
constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
                        SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
                        IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
  const SCEVConstant *Folded = nullptr;
  for (unsigned Idx = 0; Idx < Ops.size();) {
    const SCEV *Op = Ops[Idx];
    if (const auto *C = dyn_cast<SCEVConstant>(Op)) {
      if (!Folded)
        Folded = C;
      else
        Folded = cast<SCEVConstant>(
            SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
      Ops.erase(Ops.begin() + Idx);
      continue;
    }
    ++Idx;
  }
 
  if (Ops.empty()) {
    assert(Folded && "Must have folded value");
    return Folded;
  }
 
  if (Folded && IsAbsorber(Folded->getAPInt()))
    return Folded;
 
  GroupByComplexity(Ops, &LI, DT);
  if (Folded && !IsIdentity(Folded->getAPInt()))
    Ops.insert(Ops.begin(), Folded);
 
  return Ops.size() == 1 ? Ops[0] : nullptr;
}
 
//===----------------------------------------------------------------------===//
//                      Simple SCEV method implementations
//===----------------------------------------------------------------------===//
 
/// Compute BC(It, K).  The result has width W.  Assume, K > 0.
static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
                                       ScalarEvolution &SE,
                                       Type *ResultTy) {
  // Handle the simplest case efficiently.
  if (K == 1)
    return SE.getTruncateOrZeroExtend(It, ResultTy);
 
  // We are using the following formula for BC(It, K):
  //
  //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
  //
  // Suppose, W is the bitwidth of the return value.  We must be prepared for
  // overflow.  Hence, we must assure that the result of our computation is
  // equal to the accurate one modulo 2^W.  Unfortunately, division isn't
  // safe in modular arithmetic.
  //
  // However, this code doesn't use exactly that formula; the formula it uses
  // is something like the following, where T is the number of factors of 2 in
  // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
  // exponentiation:
  //
  //   BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
  //
  // This formula is trivially equivalent to the previous formula.  However,
  // this formula can be implemented much more efficiently.  The trick is that
  // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
  // arithmetic.  To do exact division in modular arithmetic, all we have
  // to do is multiply by the inverse.  Therefore, this step can be done at
  // width W.
  //
  // The next issue is how to safely do the division by 2^T.  The way this
  // is done is by doing the multiplication step at a width of at least W + T
  // bits.  This way, the bottom W+T bits of the product are accurate. Then,
  // when we perform the division by 2^T (which is equivalent to a right shift
  // by T), the bottom W bits are accurate.  Extra bits are okay; they'll get
  // truncated out after the division by 2^T.
  //
  // In comparison to just directly using the first formula, this technique
  // is much more efficient; using the first formula requires W * K bits,
  // but this formula less than W + K bits. Also, the first formula requires
  // a division step, whereas this formula only requires multiplies and shifts.
  //
  // It doesn't matter whether the subtraction step is done in the calculation
  // width or the input iteration count's width; if the subtraction overflows,
  // the result must be zero anyway.  We prefer here to do it in the width of
  // the induction variable because it helps a lot for certain cases; CodeGen
  // isn't smart enough to ignore the overflow, which leads to much less
  // efficient code if the width of the subtraction is wider than the native
  // register width.
  //
  // (It's possible to not widen at all by pulling out factors of 2 before
  // the multiplication; for example, K=2 can be calculated as
  // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
  // extra arithmetic, so it's not an obvious win, and it gets
  // much more complicated for K > 3.)
 
  // Protection from insane SCEVs; this bound is conservative,
  // but it probably doesn't matter.
  if (K > 1000)
    return SE.getCouldNotCompute();
 
  unsigned W = SE.getTypeSizeInBits(ResultTy);
 
  // Calculate K! / 2^T and T; we divide out the factors of two before
  // multiplying for calculating K! / 2^T to avoid overflow.
  // Other overflow doesn't matter because we only care about the bottom
  // W bits of the result.
  APInt OddFactorial(W, 1);
  unsigned T = 1;
  for (unsigned i = 3; i <= K; ++i) {
    unsigned TwoFactors = countr_zero(i);
    T += TwoFactors;
    OddFactorial *= (i >> TwoFactors);
  }
 
  // We need at least W + T bits for the multiplication step
  unsigned CalculationBits = W + T;
 
  // Calculate 2^T, at width T+W.
  APInt DivFactor = APInt::getOneBitSet(CalculationBits, T);
 
  // Calculate the multiplicative inverse of K! / 2^T;
  // this multiplication factor will perform the exact division by
  // K! / 2^T.
  APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
 
  // Calculate the product, at width T+W
  IntegerType *CalculationTy = IntegerType::get(SE.getContext(),
                                                      CalculationBits);
  const SCEV *Dividend = SE.getTruncateOrZeroExtend(It, CalculationTy);
  for (unsigned i = 1; i != K; ++i) {
    const SCEV *S = SE.getMinusSCEV(It, SE.getConstant(It->getType(), i));
    Dividend = SE.getMulExpr(Dividend,
                             SE.getTruncateOrZeroExtend(S, CalculationTy));
  }
 
  // Divide by 2^T
  const SCEV *DivResult = SE.getUDivExpr(Dividend, SE.getConstant(DivFactor));
 
  // Truncate the result, and divide by K! / 2^T.
 
  return SE.getMulExpr(SE.getConstant(MultiplyFactor),
                       SE.getTruncateOrZeroExtend(DivResult, ResultTy));
}
 
/// Return the value of this chain of recurrences at the specified iteration
/// number.  We can evaluate this recurrence by multiplying each element in the
/// chain by the binomial coefficient corresponding to it.  In other words, we
/// can evaluate {A,+,B,+,C,+,D} as:
///
///   A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
///
/// where BC(It, k) stands for binomial coefficient.
const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
                                                ScalarEvolution &SE) const {
  return evaluateAtIteration(operands(), It, SE);
}
 
const SCEV *SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
                                                const SCEV *It,
                                                ScalarEvolution &SE) {
  assert(Operands.size() > 0);
  const SCEV *Result = Operands[0].getPointer();
  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
    // The computation is correct in the face of overflow provided that the
    // multiplication is performed _after_ the evaluation of the binomial
    // coefficient.
    const SCEV *Coeff = BinomialCoefficient(It, i, SE, Result->getType());
    if (isa<SCEVCouldNotCompute>(Coeff))
      return Coeff;
 
    Result =
        SE.getAddExpr(Result, SE.getMulExpr(Operands[i].getPointer(), Coeff));
  }
  return Result;
}
 
//===----------------------------------------------------------------------===//
//                    SCEV Expression folder implementations
//===----------------------------------------------------------------------===//
 
/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
/// which computes a pointer-typed value, and rewrites the whole expression
/// tree so that *all* the computations are done on integers, and the only
/// pointer-typed operands in the expression are SCEVUnknown.
/// The CreatePtrCast callback is invoked to create the actual conversion
/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
class SCEVCastSinkingRewriter
    : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
  using Base = SCEVRewriteVisitor<SCEVCastSinkingRewriter>;
  using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
  Type *TargetTy;
  ConversionFn CreatePtrCast;
 
public:
  SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy,
                          ConversionFn CreatePtrCast)
      : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
 
  static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
                             Type *TargetTy, ConversionFn CreatePtrCast) {
    SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
    return Rewriter.visit(Scev);
  }
 
  const SCEV *visit(const SCEV *S) {
    Type *STy = S->getType();
    // If the expression is not pointer-typed, just keep it as-is.
    if (!STy->isPointerTy())
      return S;
    // Else, recursively sink the cast down into it.
    return Base::visit(S);
  }
 
  const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
    // Preserve wrap flags on rewritten SCEVAddExpr, which the default
    // implementation drops.
    SmallVector<SCEVUse, 2> Operands;
    bool Changed = false;
    for (SCEVUse Op : Expr->operands()) {
      Operands.push_back(visit(Op.getPointer()));
      Changed |= Op.getPointer() != Operands.back();
    }
    return !Changed ? Expr : SE.getAddExpr(Operands, Expr->getNoWrapFlags());
  }
 
  const SCEV *visitUnknown(const SCEVUnknown *Expr) {
    assert(Expr->getType()->isPointerTy() &&
           "Should only reach pointer-typed SCEVUnknown's.");
    // Perform some basic constant folding. If the operand of the cast is a
    // null pointer, don't create a cast SCEV expression (that will be left
    // as-is), but produce a zero constant.
    if (isa<ConstantPointerNull>(Expr->getValue()))
      return SE.getZero(TargetTy);
    return CreatePtrCast(Expr);
  }
};
 
const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
  assert(Op->getType()->isPointerTy() && "Op must be a pointer");
 
  // Treat pointers with unstable representation conservatively, since the
  // address bits may change.
  if (DL.hasUnstableRepresentation(Op->getType()))
    return getCouldNotCompute();
 
  Type *Ty = DL.getAddressType(Op->getType());
 
  // Use the rewriter to sink the cast down to SCEVUnknown leaves.
  // The rewriter handles null pointer constant folding.
  const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
      Op, *this, Ty, [this, Ty](const SCEVUnknown *U) {
        FoldingSetNodeID ID;
        ID.AddInteger(scPtrToAddr);
        ID.AddPointer(U);
        ID.AddPointer(Ty);
        FoldingSetInsertToken Token;
        if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
          return S;
        SCEV *S = new (SCEVAllocator)
            SCEVPtrToAddrExpr(ID.Intern(SCEVAllocator), U, Ty);
        UniqueSCEVs.insert(S, Token);
        S->computeAndSetCanonical(*this);
        registerUser(S, U);
        return static_cast<const SCEV *>(S);
      });
  assert(IntOp->getType()->isIntegerTy() &&
         "We must have succeeded in sinking the cast, "
         "and ending up with an integer-typed expression!");
  return IntOp;
}
 
const SCEV *ScalarEvolution::getTruncateExpr(SCEVUse Op, Type *Ty,
                                             unsigned Depth) {
  assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
         "This is not a truncating conversion!");
  assert(isSCEVable(Ty) &&
         "This is not a conversion to a SCEVable type!");
  assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
  Ty = getEffectiveSCEVType(Ty);
 
  FoldingSetNodeID ID;
  ID.AddInteger(scTruncate);
  ID.AddPointer(Op.getOpaqueValue());
  ID.AddPointer(Ty);
  FoldingSetInsertToken Token;
  if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
    return S;
 
  // Fold if the operand is constant.
  if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
    return getConstant(
      cast<ConstantInt>(ConstantExpr::getTrunc(SC->getValue(), Ty)));
 
  // trunc(trunc(x)) --> trunc(x)
  if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op))
    return getTruncateExpr(ST->getOperand(), Ty, Depth + 1);
 
  // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
  if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
    return getTruncateOrSignExtend(SS->getOperand(), Ty, Depth + 1);
 
  // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
  if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
    return getTruncateOrZeroExtend(SZ->getOperand(), Ty, Depth + 1);
 
  if (Depth > MaxCastDepth) {
    SCEV *S =
        new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty);
    UniqueSCEVs.insert(S, Token);
    S->computeAndSetCanonical(*this);
    registerUser(S, Op);
    return S;
  }
 
  // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
  // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
  // if after transforming we have at most one truncate, not counting truncates
  // that replace other casts.
  if (isa<SCEVAddExpr>(Op) || isa<SCEVMulExpr>(Op)) {
    auto *CommOp = cast<SCEVCommutativeExpr>(Op);
    SmallVector<SCEVUse, 4> Operands;
    unsigned numTruncs = 0;
    for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
         ++i) {
      const SCEV *S = getTruncateExpr(CommOp->getOperand(i), Ty, Depth + 1);
      if (!isa<SCEVIntegralCastExpr>(CommOp->getOperand(i)) &&
          isa<SCEVTruncateExpr>(S))
        numTruncs++;
      Operands.push_back(S);
    }
    if (numTruncs < 2) {
      if (isa<SCEVAddExpr>(Op))
        return getAddExpr(Operands);
      if (isa<SCEVMulExpr>(Op))
        return getMulExpr(Operands);
      llvm_unreachable("Unexpected SCEV type for Op.");
    }
    // Although we checked in the beginning that ID is not in the cache, it is
    // possible that during recursion and different modification ID was inserted
    // into the cache. So if we find it, just return it.
    if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
      return S;
  }
 
  // If the input value is a chrec scev, truncate the chrec's operands.
  if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Op)) {
    SmallVector<SCEVUse, 4> Operands;
    for (const SCEV *Op : AddRec->operands())
      Operands.push_back(getTruncateExpr(Op, Ty, Depth + 1));
    return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap);
  }
 
  // Return zero if truncating to known zeros.
  uint32_t MinTrailingZeros = getMinTrailingZeros(Op);
  if (MinTrailingZeros >= getTypeSizeInBits(Ty))
    return getZero(Ty);
 
  // The cast wasn't folded; create an explicit cast node. We can reuse
  // the existing insert position since if we get here, we won't have
  // made any changes which would invalidate it.
  SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator),
                                                 Op, Ty);
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  registerUser(S, Op);
  return S;
}
 
// Get the limit of a recurrence such that incrementing by Step cannot cause
// signed overflow as long as the value of the recurrence within the
// loop does not exceed this limit before incrementing.
static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
                                                 ICmpInst::Predicate *Pred,
                                                 ScalarEvolution *SE) {
  unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
  if (SE->isKnownPositive(Step)) {
    *Pred = ICmpInst::ICMP_SLT;
    return SE->getConstant(APInt::getSignedMinValue(BitWidth) -
                           SE->getSignedRangeMax(Step));
  }
  if (SE->isKnownNegative(Step)) {
    *Pred = ICmpInst::ICMP_SGT;
    return SE->getConstant(APInt::getSignedMaxValue(BitWidth) -
                           SE->getSignedRangeMin(Step));
  }
  return nullptr;
}
 
// Get the limit of a recurrence such that incrementing by Step cannot cause
// unsigned overflow as long as the value of the recurrence within the loop does
// not exceed this limit before incrementing.
static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
                                                   ICmpInst::Predicate *Pred,
                                                   ScalarEvolution *SE) {
  unsigned BitWidth = SE->getTypeSizeInBits(Step->getType());
  *Pred = ICmpInst::ICMP_ULT;
 
  return SE->getConstant(APInt::getMinValue(BitWidth) -
                         SE->getUnsignedRangeMax(Step));
}
 
namespace {
 
struct ExtendOpTraitsBase {
  typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(SCEVUse, Type *,
                                                          unsigned);
};
 
// Used to make code generic over signed and unsigned overflow.
template <typename ExtendOp> struct ExtendOpTraits {
  // Members present:
  //
  // static const SCEV::NoWrapFlags WrapType;
  //
  // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
  //
  // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
  //                                           ICmpInst::Predicate *Pred,
  //                                           ScalarEvolution *SE);
};
 
template <>
struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
  static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
 
  static const GetExtendExprTy GetExtendExpr;
 
  static const SCEV *getOverflowLimitForStep(const SCEV *Step,
                                             ICmpInst::Predicate *Pred,
                                             ScalarEvolution *SE) {
    return getSignedOverflowLimitForStep(Step, Pred, SE);
  }
};
 
const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
    SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
 
template <>
struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
  static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
 
  static const GetExtendExprTy GetExtendExpr;
 
  static const SCEV *getOverflowLimitForStep(const SCEV *Step,
                                             ICmpInst::Predicate *Pred,
                                             ScalarEvolution *SE) {
    return getUnsignedOverflowLimitForStep(Step, Pred, SE);
  }
};
 
const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
    SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
 
} // end anonymous namespace
 
// The recurrence AR has been shown to have no signed/unsigned wrap or something
// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
// easily prove NSW/NUW for its preincrement or postincrement sibling. This
// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
// expression "Step + sext/zext(PreIncAR)" is congruent with
// "sext/zext(PostIncAR)"
template <typename ExtendOpTy>
static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR,
                                        ScalarEvolution *SE, unsigned Depth) {
  auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
  auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
 
  const Loop *L = AR->getLoop();
  const SCEV *Start = AR->getStart();
  const SCEV *Step = AR->getStepRecurrence(*SE);
 
  // Check for a simple looking step prior to loop entry.
  const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Start);
  if (!SA)
    return nullptr;
 
  // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
  // subtraction is expensive. For this purpose, perform a quick and dirty
  // difference, by checking for Step in the operand list. Note, that
  // SA might have repeated ops, like %a + %a + ..., so only remove one.
  SmallVector<SCEVUse, 4> DiffOps(SA->operands());
  for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
    if (*It == Step) {
      DiffOps.erase(It);
      break;
    }
 
  if (DiffOps.size() == SA->getNumOperands())
    return nullptr;
 
  // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
  // `Step`:
 
  // 1. NSW/NUW flags on the step increment.
  auto PreStartFlags =
    ScalarEvolution::maskFlags(SA->getNoWrapFlags(), SCEV::FlagNUW);
  const SCEV *PreStart = SE->getAddExpr(DiffOps, PreStartFlags);
  const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
      SE->getAddRecExpr(PreStart, Step, L, SCEV::FlagAnyWrap));
 
  // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
  // "S+X does not sign/unsign-overflow".
  //
 
  const SCEV *BECount = SE->getBackedgeTakenCount(L);
  if (PreAR && any(PreAR->getNoWrapFlags(WrapType)) &&
      !isa<SCEVCouldNotCompute>(BECount) && SE->isKnownPositive(BECount))
    return PreStart;
 
  // 2. Direct overflow check on the step operation's expression.
  unsigned BitWidth = SE->getTypeSizeInBits(AR->getType());
  Type *WideTy = IntegerType::get(SE->getContext(), BitWidth * 2);
  const SCEV *OperandExtendedStart =
      SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
                     (SE->*GetExtendExpr)(Step, WideTy, Depth));
  if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
    if (PreAR && any(AR->getNoWrapFlags(WrapType))) {
      // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
      // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
      // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`.  Cache this fact.
      SE->setNoWrapFlags(const_cast<SCEVAddRecExpr *>(PreAR), WrapType);
    }
    return PreStart;
  }
 
  // 3. Loop precondition.
  ICmpInst::Predicate Pred;
  const SCEV *OverflowLimit =
      ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
 
  if (OverflowLimit &&
      SE->isLoopEntryGuardedByCond(L, Pred, PreStart, OverflowLimit))
    return PreStart;
 
  return nullptr;
}
 
// Get the normalized zero or sign extended expression for this AddRec's Start.
template <typename ExtendOpTy>
static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
                                        ScalarEvolution *SE,
                                        unsigned Depth) {
  auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
 
  const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, SE, Depth);
  if (!PreStart)
    return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
 
  return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(*SE), Ty,
                                             Depth),
                        (SE->*GetExtendExpr)(PreStart, Ty, Depth));
}
 
// Try to prove away overflow by looking at "nearby" add recurrences.  A
// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
//
// Formally:
//
//     {S,+,X} == {S-T,+,X} + T
//  => Ext({S,+,X}) == Ext({S-T,+,X} + T)
//
// If ({S-T,+,X} + T) does not overflow  ... (1)
//
//  RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
//
// If {S-T,+,X} does not overflow  ... (2)
//
//  RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
//      == {Ext(S-T)+Ext(T),+,Ext(X)}
//
// If (S-T)+T does not overflow  ... (3)
//
//  RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
//      == {Ext(S),+,Ext(X)} == LHS
//
// Thus, if (1), (2) and (3) are true for some T, then
//   Ext({S,+,X}) == {Ext(S),+,Ext(X)}
//
// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
// does not overflow" restricted to the 0th iteration.  Therefore we only need
// to check for (1) and (2).
//
// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
// is `Delta` (defined below).
template <typename ExtendOpTy>
bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
                                                const SCEV *Step,
                                                const Loop *L) {
  auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
 
  // We restrict `Start` to a constant to prevent SCEV from spending too much
  // time here.  It is correct (but more expensive) to continue with a
  // non-constant `Start` and do a general SCEV subtraction to compute
  // `PreStart` below.
  const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Start);
  if (!StartC)
    return false;
 
  APInt StartAI = StartC->getAPInt();
 
  for (unsigned Delta : {-2, -1, 1, 2}) {
    const SCEV *PreStart = getConstant(StartAI - Delta);
 
    FoldingSetNodeID ID;
    ID.AddInteger(scAddRecExpr);
    ID.AddPointer(PreStart);
    ID.AddPointer(Step);
    ID.AddPointer(L);
    FoldingSetInsertToken Token;
    const auto *PreAR =
        static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
 
    // Give up if we don't already have the add recurrence we need because
    // actually constructing an add recurrence is relatively expensive.
    if (PreAR && any(PreAR->getNoWrapFlags(WrapType))) { // proves (2)
      const SCEV *DeltaS = getConstant(StartC->getType(), Delta);
      ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
      const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
          DeltaS, &Pred, this);
      if (Limit && isKnownPredicate(Pred, PreAR, Limit))  // proves (1)
        return true;
    }
  }
 
  return false;
}
 
// Finds an integer D for an expression (C + x + y + ...) such that the top
// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
// the (C + x + y + ...) expression is \p WholeAddExpr.
static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
                                            const SCEVConstant *ConstantTerm,
                                            const SCEVAddExpr *WholeAddExpr) {
  const APInt &C = ConstantTerm->getAPInt();
  const unsigned BitWidth = C.getBitWidth();
  // Find number of trailing zeros of (x + y + ...) w/o the C first:
  uint32_t TZ = BitWidth;
  for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
    TZ = std::min(TZ, SE.getMinTrailingZeros(WholeAddExpr->getOperand(I)));
  if (TZ) {
    // Set D to be as many least significant bits of C as possible while still
    // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
    return TZ < BitWidth ? C.trunc(TZ).zext(BitWidth) : C;
  }
  return APInt(BitWidth, 0);
}
 
// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
                                            const APInt &ConstantStart,
                                            const SCEV *Step) {
  const unsigned BitWidth = ConstantStart.getBitWidth();
  const uint32_t TZ = SE.getMinTrailingZeros(Step);
  if (TZ)
    return TZ < BitWidth ? ConstantStart.trunc(TZ).zext(BitWidth)
                         : ConstantStart;
  return APInt(BitWidth, 0);
}
 
static void insertFoldCacheEntry(
    const ScalarEvolution::FoldID &ID, const SCEV *S,
    DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
    DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
        &FoldCacheUser) {
  auto I = FoldCache.insert({ID, S});
  if (!I.second) {
    // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
    // entry.
    auto &UserIDs = FoldCacheUser[I.first->second];
    assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
    for (unsigned I = 0; I != UserIDs.size(); ++I)
      if (UserIDs[I] == ID) {
        std::swap(UserIDs[I], UserIDs.back());
        break;
      }
    UserIDs.pop_back();
    I.first->second = S;
  }
  FoldCacheUser[S].push_back(ID);
}
 
const SCEV *ScalarEvolution::getZeroExtendExpr(SCEVUse Op, Type *Ty,
                                               unsigned Depth) {
  assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
         "This is not an extending conversion!");
  assert(isSCEVable(Ty) &&
         "This is not a conversion to a SCEVable type!");
  assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
  Ty = getEffectiveSCEVType(Ty);
 
  FoldID ID(scZeroExtend, Op, Ty);
  if (const SCEV *S = FoldCache.lookup(ID))
    return S;
 
  const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
  if (!isa<SCEVZeroExtendExpr>(S))
    insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
  return S;
}
 
const SCEV *ScalarEvolution::getZeroExtendExprImpl(SCEVUse Op, Type *Ty,
                                                   unsigned Depth) {
  assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
         "This is not an extending conversion!");
  assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
  assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
 
  // Fold if the operand is constant.
  if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
    return getConstant(SC->getAPInt().zext(getTypeSizeInBits(Ty)));
 
  // zext(zext(x)) --> zext(x)
  if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
    return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
 
  // If the operand is an affine AddRec with the no-unsigned-wrap flag, the
  // zero-extension distributes over the recurrence.
  const SCEV *Start, *Step;
  const Loop *L;
  if (Depth <= MaxCastDepth &&
      match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
    const auto *AR = cast<SCEVAddRecExpr>(Op);
    if (AR->hasNoUnsignedWrap()) {
      Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
      Step = getZeroExtendExpr(Step, Ty, Depth + 1);
      return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
    }
  }
 
  // Before doing any expensive analysis, check to see if we've already
  // computed a SCEV for this Op and Ty.
  FoldingSetNodeID ID;
  ID.AddInteger(scZeroExtend);
  ID.AddPointer(Op.getOpaqueValue());
  ID.AddPointer(Ty);
  FoldingSetInsertToken Token;
  if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
    return S;
  if (Depth > MaxCastDepth) {
    SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
                                                     Op, Ty);
    UniqueSCEVs.insert(S, Token);
    S->computeAndSetCanonical(*this);
    registerUser(S, Op);
    return S;
  }
 
  // zext(trunc(x)) --> zext(x) or x or trunc(x)
  if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
    // It's possible the bits taken off by the truncate were all zero bits. If
    // so, we should be able to simplify this further.
    const SCEV *X = ST->getOperand();
    ConstantRange CR = getUnsignedRange(X);
    unsigned TruncBits = getTypeSizeInBits(ST->getType());
    unsigned NewBits = getTypeSizeInBits(Ty);
    if (CR.truncate(TruncBits).zeroExtend(NewBits).contains(
            CR.zextOrTrunc(NewBits)))
      return getTruncateOrZeroExtend(X, Ty, Depth);
  }
 
  // If the input value is a chrec scev, and we can prove that the value
  // did not overflow the old, smaller, value, we can zero extend all of the
  // operands (often constants).  This allows analysis of something like
  // this:  for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
  if (match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
    const auto *AR = cast<SCEVAddRecExpr>(Op);
    unsigned BitWidth = getTypeSizeInBits(AR->getType());
 
    // The no-unsigned-wrap case is handled before the uniquing lookup above.
 
    // Check whether the backedge-taken count is SCEVCouldNotCompute.
    // Note that this serves two purposes: It filters out loops that are
    // simply not analyzable, and it covers the case where this code is
    // being called from within backedge-taken count analysis, such that
    // attempting to ask for the backedge-taken count would likely result
    // in infinite recursion. In the later case, the analysis code will
    // cope with a conservative value, and it will take care to purge
    // that value once it has finished.
    const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
    if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
      // Manually compute the final value for AR, checking for overflow.
 
      // Check whether the backedge-taken count can be losslessly casted to
      // the addrec's type. The count is always unsigned.
      const SCEV *CastedMaxBECount =
          getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
      const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
          CastedMaxBECount, MaxBECount->getType(), Depth);
      if (MaxBECount == RecastedMaxBECount) {
        Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
        // Check whether Start+Step*MaxBECount has no unsigned overflow.
        const SCEV *ZMul =
            getMulExpr(CastedMaxBECount, Step, SCEV::FlagAnyWrap, Depth + 1);
        const SCEV *ZAdd = getZeroExtendExpr(
            getAddExpr(Start, ZMul, SCEV::FlagAnyWrap, Depth + 1), WideTy,
            Depth + 1);
        const SCEV *WideStart = getZeroExtendExpr(Start, WideTy, Depth + 1);
        const SCEV *WideMaxBECount =
            getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
        const SCEV *OperandExtendedAdd =
            getAddExpr(WideStart,
                       getMulExpr(WideMaxBECount,
                                  getZeroExtendExpr(Step, WideTy, Depth + 1),
                                  SCEV::FlagAnyWrap, Depth + 1),
                       SCEV::FlagAnyWrap, Depth + 1);
        if (ZAdd == OperandExtendedAdd) {
          // Cache knowledge of AR NUW, which is propagated to this AddRec.
          setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
          // Return the expression with the addrec on the outside.
          Start =
              getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
          Step = getZeroExtendExpr(Step, Ty, Depth + 1);
          return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
        }
        // Similar to above, only this time treat the step value as signed.
        // This covers loops that count down.
        OperandExtendedAdd =
            getAddExpr(WideStart,
                       getMulExpr(WideMaxBECount,
                                  getSignExtendExpr(Step, WideTy, Depth + 1),
                                  SCEV::FlagAnyWrap, Depth + 1),
                       SCEV::FlagAnyWrap, Depth + 1);
        if (ZAdd == OperandExtendedAdd) {
          // Cache knowledge of AR NW, which is propagated to this AddRec.
          // Negative step causes unsigned wrap, but it still can't self-wrap.
          setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
          // Return the expression with the addrec on the outside.
          Start =
              getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
          Step = getSignExtendExpr(Step, Ty, Depth + 1);
          return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
        }
      }
    }
 
    // Normally, in the cases we can prove no-overflow via a
    // backedge guarding condition, we can also compute a backedge
    // taken count for the loop.  The exceptions are assumptions and
    // guards present in the loop -- SCEV is not great at exploiting
    // these to compute max backedge taken counts, but can still use
    // these to prove lack of overflow.  Use this fact to avoid
    // doing extra work that may not pay off.
    if (!isa<SCEVCouldNotCompute>(MaxBECount) || HasGuards ||
        !AC.assumptions().empty()) {
 
      auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
      setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
      if (AR->hasNoUnsignedWrap()) {
        // Same as nuw case above - duplicated here to avoid a compile time
        // issue.  It's not clear that the order of checks does matter, but
        // it's one of two issue possible causes for a change which was
        // reverted.  Be conservative for the moment.
        Start =
            getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
        Step = getZeroExtendExpr(Step, Ty, Depth + 1);
        return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
      }
 
      // For a negative step, we can extend the operands iff doing so only
      // traverses values in the range zext([0,UINT_MAX]).
      if (isKnownNegative(Step)) {
        const SCEV *N =
            getConstant(APInt::getMaxValue(BitWidth) - getSignedRangeMin(Step));
        if (isLoopBackedgeGuardedByCond(L, ICmpInst::ICMP_UGT, AR, N) ||
            isKnownOnEveryIteration(ICmpInst::ICMP_UGT, AR, N)) {
          // Cache knowledge of AR NW, which is propagated to this
          // AddRec.  Negative step causes unsigned wrap, but it
          // still can't self-wrap.
          setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
          // Return the expression with the addrec on the outside.
          Start =
              getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
          Step = getSignExtendExpr(Step, Ty, Depth + 1);
          return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
        }
      }
    }
 
    // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
    // if D + (C - D + Step * n) could be proven to not unsigned wrap
    // where D maximizes the number of trailing zeros of (C - D + Step * n)
    if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
      const APInt &C = SC->getAPInt();
      const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
      if (D != 0) {
        const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
        const SCEV *SResidual =
            getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
        const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
        return getAddExpr(SZExtD, SZExtR, SCEV::FlagNSW | SCEV::FlagNUW,
                          Depth + 1);
      }
    }
 
    if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
      setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNUW);
      Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, this, Depth + 1);
      Step = getZeroExtendExpr(Step, Ty, Depth + 1);
      return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
    }
  }
 
  // zext(A % B) --> zext(A) % zext(B)
  {
    const SCEV *LHS;
    const SCEV *RHS;
    if (match(Op, m_scev_URem(m_SCEV(LHS), m_SCEV(RHS), *this)))
      return getURemExpr(getZeroExtendExpr(LHS, Ty, Depth + 1),
                         getZeroExtendExpr(RHS, Ty, Depth + 1));
  }
 
  // zext(A / B) --> zext(A) / zext(B).
  if (auto *Div = dyn_cast<SCEVUDivExpr>(Op))
    return getUDivExpr(getZeroExtendExpr(Div->getLHS(), Ty, Depth + 1),
                       getZeroExtendExpr(Div->getRHS(), Ty, Depth + 1));
 
  if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
    // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
    if (SA->hasNoUnsignedWrap()) {
      // If the addition does not unsign overflow then we can, by definition,
      // commute the zero extension with the addition operation.
      SmallVector<SCEVUse, 4> Ops;
      for (SCEVUse Op : SA->operands())
        Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
      return getAddExpr(Ops, SCEV::FlagNUW, Depth + 1);
    }
 
    const APInt *C, *C2;
    // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
    // Currently the non-negative check is done manually, as isKnownNonNegative
    // is too expensive.
    if (SA->hasNoSignedWrap() &&
        match(SA, m_scev_Add(m_scev_APInt(C),
                             m_scev_SMax(m_scev_APInt(C2), m_SCEV()))) &&
        C->isNegative() && !C->isMinSignedValue() && C2->sge(C->abs())) {
      assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
      return getAddExpr(getSignExtendExpr(SA->getOperand(0), Ty, Depth + 1),
                        getSignExtendExpr(SA->getOperand(1), Ty, Depth + 1),
                        SCEV::FlagNSW, Depth + 1);
    }
 
    // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
    // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
    // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
    //
    // Often address arithmetics contain expressions like
    // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
    // This transformation is useful while proving that such expressions are
    // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
    if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
      const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
      if (D != 0) {
        const SCEV *SZExtD = getZeroExtendExpr(getConstant(D), Ty, Depth);
        const SCEV *SResidual =
            getAddExpr(getConstant(-D), SA, SCEV::FlagAnyWrap, Depth);
        const SCEV *SZExtR = getZeroExtendExpr(SResidual, Ty, Depth + 1);
        return getAddExpr(SZExtD, SZExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
                          Depth + 1);
      }
    }
  }
 
  if (auto *SM = dyn_cast<SCEVMulExpr>(Op)) {
    // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
    if (SM->hasNoUnsignedWrap()) {
      // If the multiply does not unsign overflow then we can, by definition,
      // commute the zero extension with the multiply operation.
      SmallVector<SCEVUse, 4> Ops;
      for (SCEVUse Op : SM->operands())
        Ops.push_back(getZeroExtendExpr(Op, Ty, Depth + 1));
      return getMulExpr(Ops, SCEV::FlagNUW, Depth + 1);
    }
 
    // zext(2^K * (trunc X to iN)) to iM ->
    // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
    //
    // Proof:
    //
    //     zext(2^K * (trunc X to iN)) to iM
    //   = zext((trunc X to iN) << K) to iM
    //   = zext((trunc X to i{N-K}) << K)<nuw> to iM
    //     (because shl removes the top K bits)
    //   = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
    //   = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
    //
    const APInt *C;
    const SCEV *TruncRHS;
    if (match(SM,
              m_scev_Mul(m_scev_APInt(C), m_scev_Trunc(m_SCEV(TruncRHS)))) &&
        C->isPowerOf2()) {
      int NewTruncBits =
          getTypeSizeInBits(SM->getOperand(1)->getType()) - C->logBase2();
      Type *NewTruncTy = IntegerType::get(getContext(), NewTruncBits);
      return getMulExpr(
          getZeroExtendExpr(SM->getOperand(0), Ty),
          getZeroExtendExpr(getTruncateExpr(TruncRHS, NewTruncTy), Ty),
          SCEV::FlagNUW, Depth + 1);
    }
  }
 
  // zext(umin(x, y)) -> umin(zext(x), zext(y))
  // zext(umax(x, y)) -> umax(zext(x), zext(y))
  if (isa<SCEVUMinExpr>(Op) || isa<SCEVUMaxExpr>(Op)) {
    auto *MinMax = cast<SCEVMinMaxExpr>(Op);
    SmallVector<SCEVUse, 4> Operands;
    for (SCEVUse Operand : MinMax->operands())
      Operands.push_back(getZeroExtendExpr(Operand, Ty));
    if (isa<SCEVUMinExpr>(MinMax))
      return getUMinExpr(Operands);
    return getUMaxExpr(Operands);
  }
 
  // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
  if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Op)) {
    assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
    SmallVector<SCEVUse, 4> Operands;
    for (SCEVUse Operand : MinMax->operands())
      Operands.push_back(getZeroExtendExpr(Operand, Ty));
    return getUMinExpr(Operands, /*Sequential*/ true);
  }
 
  // The cast wasn't folded; create an explicit cast node.
  // Recompute the insert position, as it may have been invalidated.
  if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
    return S;
  SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator),
                                                   Op, Ty);
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  registerUser(S, Op);
  return S;
}
 
const SCEV *ScalarEvolution::getSignExtendExpr(SCEVUse Op, Type *Ty,
                                               unsigned Depth) {
  assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
         "This is not an extending conversion!");
  assert(isSCEVable(Ty) &&
         "This is not a conversion to a SCEVable type!");
  assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
  Ty = getEffectiveSCEVType(Ty);
 
  FoldID ID(scSignExtend, Op, Ty);
  if (const SCEV *S = FoldCache.lookup(ID))
    return S;
 
  const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
  if (!isa<SCEVSignExtendExpr>(S))
    insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
  return S;
}
 
const SCEV *ScalarEvolution::getSignExtendExprImpl(SCEVUse Op, Type *Ty,
                                                   unsigned Depth) {
  assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
         "This is not an extending conversion!");
  assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
  assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
  Ty = getEffectiveSCEVType(Ty);
 
  // Fold if the operand is constant.
  if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
    return getConstant(SC->getAPInt().sext(getTypeSizeInBits(Ty)));
 
  // sext(sext(x)) --> sext(x)
  if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Op))
    return getSignExtendExpr(SS->getOperand(), Ty, Depth + 1);
 
  // sext(zext(x)) --> zext(x)
  if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Op))
    return getZeroExtendExpr(SZ->getOperand(), Ty, Depth + 1);
 
  // If the operand is an affine AddRec with the no-signed-wrap flag, the
  // sign-extension distributes over the recurrence.
  const SCEV *Start, *Step;
  const Loop *L;
  if (Depth <= MaxCastDepth &&
      match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
    const auto *AR = cast<SCEVAddRecExpr>(Op);
    if (AR->hasNoSignedWrap()) {
      Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
      Step = getSignExtendExpr(Step, Ty, Depth + 1);
      return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
    }
  }
 
  // Before doing any expensive analysis, check to see if we've already
  // computed a SCEV for this Op and Ty.
  FoldingSetNodeID ID;
  ID.AddInteger(scSignExtend);
  ID.AddPointer(Op.getOpaqueValue());
  ID.AddPointer(Ty);
  FoldingSetInsertToken Token;
  if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
    return S;
  // Limit recursion depth.
  if (Depth > MaxCastDepth) {
    SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
                                                     Op, Ty);
    UniqueSCEVs.insert(S, Token);
    S->computeAndSetCanonical(*this);
    registerUser(S, Op);
    return S;
  }
 
  // sext(trunc(x)) --> sext(x) or x or trunc(x)
  if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Op)) {
    // It's possible the bits taken off by the truncate were all sign bits. If
    // so, we should be able to simplify this further.
    const SCEV *X = ST->getOperand();
    ConstantRange CR = getSignedRange(X);
    unsigned TruncBits = getTypeSizeInBits(ST->getType());
    unsigned NewBits = getTypeSizeInBits(Ty);
    if (CR.truncate(TruncBits).signExtend(NewBits).contains(
            CR.sextOrTrunc(NewBits)))
      return getTruncateOrSignExtend(X, Ty, Depth);
  }
 
  if (auto *SA = dyn_cast<SCEVAddExpr>(Op)) {
    // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
    if (SA->hasNoSignedWrap()) {
      // If the addition does not sign overflow then we can, by definition,
      // commute the sign extension with the addition operation.
      SmallVector<SCEVUse, 4> Ops;
      for (SCEVUse Op : SA->operands())
        Ops.push_back(getSignExtendExpr(Op, Ty, Depth + 1));
      return getAddExpr(Ops, SCEV::FlagNSW, Depth + 1);
    }
 
    // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
    // if D + (C - D + x + y + ...) could be proven to not signed wrap
    // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
    //
    // For instance, this will bring two seemingly different expressions:
    //     1 + sext(5 + 20 * %x + 24 * %y)  and
    //         sext(6 + 20 * %x + 24 * %y)
    // to the same form:
    //     2 + sext(4 + 20 * %x + 24 * %y)
    if (const auto *SC = dyn_cast<SCEVConstant>(SA->getOperand(0))) {
      const APInt &D = extractConstantWithoutWrapping(*this, SC, SA);
      if (D != 0) {
        const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
        const SCEV *SResidual =
            getAddExpr(getConstant(-D), SA, SCEV::FlagAnyWrap, Depth);
        const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
        return getAddExpr(SSExtD, SSExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
                          Depth + 1);
      }
    }
  }
  // If the input value is a chrec scev, and we can prove that the value
  // did not overflow the old, smaller, value, we can sign extend all of the
  // operands (often constants).  This allows analysis of something like
  // this:  for (signed char X = 0; X < 100; ++X) { int Y = X; }
  if (match(Op, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step), m_Loop(L)))) {
    const auto *AR = cast<SCEVAddRecExpr>(Op);
    unsigned BitWidth = getTypeSizeInBits(AR->getType());
 
    // The no-signed-wrap case is handled before the uniquing lookup above.
 
    // Check whether the backedge-taken count is SCEVCouldNotCompute.
    // Note that this serves two purposes: It filters out loops that are
    // simply not analyzable, and it covers the case where this code is
    // being called from within backedge-taken count analysis, such that
    // attempting to ask for the backedge-taken count would likely result
    // in infinite recursion. In the later case, the analysis code will
    // cope with a conservative value, and it will take care to purge
    // that value once it has finished.
    const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
    if (!isa<SCEVCouldNotCompute>(MaxBECount)) {
      // Manually compute the final value for AR, checking for
      // overflow.
 
      // Check whether the backedge-taken count can be losslessly casted to
      // the addrec's type. The count is always unsigned.
      const SCEV *CastedMaxBECount =
          getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth);
      const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
          CastedMaxBECount, MaxBECount->getType(), Depth);
      if (MaxBECount == RecastedMaxBECount) {
        Type *WideTy = IntegerType::get(getContext(), BitWidth * 2);
        // Check whether Start+Step*MaxBECount has no signed overflow.
        const SCEV *SMul =
            getMulExpr(CastedMaxBECount, Step, SCEV::FlagAnyWrap, Depth + 1);
        const SCEV *SAdd = getSignExtendExpr(
            getAddExpr(Start, SMul, SCEV::FlagAnyWrap, Depth + 1), WideTy,
            Depth + 1);
        const SCEV *WideStart = getSignExtendExpr(Start, WideTy, Depth + 1);
        const SCEV *WideMaxBECount =
            getZeroExtendExpr(CastedMaxBECount, WideTy, Depth + 1);
        const SCEV *OperandExtendedAdd =
            getAddExpr(WideStart,
                       getMulExpr(WideMaxBECount,
                                  getSignExtendExpr(Step, WideTy, Depth + 1),
                                  SCEV::FlagAnyWrap, Depth + 1),
                       SCEV::FlagAnyWrap, Depth + 1);
        if (SAdd == OperandExtendedAdd) {
          // Cache knowledge of AR NSW, which is propagated to this AddRec.
          setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
          // Return the expression with the addrec on the outside.
          Start =
              getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
          Step = getSignExtendExpr(Step, Ty, Depth + 1);
          return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
        }
        // Similar to above, only this time treat the step value as unsigned.
        // This covers loops that count up with an unsigned step.
        OperandExtendedAdd =
            getAddExpr(WideStart,
                       getMulExpr(WideMaxBECount,
                                  getZeroExtendExpr(Step, WideTy, Depth + 1),
                                  SCEV::FlagAnyWrap, Depth + 1),
                       SCEV::FlagAnyWrap, Depth + 1);
        if (SAdd == OperandExtendedAdd) {
          // If AR wraps around then
          //
          //    abs(Step) * MaxBECount > unsigned-max(AR->getType())
          // => SAdd != OperandExtendedAdd
          //
          // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
          // (SAdd == OperandExtendedAdd => AR is NW)
 
          setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNW);
 
          // Return the expression with the addrec on the outside.
          Start =
              getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
          Step = getZeroExtendExpr(Step, Ty, Depth + 1);
          return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
        }
      }
    }
 
    auto NewFlags = proveNoSignedWrapViaInduction(AR);
    setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), NewFlags);
    if (AR->hasNoSignedWrap()) {
      // Same as nsw case above - duplicated here to avoid a compile time
      // issue.  It's not clear that the order of checks does matter, but
      // it's one of two issue possible causes for a change which was
      // reverted.  Be conservative for the moment.
      Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
      Step = getSignExtendExpr(Step, Ty, Depth + 1);
      return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
    }
 
    // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
    // if D + (C - D + Step * n) could be proven to not signed wrap
    // where D maximizes the number of trailing zeros of (C - D + Step * n)
    if (const auto *SC = dyn_cast<SCEVConstant>(Start)) {
      const APInt &C = SC->getAPInt();
      const APInt &D = extractConstantWithoutWrapping(*this, C, Step);
      if (D != 0) {
        const SCEV *SSExtD = getSignExtendExpr(getConstant(D), Ty, Depth);
        const SCEV *SResidual =
            getAddRecExpr(getConstant(C - D), Step, L, AR->getNoWrapFlags());
        const SCEV *SSExtR = getSignExtendExpr(SResidual, Ty, Depth + 1);
        return getAddExpr(SSExtD, SSExtR, (SCEV::FlagNSW | SCEV::FlagNUW),
                          Depth + 1);
      }
    }
 
    if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
      setNoWrapFlags(const_cast<SCEVAddRecExpr *>(AR), SCEV::FlagNSW);
      Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, this, Depth + 1);
      Step = getSignExtendExpr(Step, Ty, Depth + 1);
      return getAddRecExpr(Start, Step, L, AR->getNoWrapFlags());
    }
  }
 
  // If the input value is provably positive and we could not simplify
  // away the sext build a zext instead.
  if (isKnownNonNegative(Op))
    return getZeroExtendExpr(Op, Ty, Depth + 1);
 
  // sext(smin(x, y)) -> smin(sext(x), sext(y))
  // sext(smax(x, y)) -> smax(sext(x), sext(y))
  if (isa<SCEVSMinExpr>(Op) || isa<SCEVSMaxExpr>(Op)) {
    auto *MinMax = cast<SCEVMinMaxExpr>(Op);
    SmallVector<SCEVUse, 4> Operands;
    for (SCEVUse Operand : MinMax->operands())
      Operands.push_back(getSignExtendExpr(Operand, Ty));
    if (isa<SCEVSMinExpr>(MinMax))
      return getSMinExpr(Operands);
    return getSMaxExpr(Operands);
  }
 
  // The cast wasn't folded; create an explicit cast node.
  // Recompute the insert position, as it may have been invalidated.
  if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
    return S;
  SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator),
                                                   Op, Ty);
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  registerUser(S, Op);
  return S;
}
 
const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, SCEVUse Op, Type *Ty) {
  switch (Kind) {
  case scTruncate:
    return getTruncateExpr(Op, Ty);
  case scZeroExtend:
    return getZeroExtendExpr(Op, Ty);
  case scSignExtend:
    return getSignExtendExpr(Op, Ty);
  case scPtrToAddr: {
    const SCEV *Expr = getPtrToAddrExpr(Op);
    assert(Expr->getType() == Ty && "requested type must match");
    return Expr;
  }
  default:
    llvm_unreachable("Not a SCEV cast expression!");
  }
}
 
/// getAnyExtendExpr - Return a SCEV for the given operand extended with
/// unspecified bits out to the given type.
const SCEV *ScalarEvolution::getAnyExtendExpr(SCEVUse Op, Type *Ty) {
  assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
         "This is not an extending conversion!");
  assert(isSCEVable(Ty) &&
         "This is not a conversion to a SCEVable type!");
  Ty = getEffectiveSCEVType(Ty);
 
  // Sign-extend negative constants.
  if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Op))
    if (SC->getAPInt().isNegative())
      return getSignExtendExpr(Op, Ty);
 
  // Peel off a truncate cast.
  if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) {
    const SCEV *NewOp = T->getOperand();
    if (getTypeSizeInBits(NewOp->getType()) < getTypeSizeInBits(Ty))
      return getAnyExtendExpr(NewOp, Ty);
    return getTruncateOrNoop(NewOp, Ty);
  }
 
  // Next try a zext cast. If the cast is folded, use it.
  const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
  if (!isa<SCEVZeroExtendExpr>(ZExt))
    return ZExt;
 
  // Next try a sext cast. If the cast is folded, use it.
  const SCEV *SExt = getSignExtendExpr(Op, Ty);
  if (!isa<SCEVSignExtendExpr>(SExt))
    return SExt;
 
  // Force the cast to be folded into the operands of an addrec.
  if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Op)) {
    SmallVector<SCEVUse, 4> Ops;
    for (const SCEV *Op : AR->operands())
      Ops.push_back(getAnyExtendExpr(Op, Ty));
    return getAddRecExpr(Ops, AR->getLoop(), SCEV::FlagNW);
  }
 
  // If the expression is obviously signed, use the sext cast value.
  if (isa<SCEVSMaxExpr>(Op))
    return SExt;
 
  // Absent any other information, use the zext cast value.
  return ZExt;
}
 
/// Process the given Ops list, which is a list of operands to be added under
/// the given scale, update the given map. This is a helper function for
/// getAddRecExpr. As an example of what it does, given a sequence of operands
/// that would form an add expression like this:
///
///    m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
///
/// where A and B are constants, update the map with these values:
///
///    (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
///
/// and add 13 + A*B*29 to AccumulatedConstant.
/// This will allow getAddRecExpr to produce this:
///
///    13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
///
/// This form often exposes folding opportunities that are hidden in
/// the original operand list.
///
/// Return true iff it appears that any interesting folding opportunities
/// may be exposed. This helps getAddRecExpr short-circuit extra work in
/// the common case where no interesting opportunities are present, and
/// is also used as a check to avoid infinite recursion.
static bool CollectAddOperandsWithScales(SmallDenseMap<SCEVUse, APInt, 16> &M,
                                         SmallVectorImpl<SCEVUse> &NewOps,
                                         APInt &AccumulatedConstant,
                                         ArrayRef<SCEVUse> Ops,
                                         const APInt &Scale,
                                         ScalarEvolution &SE) {
  bool Interesting = false;
 
  // Iterate over the add operands. They are sorted, with constants first.
  unsigned i = 0;
  while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[i])) {
    ++i;
    // Pull a buried constant out to the outside.
    if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
      Interesting = true;
    AccumulatedConstant += Scale * C->getAPInt();
  }
 
  // Next comes everything else. We're especially interested in multiplies
  // here, but they're in the middle, so just visit the rest with one loop.
  for (; i != Ops.size(); ++i) {
    const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[i]);
    if (Mul && isa<SCEVConstant>(Mul->getOperand(0))) {
      APInt NewScale =
          Scale * cast<SCEVConstant>(Mul->getOperand(0))->getAPInt();
      if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Mul->getOperand(1))) {
        // A multiplication of a constant with another add; recurse.
        const SCEVAddExpr *Add = cast<SCEVAddExpr>(Mul->getOperand(1));
        Interesting |= CollectAddOperandsWithScales(
            M, NewOps, AccumulatedConstant, Add->operands(), NewScale, SE);
      } else {
        // A multiplication of a constant with some other value. Update
        // the map.
        SmallVector<SCEVUse, 4> MulOps(drop_begin(Mul->operands()));
        const SCEV *Key = SE.getMulExpr(MulOps);
        auto Pair = M.insert({Key, NewScale});
        if (Pair.second) {
          NewOps.push_back(Pair.first->first);
        } else {
          Pair.first->second += NewScale;
          // The map already had an entry for this value, which may indicate
          // a folding opportunity.
          Interesting = true;
        }
      }
    } else {
      // An ordinary operand. Update the map.
      auto Pair = M.insert({Ops[i], Scale});
      if (Pair.second) {
        NewOps.push_back(Pair.first->first);
      } else {
        Pair.first->second += Scale;
        // The map already had an entry for this value, which may indicate
        // a folding opportunity.
        Interesting = true;
      }
    }
  }
 
  return Interesting;
}
 
bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
                                      const SCEV *LHS, const SCEV *RHS,
                                      const Instruction *CtxI) {
  const SCEV *(ScalarEvolution::*Operation)(SCEVUse, SCEVUse, SCEV::NoWrapFlags,
                                            unsigned);
  switch (BinOp) {
  default:
    llvm_unreachable("Unsupported binary op");
  case Instruction::Add:
    Operation = &ScalarEvolution::getAddExpr;
    break;
  case Instruction::Sub:
    Operation = &ScalarEvolution::getMinusSCEV;
    break;
  case Instruction::Mul:
    Operation = &ScalarEvolution::getMulExpr;
    break;
  }
 
  const SCEV *(ScalarEvolution::*Extension)(SCEVUse, Type *, unsigned) =
      Signed ? &ScalarEvolution::getSignExtendExpr
             : &ScalarEvolution::getZeroExtendExpr;
 
  // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
  auto *NarrowTy = cast<IntegerType>(LHS->getType());
  auto *WideTy =
      IntegerType::get(NarrowTy->getContext(), NarrowTy->getBitWidth() * 2);
 
  const SCEV *A = (this->*Extension)(
      (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0);
  const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
  const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
  const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0);
  if (A == B)
    return true;
  // Can we use context to prove the fact we need?
  if (!CtxI)
    return false;
  // TODO: Support mul.
  if (BinOp == Instruction::Mul)
    return false;
  auto *RHSC = dyn_cast<SCEVConstant>(RHS);
  // TODO: Lift this limitation.
  if (!RHSC)
    return false;
  APInt C = RHSC->getAPInt();
  unsigned NumBits = C.getBitWidth();
  bool IsSub = (BinOp == Instruction::Sub);
  bool IsNegativeConst = (Signed && C.isNegative());
  // Compute the direction and magnitude by which we need to check overflow.
  bool OverflowDown = IsSub ^ IsNegativeConst;
  APInt Magnitude = C;
  if (IsNegativeConst) {
    if (C == APInt::getSignedMinValue(NumBits))
      // TODO: SINT_MIN on inversion gives the same negative value, we don't
      // want to deal with that.
      return false;
    Magnitude = -C;
  }
 
  ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
  if (OverflowDown) {
    // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
    APInt Min = Signed ? APInt::getSignedMinValue(NumBits)
                       : APInt::getMinValue(NumBits);
    APInt Limit = Min + Magnitude;
    return isKnownPredicateAt(Pred, getConstant(Limit), LHS, CtxI);
  } else {
    // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
    APInt Max = Signed ? APInt::getSignedMaxValue(NumBits)
                       : APInt::getMaxValue(NumBits);
    APInt Limit = Max - Magnitude;
    return isKnownPredicateAt(Pred, LHS, getConstant(Limit), CtxI);
  }
}
 
std::optional<SCEV::NoWrapFlags>
ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
    const OverflowingBinaryOperator *OBO) {
  // It cannot be done any better.
  if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
    return std::nullopt;
 
  SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap;
 
  if (OBO->hasNoUnsignedWrap())
    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
  if (OBO->hasNoSignedWrap())
    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
 
  bool Deduced = false;
 
  Instruction::BinaryOps Opcode = (Instruction::BinaryOps)OBO->getOpcode();
  const SCEV *LHS = getSCEV(OBO->getOperand(0));
  const SCEV *RHS = getSCEV(OBO->getOperand(1));
 
  bool CanUseNSW = true;
  const APInt *ShiftAmt;
  // Treat `shl %a, C` as `mul %a, 1 << C`.
  if (match(OBO, m_Shl(m_Value(), m_APInt(ShiftAmt)))) {
    unsigned BitWidth = ShiftAmt->getBitWidth();
    if (ShiftAmt->uge(BitWidth))
      return std::nullopt;
    // NSW only transfers if the shift amount is < BitWidth - 1, as INT_MIN * -1
    // overflows.
    CanUseNSW = ShiftAmt->ult(BitWidth - 1);
    Opcode = Instruction::Mul;
    RHS = getConstant(APInt::getOneBitSet(BitWidth, ShiftAmt->getZExtValue()));
  } else if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
             Opcode != Instruction::Mul) {
    return std::nullopt;
  }
 
  const Instruction *CtxI =
      UseContextForNoWrapFlagInference ? dyn_cast<Instruction>(OBO) : nullptr;
  if (!OBO->hasNoUnsignedWrap() &&
      willNotOverflow(Opcode, /* Signed */ false, LHS, RHS, CtxI)) {
    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
    Deduced = true;
  }
 
  if (CanUseNSW && !OBO->hasNoSignedWrap() &&
      willNotOverflow(Opcode, /* Signed */ true, LHS, RHS, CtxI)) {
    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
    Deduced = true;
  }
 
  if (Deduced)
    return Flags;
  return std::nullopt;
}
 
// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
// `OldFlags' as can't-wrap behavior.  Infer a more aggressive set of
// can't-overflow flags for the operation if possible.
static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
                                               SCEVTypes Type,
                                               ArrayRef<SCEVUse> Ops,
                                               SCEV::NoWrapFlags Flags) {
  using namespace std::placeholders;
 
  using OBO = OverflowingBinaryOperator;
 
  bool CanAnalyze =
      Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
  (void)CanAnalyze;
  assert(CanAnalyze && "don't call from other places!");
 
  SCEV::NoWrapFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
  SCEV::NoWrapFlags SignOrUnsignWrap =
      ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
 
  // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
  auto IsKnownNonNegative = [&](SCEVUse U) {
    return SE->isKnownNonNegative(U);
  };
 
  if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Ops, IsKnownNonNegative))
    Flags = ScalarEvolution::setFlags(Flags, SignOrUnsignMask);
 
  SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, SignOrUnsignMask);
 
  if (SignOrUnsignWrap != SignOrUnsignMask &&
      (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
      isa<SCEVConstant>(Ops[0])) {
 
    auto Opcode = [&] {
      switch (Type) {
      case scAddExpr:
        return Instruction::Add;
      case scMulExpr:
        return Instruction::Mul;
      default:
        llvm_unreachable("Unexpected SCEV op.");
      }
    }();
 
    const APInt &C = cast<SCEVConstant>(Ops[0])->getAPInt();
 
    // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
    if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
      auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
          Opcode, C, OBO::NoSignedWrap);
      if (NSWRegion.contains(SE->getSignedRange(Ops[1])))
        Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
    }
 
    // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
    if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
      auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
          Opcode, C, OBO::NoUnsignedWrap);
      if (NUWRegion.contains(SE->getUnsignedRange(Ops[1])))
        Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
    }
  }
 
  // <0,+,nonnegative><nw> is also nuw
  // TODO: Add corresponding nsw case
  if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, SCEV::FlagNW) &&
      !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) && Ops.size() == 2 &&
      Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
 
  // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
  if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, SCEV::FlagNUW) &&
      Ops.size() == 2) {
    if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[0]))
      if (UDiv->getOperand(1) == Ops[1])
        Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
    if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Ops[1]))
      if (UDiv->getOperand(1) == Ops[0])
        Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
  }
 
  return Flags;
}
 
bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
  return isLoopInvariant(S, L) && properlyDominates(S, L->getHeader());
}
 
/// Get a canonical add expression, or something simpler if possible.
const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
                                        SCEV::NoWrapFlags OrigFlags,
                                        unsigned Depth) {
  assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
         "only nuw or nsw allowed");
  assert(!Ops.empty() && "Cannot get empty add!");
  if (Ops.size() == 1) return Ops[0];
#ifndef NDEBUG
  Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
  for (unsigned i = 1, e = Ops.size(); i != e; ++i)
    assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
           "SCEVAddExpr operand types don't match!");
  unsigned NumPtrs = count_if(
      Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
  assert(NumPtrs <= 1 && "add has at most one pointer operand");
#endif
 
  const SCEV *Folded = constantFoldAndGroupOps(
      *this, LI, DT, Ops,
      [](const APInt &C1, const APInt &C2) { return C1 + C2; },
      [](const APInt &C) { return C.isZero(); }, // identity
      [](const APInt &C) { return false; });     // absorber
  if (Folded)
    return Folded;
 
  unsigned Idx = isa<SCEVConstant>(Ops[0]) ? 1 : 0;
 
  // Delay expensive flag strengthening until necessary.
  auto ComputeFlags = [this, OrigFlags](ArrayRef<SCEVUse> Ops) {
    return StrengthenNoWrapFlags(this, scAddExpr, Ops, OrigFlags);
  };
 
  // Limit recursion calls depth.
  if (Depth > MaxArithDepth || hasHugeExpression(Ops))
    return getOrCreateAddExpr(Ops, ComputeFlags(Ops));
 
  if (SCEV *S = findExistingSCEVInCache(scAddExpr, Ops)) {
    // Don't strengthen flags if we have no new information.
    SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
    if (Add->getNoWrapFlags(OrigFlags) != OrigFlags)
      Add->setNoWrapFlags(ComputeFlags(Ops));
    return S;
  }
 
  // Okay, check to see if the same value occurs in the operand list more than
  // once.  If so, merge them together into an multiply expression.  Since we
  // sorted the list, these values are required to be adjacent.
  Type *Ty = Ops[0]->getType();
  bool FoundMatch = false;
  for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
    if (Ops[i] == Ops[i+1]) {      //  X + Y + Y  -->  X + Y*2
      // Scan ahead to count how many equal operands there are.
      unsigned Count = 2;
      while (i+Count != e && Ops[i+Count] == Ops[i])
        ++Count;
      // Merge the values into a multiply.
      SCEVUse Scale = getConstant(Ty, Count);
      const SCEV *Mul = getMulExpr(Scale, Ops[i], SCEV::FlagAnyWrap, Depth + 1);
      if (Ops.size() == Count)
        return Mul;
      Ops[i] = Mul;
      Ops.erase(Ops.begin()+i+1, Ops.begin()+i+Count);
      --i; e -= Count - 1;
      FoundMatch = true;
    }
  if (FoundMatch)
    return getAddExpr(Ops, OrigFlags, Depth + 1);
 
  // Check for truncates. If all the operands are truncated from the same
  // type, see if factoring out the truncate would permit the result to be
  // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
  // if the contents of the resulting outer trunc fold to something simple.
  auto FindTruncSrcType = [&]() -> Type * {
    // We're ultimately looking to fold an addrec of truncs and muls of only
    // constants and truncs, so if we find any other types of SCEV
    // as operands of the addrec then we bail and return nullptr here.
    // Otherwise, we return the type of the operand of a trunc that we find.
    if (auto *T = dyn_cast<SCEVTruncateExpr>(Ops[Idx]))
      return T->getOperand()->getType();
    if (const auto *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
      SCEVUse LastOp = Mul->getOperand(Mul->getNumOperands() - 1);
      if (const auto *T = dyn_cast<SCEVTruncateExpr>(LastOp))
        return T->getOperand()->getType();
    }
    return nullptr;
  };
  if (auto *SrcType = FindTruncSrcType()) {
    SmallVector<SCEVUse, 8> LargeOps;
    bool Ok = true;
    // Check all the operands to see if they can be represented in the
    // source type of the truncate.
    for (const SCEV *Op : Ops) {
      if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Op)) {
        if (T->getOperand()->getType() != SrcType) {
          Ok = false;
          break;
        }
        LargeOps.push_back(T->getOperand());
      } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Op)) {
        LargeOps.push_back(getAnyExtendExpr(C, SrcType));
      } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Op)) {
        SmallVector<SCEVUse, 8> LargeMulOps;
        for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
          if (const SCEVTruncateExpr *T =
                dyn_cast<SCEVTruncateExpr>(M->getOperand(j))) {
            if (T->getOperand()->getType() != SrcType) {
              Ok = false;
              break;
            }
            LargeMulOps.push_back(T->getOperand());
          } else if (const auto *C = dyn_cast<SCEVConstant>(M->getOperand(j))) {
            LargeMulOps.push_back(getAnyExtendExpr(C, SrcType));
          } else {
            Ok = false;
            break;
          }
        }
        if (Ok)
          LargeOps.push_back(getMulExpr(LargeMulOps, SCEV::FlagAnyWrap, Depth + 1));
      } else {
        Ok = false;
        break;
      }
    }
    if (Ok) {
      // Evaluate the expression in the larger type.
      const SCEV *Fold = getAddExpr(LargeOps, SCEV::FlagAnyWrap, Depth + 1);
      // If it folds to something simple, use it. Otherwise, don't.
      if (isa<SCEVConstant>(Fold) || isa<SCEVUnknown>(Fold))
        return getTruncateExpr(Fold, Ty);
    }
  }
 
  if (Ops.size() == 2) {
    // Check if we have an expression of the form ((X + C1) - C2), where C1 and
    // C2 can be folded in a way that allows retaining wrapping flags of (X +
    // C1).
    const SCEV *A = Ops[0];
    const SCEV *B = Ops[1];
    auto *AddExpr = dyn_cast<SCEVAddExpr>(B);
    auto *C = dyn_cast<SCEVConstant>(A);
    if (AddExpr && C && isa<SCEVConstant>(AddExpr->getOperand(0))) {
      auto C1 = cast<SCEVConstant>(AddExpr->getOperand(0))->getAPInt();
      auto C2 = C->getAPInt();
      SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
 
      APInt ConstAdd = C1 + C2;
      auto AddFlags = AddExpr->getNoWrapFlags();
      // Adding a smaller constant is NUW if the original AddExpr was NUW.
      if (ScalarEvolution::hasFlags(AddFlags, SCEV::FlagNUW) &&
          ConstAdd.ule(C1)) {
        PreservedFlags =
            ScalarEvolution::setFlags(PreservedFlags, SCEV::FlagNUW);
      }
 
      // Adding a constant with the same sign and small magnitude is NSW, if the
      // original AddExpr was NSW.
      if (ScalarEvolution::hasFlags(AddFlags, SCEV::FlagNSW) &&
          C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
          ConstAdd.abs().ule(C1.abs())) {
        PreservedFlags =
            ScalarEvolution::setFlags(PreservedFlags, SCEV::FlagNSW);
      }
 
      if (PreservedFlags != SCEV::FlagAnyWrap) {
        SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
        NewOps[0] = getConstant(ConstAdd);
        return getAddExpr(NewOps, PreservedFlags);
      }
    }
 
    // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
    // (B), if trunc (A) + -A + B  does not unsigned-wrap.
    const SCEVAddExpr *InnerAdd;
    if (match(B, m_scev_ZExt(m_scev_Add(InnerAdd)))) {
      const SCEV *NarrowA = getTruncateExpr(A, InnerAdd->getType());
      if (NarrowA == getNegativeSCEV(InnerAdd->getOperand(0)) &&
          getZeroExtendExpr(NarrowA, B->getType()) == A &&
          hasFlags(StrengthenNoWrapFlags(this, scAddExpr, {NarrowA, InnerAdd},
                                         SCEV::FlagAnyWrap),
                   SCEV::FlagNUW)) {
        return getZeroExtendExpr(getAddExpr(NarrowA, InnerAdd), B->getType());
      }
    }
  }
 
  // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
  const SCEV *Y;
  if (Ops.size() == 2 &&
      match(Ops[0],
            m_scev_Mul(m_scev_AllOnes(),
                       m_scev_URem(m_scev_Specific(Ops[1]), m_SCEV(Y), *this))))
    return getMulExpr(Y, getUDivExpr(Ops[1], Y));
 
  // Skip past any other cast SCEVs.
  while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
    ++Idx;
 
  // If there are add operands they would be next.
  if (Idx < Ops.size()) {
    bool DeletedAdd = false;
    // If the original flags and all inlined SCEVAddExprs are NUW, use the
    // common NUW flag for expression after inlining. Other flags cannot be
    // preserved, because they may depend on the original order of operations.
    SCEV::NoWrapFlags CommonFlags = maskFlags(OrigFlags, SCEV::FlagNUW);
    while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[Idx])) {
      if (Ops.size() > AddOpsInlineThreshold ||
          Add->getNumOperands() > AddOpsInlineThreshold)
        break;
      // If we have an add, expand the add operands onto the end of the operands
      // list.
      Ops.erase(Ops.begin()+Idx);
      append_range(Ops, Add->operands());
      DeletedAdd = true;
      CommonFlags = maskFlags(CommonFlags, Add->getNoWrapFlags());
    }
 
    // If we deleted at least one add, we added operands to the end of the list,
    // and they are not necessarily sorted.  Recurse to resort and resimplify
    // any operands we just acquired.
    if (DeletedAdd)
      return getAddExpr(Ops, CommonFlags, Depth + 1);
  }
 
  // Skip over the add expression until we get to a multiply.
  while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
    ++Idx;
 
  // Check to see if there are any folding opportunities present with
  // operands multiplied by constant values.
  if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
    uint64_t BitWidth = getTypeSizeInBits(Ty);
    SmallDenseMap<SCEVUse, APInt, 16> M;
    SmallVector<SCEVUse, 8> NewOps;
    APInt AccumulatedConstant(BitWidth, 0);
    if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
                                     Ops, APInt(BitWidth, 1), *this)) {
      struct APIntCompare {
        bool operator()(const APInt &LHS, const APInt &RHS) const {
          return LHS.ult(RHS);
        }
      };
 
      // Some interesting folding opportunity is present, so its worthwhile to
      // re-generate the operands list. Group the operands by constant scale,
      // to avoid multiplying by the same constant scale multiple times.
      std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
      for (const SCEV *NewOp : NewOps)
        MulOpLists[M.find(NewOp)->second].push_back(NewOp);
      // Re-generate the operands list.
      Ops.clear();
      if (AccumulatedConstant != 0)
        Ops.push_back(getConstant(AccumulatedConstant));
      for (auto &MulOp : MulOpLists) {
        if (MulOp.first == 1) {
          Ops.push_back(getAddExpr(MulOp.second, SCEV::FlagAnyWrap, Depth + 1));
        } else if (MulOp.first != 0) {
          Ops.push_back(getMulExpr(
              getConstant(MulOp.first),
              getAddExpr(MulOp.second, SCEV::FlagAnyWrap, Depth + 1),
              SCEV::FlagAnyWrap, Depth + 1));
        }
      }
      if (Ops.empty())
        return getZero(Ty);
      if (Ops.size() == 1)
        return Ops[0];
      return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
    }
  }
 
  // Given a SCEVMulExpr and an operand index, return the product of all
  // operands except the one at OpIdx.
  auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
    if (M->getNumOperands() == 2)
      return M->getOperand(OpIdx == 0);
    SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(OpIdx));
    append_range(Remaining, M->operands().drop_front(OpIdx + 1));
    return getMulExpr(Remaining, SCEV::FlagAnyWrap, Depth + 1);
  };
 
  // If we are adding something to a multiply expression, make sure the
  // something is not already an operand of the multiply.  If so, merge it into
  // the multiply.
  for (; Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx]); ++Idx) {
    const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Ops[Idx]);
    for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
      // Scan all terms to find every occurrence of common factor MulOpSCEV
      // and fold them in one shot:
      //   A1*X + A2*X + ... + An*X  -->  X * (A1 + A2 + ... + An)
      const SCEV *MulOpSCEV = Mul->getOperand(MulOp);
      if (isa<SCEVConstant>(MulOpSCEV))
        continue;
 
      // Cofactors: 1 for bare addends matching MulOpSCEV, or the
      // remaining product for multiply terms containing MulOpSCEV.
      SmallVector<SCEVUse, 4> Cofactors;
      SmallVector<unsigned, 4> DeadIndices;
      for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
        if (MulOpSCEV == Ops[AddOp]) {
          // W + X + (X * Y * Z)  -->  W + (X * ((Y*Z)+1))
          Cofactors.push_back(getOne(Ty));
          DeadIndices.push_back(AddOp);
          continue;
        }
 
        if (AddOp <= Idx || !isa<SCEVMulExpr>(Ops[AddOp]))
          continue;
 
        const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Ops[AddOp]);
        for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
             ++OMulOp) {
          if (OtherMul->getOperand(OMulOp) == MulOpSCEV) {
            // (A*B*C) + (A*D*E)  -->  A * (B*C + D*E)
            Cofactors.push_back(StripFactor(OtherMul, OMulOp));
            DeadIndices.push_back(AddOp);
            break;
          }
        }
      }
 
      // Fold all collected cofactors with the anchor multiply's cofactor:
      //   MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
      if (!Cofactors.empty()) {
        Cofactors.push_back(StripFactor(Mul, MulOp));
 
        SCEVUse InnerSum = getAddExpr(Cofactors, SCEV::FlagAnyWrap, Depth + 1);
        SCEVUse OuterMul =
            getMulExpr(MulOpSCEV, InnerSum, SCEV::FlagAnyWrap, Depth + 1);
 
        // DeadIndices does not include Idx (the anchor), hence +1.
        if (Ops.size() == DeadIndices.size() + 1)
          return OuterMul;
 
        // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
        // The -1 adjustment accounts for the shift from removing Idx;
        // reverse order means each erasure only shifts later positions,
        // which have already been processed.
        Ops.erase(Ops.begin() + Idx);
        for (unsigned Dead : reverse(DeadIndices))
          Ops.erase(Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
 
        Ops.push_back(OuterMul);
        return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
      }
    }
  }
 
  // If there are any add recurrences in the operands list, see if any other
  // added values are loop invariant.  If so, we can fold them into the
  // recurrence.
  while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
    ++Idx;
 
  // Scan over all recurrences, trying to fold loop invariants into them.
  for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
    // Scan all of the other operands to this add and add them to the vector if
    // they are loop invariant w.r.t. the recurrence.
    SmallVector<SCEVUse, 8> LIOps;
    const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
    const Loop *AddRecLoop = AddRec->getLoop();
    for (unsigned i = 0, e = Ops.size(); i != e; ++i)
      if (isAvailableAtLoopEntry(Ops[i], AddRecLoop)) {
        LIOps.push_back(Ops[i]);
        Ops.erase(Ops.begin()+i);
        --i; --e;
      }
 
    // If we found some loop invariants, fold them into the recurrence.
    if (!LIOps.empty()) {
      // Compute nowrap flags for the addition of the loop-invariant ops and
      // the addrec. Temporarily push it as an operand for that purpose. These
      // flags are valid in the scope of the addrec only.
      LIOps.push_back(AddRec);
      SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
      LIOps.pop_back();
 
      //  NLI + LI + {Start,+,Step}  -->  NLI + {LI+Start,+,Step}
      LIOps.push_back(AddRec->getStart());
 
      SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
 
      // It is not in general safe to propagate flags valid on an add within
      // the addrec scope to one outside it.  We must prove that the inner
      // scope is guaranteed to execute if the outer one does to be able to
      // safely propagate.  We know the program is undefined if poison is
      // produced on the inner scoped addrec.  We also know that *for this use*
      // the outer scoped add can't overflow (because of the flags we just
      // computed for the inner scoped add) without the program being undefined.
      // Proving that entry to the outer scope neccesitates entry to the inner
      // scope, thus proves the program undefined if the flags would be violated
      // in the outer scope.
      SCEV::NoWrapFlags AddFlags = Flags;
      if (AddFlags != SCEV::FlagAnyWrap) {
        auto *DefI = getDefiningScopeBound(LIOps);
        auto *ReachI = &*AddRecLoop->getHeader()->begin();
        if (!isGuaranteedToTransferExecutionTo(DefI, ReachI))
          AddFlags = SCEV::FlagAnyWrap;
      }
      AddRecOps[0] = getAddExpr(LIOps, AddFlags, Depth + 1);
 
      // Build the new addrec. Propagate the NUW and NSW flags if both the
      // outer add and the inner addrec are guaranteed to have no overflow.
      // Always propagate NW.
      Flags = AddRec->getNoWrapFlags(setFlags(Flags, SCEV::FlagNW));
      const SCEV *NewRec = getAddRecExpr(AddRecOps, AddRecLoop, Flags);
 
      // If all of the other operands were loop invariant, we are done.
      if (Ops.size() == 1) return NewRec;
 
      // Otherwise, add the folded AddRec by the non-invariant parts.
      for (unsigned i = 0;; ++i)
        if (Ops[i] == AddRec) {
          Ops[i] = NewRec;
          break;
        }
      return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
    }
 
    // Okay, if there weren't any loop invariants to be folded, check to see if
    // there are multiple AddRec's with the same loop induction variable being
    // added together.  If so, we can fold them.
    for (unsigned OtherIdx = Idx+1;
         OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
         ++OtherIdx) {
      // We expect the AddRecExpr's to be sorted in reverse dominance order,
      // so that the 1st found AddRecExpr is dominated by all others.
      assert(DT.dominates(
           cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
           AddRec->getLoop()->getHeader()) &&
        "AddRecExprs are not sorted in reverse dominance order?");
      if (AddRecLoop == cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()) {
        // Other + {A,+,B}<L> + {C,+,D}<L>  -->  Other + {A+C,+,B+D}<L>
        SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
        for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
             ++OtherIdx) {
          const auto *OtherAddRec = cast<SCEVAddRecExpr>(Ops[OtherIdx]);
          if (OtherAddRec->getLoop() == AddRecLoop) {
            for (unsigned i = 0, e = OtherAddRec->getNumOperands();
                 i != e; ++i) {
              if (i >= AddRecOps.size()) {
                append_range(AddRecOps, OtherAddRec->operands().drop_front(i));
                break;
              }
              AddRecOps[i] =
                  getAddExpr(AddRecOps[i], OtherAddRec->getOperand(i),
                             SCEV::FlagAnyWrap, Depth + 1);
            }
            Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
          }
        }
        // Step size has changed, so we cannot guarantee no self-wraparound.
        Ops[Idx] = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap);
        return getAddExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
      }
    }
 
    // Otherwise couldn't fold anything into this recurrence.  Move onto the
    // next one.
  }
 
  // Okay, it looks like we really DO need an add expr.  Check to see if we
  // already have one, otherwise create a new one.
  return getOrCreateAddExpr(Ops, ComputeFlags(Ops));
}
 
const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
                                                SCEV::NoWrapFlags Flags) {
  FoldingSetNodeID ID;
  ID.AddInteger(scAddExpr);
  for (SCEVUse Op : Ops)
    ID.AddPointer(Op.getOpaqueValue());
  FoldingSetInsertToken Token;
  SCEVAddExpr *S = static_cast<SCEVAddExpr *>(UniqueSCEVs.lookup(ID, Token));
  if (!S) {
    SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
    llvm::uninitialized_copy(Ops, O);
    S = new (SCEVAllocator)
        SCEVAddExpr(ID.Intern(SCEVAllocator), O, Ops.size());
    UniqueSCEVs.insert(S, Token);
    S->computeAndSetCanonical(*this);
    registerUser(S, Ops);
  }
  S->setNoWrapFlags(Flags);
  return S;
}
 
const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
                                                   const Loop *L,
                                                   SCEV::NoWrapFlags Flags) {
  FoldingSetNodeID ID;
  ID.AddInteger(scAddRecExpr);
  for (SCEVUse Op : Ops)
    ID.AddPointer(Op.getOpaqueValue());
  ID.AddPointer(L);
  FoldingSetInsertToken Token;
  SCEVAddRecExpr *S =
      static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
  if (!S) {
    SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
    llvm::uninitialized_copy(Ops, O);
    S = new (SCEVAllocator)
        SCEVAddRecExpr(ID.Intern(SCEVAllocator), O, Ops.size(), L);
    UniqueSCEVs.insert(S, Token);
    S->computeAndSetCanonical(*this);
    LoopUsers[L].push_back(S);
    registerUser(S, Ops);
  }
  setNoWrapFlags(S, Flags);
  return S;
}
 
const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
                                                SCEV::NoWrapFlags Flags) {
  FoldingSetNodeID ID;
  ID.AddInteger(scMulExpr);
  for (SCEVUse Op : Ops)
    ID.AddPointer(Op.getOpaqueValue());
  FoldingSetInsertToken Token;
  SCEVMulExpr *S = static_cast<SCEVMulExpr *>(UniqueSCEVs.lookup(ID, Token));
  if (!S) {
    SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
    llvm::uninitialized_copy(Ops, O);
    S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(SCEVAllocator),
                                        O, Ops.size());
    UniqueSCEVs.insert(S, Token);
    S->computeAndSetCanonical(*this);
    registerUser(S, Ops);
  }
  S->setNoWrapFlags(Flags);
  return S;
}
 
const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
  FoldingSetNodeID ID;
  ID.AddInteger(scUDivExpr);
  ID.AddPointer(LHS.getOpaqueValue());
  ID.AddPointer(RHS.getOpaqueValue());
  FoldingSetInsertToken Token;
  SCEV *S = UniqueSCEVs.lookup(ID, Token);
  if (!S) {
    S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(SCEVAllocator), LHS, RHS);
    UniqueSCEVs.insert(S, Token);
    S->computeAndSetCanonical(*this);
    registerUser(S, ArrayRef<SCEVUse>({LHS, RHS}));
  }
  return S;
}
 
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
  uint64_t k = i*j;
  if (j > 1 && k / j != i) Overflow = true;
  return k;
}
 
/// Compute the result of "n choose k", the binomial coefficient.  If an
/// intermediate computation overflows, Overflow will be set and the return will
/// be garbage. Overflow is not cleared on absence of overflow.
static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
  // We use the multiplicative formula:
  //     n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
  // At each iteration, we take the n-th term of the numeral and divide by the
  // (k-n)th term of the denominator.  This division will always produce an
  // integral result, and helps reduce the chance of overflow in the
  // intermediate computations. However, we can still overflow even when the
  // final result would fit.
 
  if (n == 0 || n == k) return 1;
  if (k > n) return 0;
 
  if (k > n/2)
    k = n-k;
 
  uint64_t r = 1;
  for (uint64_t i = 1; i <= k; ++i) {
    r = umul_ov(r, n-(i-1), Overflow);
    r /= i;
  }
  return r;
}
 
/// Determine if any of the operands in this SCEV are a constant or if
/// any of the add or multiply expressions in this SCEV contain a constant.
static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
  struct FindConstantInAddMulChain {
    bool FoundConstant = false;
 
    bool follow(const SCEV *S) {
      FoundConstant |= isa<SCEVConstant>(S);
      return isa<SCEVAddExpr>(S) || isa<SCEVMulExpr>(S);
    }
 
    bool isDone() const {
      return FoundConstant;
    }
  };
 
  FindConstantInAddMulChain F;
  SCEVTraversal<FindConstantInAddMulChain> ST(F);
  ST.visitAll(StartExpr);
  return F.FoundConstant;
}
 
/// Get a canonical multiply expression, or something simpler if possible.
const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
                                        SCEV::NoWrapFlags OrigFlags,
                                        unsigned Depth) {
  assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
         "only nuw or nsw allowed");
  assert(!Ops.empty() && "Cannot get empty mul!");
  if (Ops.size() == 1) return Ops[0];
#ifndef NDEBUG
  Type *ETy = Ops[0]->getType();
  assert(!ETy->isPointerTy());
  for (unsigned i = 1, e = Ops.size(); i != e; ++i)
    assert(Ops[i]->getType() == ETy &&
           "SCEVMulExpr operand types don't match!");
#endif
 
  const SCEV *Folded = constantFoldAndGroupOps(
      *this, LI, DT, Ops,
      [](const APInt &C1, const APInt &C2) { return C1 * C2; },
      [](const APInt &C) { return C.isOne(); },   // identity
      [](const APInt &C) { return C.isZero(); }); // absorber
  if (Folded)
    return Folded;
 
  // Delay expensive flag strengthening until necessary.
  auto ComputeFlags = [this, OrigFlags](const ArrayRef<SCEVUse> Ops) {
    return StrengthenNoWrapFlags(this, scMulExpr, Ops, OrigFlags);
  };
 
  // Limit recursion calls depth.
  if (Depth > MaxArithDepth || hasHugeExpression(Ops))
    return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
 
  if (SCEV *S = findExistingSCEVInCache(scMulExpr, Ops)) {
    // Don't strengthen flags if we have no new information.
    SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
    if (Mul->getNoWrapFlags(OrigFlags) != OrigFlags)
      Mul->setNoWrapFlags(ComputeFlags(Ops));
    return S;
  }
 
  if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Ops[0])) {
    if (Ops.size() == 2) {
      // C1*(C2+V) -> C1*C2 + C1*V
      // If any of Add's ops are Adds or Muls with a constant, apply this
      // transformation as well.
      //
      // TODO: There are some cases where this transformation is not
      // profitable; for example, Add = (C0 + X) * Y + Z.  Maybe the scope of
      // this transformation should be narrowed down.
      const SCEV *Op0, *Op1;
      if (match(Ops[1], m_scev_Add(m_SCEV(Op0), m_SCEV(Op1))) &&
          containsConstantInAddMulChain(Ops[1])) {
        const SCEV *LHS = getMulExpr(LHSC, Op0, SCEV::FlagAnyWrap, Depth + 1);
        const SCEV *RHS = getMulExpr(LHSC, Op1, SCEV::FlagAnyWrap, Depth + 1);
        return getAddExpr(LHS, RHS, SCEV::FlagAnyWrap, Depth + 1);
      }
 
      if (Ops[0]->isAllOnesValue()) {
        // If we have a mul by -1 of an add, try distributing the -1 among the
        // add operands.
        if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Ops[1])) {
          SmallVector<SCEVUse, 4> NewOps;
          bool AnyFolded = false;
          for (const SCEV *AddOp : Add->operands()) {
            const SCEV *Mul = getMulExpr(Ops[0], SCEVUse(AddOp),
                                         SCEV::FlagAnyWrap, Depth + 1);
            if (!isa<SCEVMulExpr>(Mul)) AnyFolded = true;
            NewOps.push_back(Mul);
          }
          if (AnyFolded)
            return getAddExpr(NewOps, SCEV::FlagAnyWrap, Depth + 1);
        } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Ops[1])) {
          // Negation preserves a recurrence's no self-wrap property.
          SmallVector<SCEVUse, 4> Operands;
          for (const SCEV *AddRecOp : AddRec->operands())
            Operands.push_back(getMulExpr(Ops[0], SCEVUse(AddRecOp),
                                          SCEV::FlagAnyWrap, Depth + 1));
          // Let M be the minimum representable signed value. AddRec with nsw
          // multiplied by -1 can have signed overflow if and only if it takes a
          // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
          // maximum signed value. In all other cases signed overflow is
          // impossible.
          auto FlagsMask = SCEV::FlagNW;
          if (AddRec->hasNoSignedWrap()) {
            auto MinInt =
                APInt::getSignedMinValue(getTypeSizeInBits(AddRec->getType()));
            if (getSignedRangeMin(AddRec) != MinInt)
              FlagsMask = setFlags(FlagsMask, SCEV::FlagNSW);
          }
          return getAddRecExpr(Operands, AddRec->getLoop(),
                               AddRec->getNoWrapFlags(FlagsMask));
        }
      }
 
      // Try to push the constant operand into a ZExt: C * zext (A + B) ->
      // zext (C*A + C*B) if trunc (C) * (A + B)  does not unsigned-wrap.
      const SCEVAddExpr *InnerAdd;
      if (match(Ops[1], m_scev_ZExt(m_scev_Add(InnerAdd)))) {
        const SCEV *NarrowC = getTruncateExpr(LHSC, InnerAdd->getType());
        if (isa<SCEVConstant>(InnerAdd->getOperand(0)) &&
            getZeroExtendExpr(NarrowC, Ops[1]->getType()) == LHSC &&
            hasFlags(StrengthenNoWrapFlags(this, scMulExpr, {NarrowC, InnerAdd},
                                           SCEV::FlagAnyWrap),
                     SCEV::FlagNUW)) {
          auto *Res = getMulExpr(NarrowC, InnerAdd, SCEV::FlagNUW, Depth + 1);
          return getZeroExtendExpr(Res, Ops[1]->getType(), Depth + 1);
        };
      }
 
      // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
      // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
      // of C1, fold to (D /u (C2 /u C1)).
      const SCEV *D;
      APInt C1V = LHSC->getAPInt();
      // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
      // as -1 * 1, as it won't enable additional folds.
      if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
        C1V = C1V.abs();
      const SCEVConstant *C2;
      if (C1V.isPowerOf2() &&
          match(Ops[1], m_scev_UDiv(m_SCEV(D), m_SCEVConstant(C2))) &&
          C2->getAPInt().isPowerOf2() &&
          C1V.logBase2() <= getMinTrailingZeros(D)) {
        const SCEV *NewMul = nullptr;
        if (C1V.uge(C2->getAPInt())) {
          NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D);
        } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(D)) {
          assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
          NewMul = getUDivExpr(D, getUDivExpr(C2, getConstant(C1V)));
        }
        if (NewMul)
          return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul);
      }
    }
  }
 
  // Skip over the add expression until we get to a multiply.
  unsigned Idx = 0;
  while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
    ++Idx;
 
  // If there are mul operands inline them all into this expression.
  if (Idx < Ops.size()) {
    bool DeletedMul = false;
    while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Ops[Idx])) {
      if (Ops.size() > MulOpsInlineThreshold)
        break;
      // If we have an mul, expand the mul operands onto the end of the
      // operands list.
      Ops.erase(Ops.begin()+Idx);
      append_range(Ops, Mul->operands());
      DeletedMul = true;
    }
 
    // If we deleted at least one mul, we added operands to the end of the
    // list, and they are not necessarily sorted.  Recurse to resort and
    // resimplify any operands we just acquired.
    if (DeletedMul)
      return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
  }
 
  // If there are any add recurrences in the operands list, see if any other
  // added values are loop invariant.  If so, we can fold them into the
  // recurrence.
  while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
    ++Idx;
 
  // Scan over all recurrences, trying to fold loop invariants into them.
  for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Ops[Idx]); ++Idx) {
    // Scan all of the other operands to this mul and add them to the vector
    // if they are loop invariant w.r.t. the recurrence.
    SmallVector<SCEVUse, 8> LIOps;
    const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Ops[Idx]);
    for (unsigned i = 0, e = Ops.size(); i != e; ++i)
      if (isAvailableAtLoopEntry(Ops[i], AddRec->getLoop())) {
        LIOps.push_back(Ops[i]);
        Ops.erase(Ops.begin()+i);
        --i; --e;
      }
 
    // If we found some loop invariants, fold them into the recurrence.
    if (!LIOps.empty()) {
      //  NLI * LI * {Start,+,Step}  -->  NLI * {LI*Start,+,LI*Step}
      SmallVector<SCEVUse, 4> NewOps;
      NewOps.reserve(AddRec->getNumOperands());
      const SCEV *Scale = getMulExpr(LIOps, SCEV::FlagAnyWrap, Depth + 1);
 
      // If both the mul and addrec are nuw, we can preserve nuw.
      // If both the mul and addrec are nsw, we can only preserve nsw if either
      // a) they are also nuw, or
      // b) all multiplications of addrec operands with scale are nsw.
      SCEV::NoWrapFlags Flags =
          AddRec->getNoWrapFlags(ComputeFlags({Scale, AddRec}));
 
      for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
        NewOps.push_back(getMulExpr(Scale, AddRec->getOperand(i),
                                    SCEV::FlagAnyWrap, Depth + 1));
 
        if (hasFlags(Flags, SCEV::FlagNSW) && !hasFlags(Flags, SCEV::FlagNUW)) {
          ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
              Instruction::Mul, getSignedRange(Scale),
              OverflowingBinaryOperator::NoSignedWrap);
          if (!NSWRegion.contains(getSignedRange(AddRec->getOperand(i))))
            Flags = clearFlags(Flags, SCEV::FlagNSW);
        }
      }
 
      const SCEV *NewRec = getAddRecExpr(NewOps, AddRec->getLoop(), Flags);
 
      // If all of the other operands were loop invariant, we are done.
      if (Ops.size() == 1) return NewRec;
 
      // Otherwise, multiply the folded AddRec by the non-invariant parts.
      for (unsigned i = 0;; ++i)
        if (Ops[i] == AddRec) {
          Ops[i] = NewRec;
          break;
        }
      return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
    }
 
    // Okay, if there weren't any loop invariants to be folded, check to see
    // if there are multiple AddRec's with the same loop induction variable
    // being multiplied together.  If so, we can fold them.
 
    // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
    // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
    //       choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
    //   ]]],+,...up to x=2n}.
    // Note that the arguments to choose() are always integers with values
    // known at compile time, never SCEV objects.
    //
    // The implementation avoids pointless extra computations when the two
    // addrec's are of different length (mathematically, it's equivalent to
    // an infinite stream of zeros on the right).
    bool OpsModified = false;
    for (unsigned OtherIdx = Idx+1;
         OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Ops[OtherIdx]);
         ++OtherIdx) {
      const SCEVAddRecExpr *OtherAddRec =
        dyn_cast<SCEVAddRecExpr>(Ops[OtherIdx]);
      if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
        continue;
 
      // Limit max number of arguments to avoid creation of unreasonably big
      // SCEVAddRecs with very complex operands.
      if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
          MaxAddRecSize || hasHugeExpression({AddRec, OtherAddRec}))
        continue;
 
      bool Overflow = false;
      Type *Ty = AddRec->getType();
      bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
      SmallVector<SCEVUse, 7> AddRecOps;
      for (int x = 0, xe = AddRec->getNumOperands() +
             OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
        SmallVector<SCEVUse, 7> SumOps;
        for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
          uint64_t Coeff1 = Choose(x, 2*x - y, Overflow);
          for (int z = std::max(y-x, y-(int)AddRec->getNumOperands()+1),
                 ze = std::min(x+1, (int)OtherAddRec->getNumOperands());
               z < ze && !Overflow; ++z) {
            uint64_t Coeff2 = Choose(2*x - y, x-z, Overflow);
            uint64_t Coeff;
            if (LargerThan64Bits)
              Coeff = umul_ov(Coeff1, Coeff2, Overflow);
            else
              Coeff = Coeff1*Coeff2;
            const SCEV *CoeffTerm = getConstant(Ty, Coeff);
            const SCEV *Term1 = AddRec->getOperand(y-z);
            const SCEV *Term2 = OtherAddRec->getOperand(z);
            SumOps.push_back(getMulExpr(CoeffTerm, Term1, Term2,
                                        SCEV::FlagAnyWrap, Depth + 1));
          }
        }
        if (SumOps.empty())
          SumOps.push_back(getZero(Ty));
        AddRecOps.push_back(getAddExpr(SumOps, SCEV::FlagAnyWrap, Depth + 1));
      }
      if (!Overflow) {
        const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(),
                                              SCEV::FlagAnyWrap);
        if (Ops.size() == 2) return NewAddRec;
        Ops[Idx] = NewAddRec;
        Ops.erase(Ops.begin() + OtherIdx); --OtherIdx;
        OpsModified = true;
        AddRec = dyn_cast<SCEVAddRecExpr>(NewAddRec);
        if (!AddRec)
          break;
      }
    }
    if (OpsModified)
      return getMulExpr(Ops, SCEV::FlagAnyWrap, Depth + 1);
 
    // Otherwise couldn't fold anything into this recurrence.  Move onto the
    // next one.
  }
 
  // Okay, it looks like we really DO need an mul expr.  Check to see if we
  // already have one, otherwise create a new one.
  return getOrCreateMulExpr(Ops, ComputeFlags(Ops));
}
 
/// Represents an unsigned remainder expression based on unsigned division.
const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
  assert(getEffectiveSCEVType(LHS->getType()) ==
         getEffectiveSCEVType(RHS->getType()) &&
         "SCEVURemExpr operand types don't match!");
 
  // Short-circuit easy cases
  if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
    // If constant is one, the result is trivial
    if (RHSC->getValue()->isOne())
      return getZero(LHS->getType()); // X urem 1 --> 0
 
    // If constant is a power of two, fold into a zext(trunc(LHS)).
    if (RHSC->getAPInt().isPowerOf2()) {
      Type *FullTy = LHS->getType();
      Type *TruncTy =
          IntegerType::get(getContext(), RHSC->getAPInt().logBase2());
      return getZeroExtendExpr(getTruncateExpr(LHS, TruncTy), FullTy);
    }
  }
 
  // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
  const SCEV *UDiv = getUDivExpr(LHS, RHS);
  const SCEV *Mult = getMulExpr(UDiv, RHS, SCEV::FlagNUW);
  return getMinusSCEV(LHS, Mult, SCEV::FlagNUW);
}
 
/// Get a canonical unsigned division expression, or something simpler if
/// possible.
const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
  assert(!LHS->getType()->isPointerTy() &&
         "SCEVUDivExpr operand can't be pointer!");
  assert(LHS->getType() == RHS->getType() &&
         "SCEVUDivExpr operand types don't match!");
 
  if (SCEV *S =
          findExistingSCEVInCache(scUDivExpr, ArrayRef<SCEVUse>({LHS, RHS})))
    return S;
 
  // 0 udiv Y == 0
  if (match(LHS, m_scev_Zero()))
    return LHS;
 
  if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(RHS)) {
    if (RHSC->getValue()->isOne())
      return LHS;                               // X udiv 1 --> x
    // If the denominator is zero, the result of the udiv is undefined. Don't
    // try to analyze it, because the resolution chosen here may differ from
    // the resolution chosen in other parts of the compiler.
    if (!RHSC->getValue()->isZero()) {
      // Determine if the division can be folded into the operands of
      // its operands.
      // TODO: Generalize this to non-constants by using known-bits information.
      Type *Ty = LHS->getType();
      unsigned LZ = RHSC->getAPInt().countl_zero();
      unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
      // For non-power-of-two values, effectively round the value up to the
      // nearest power of two.
      if (!RHSC->getAPInt().isPowerOf2())
        ++MaxShiftAmt;
      IntegerType *ExtTy =
        IntegerType::get(getContext(), getTypeSizeInBits(Ty) + MaxShiftAmt);
      if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(LHS))
        if (const SCEVConstant *Step =
            dyn_cast<SCEVConstant>(AR->getStepRecurrence(*this))) {
          // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
          const APInt &StepInt = Step->getAPInt();
          const APInt &DivInt = RHSC->getAPInt();
          if (!StepInt.urem(DivInt) &&
              getZeroExtendExpr(AR, ExtTy) ==
              getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
                            getZeroExtendExpr(Step, ExtTy),
                            AR->getLoop(), SCEV::FlagAnyWrap)) {
            SmallVector<SCEVUse, 4> Operands;
            for (const SCEV *Op : AR->operands())
              Operands.push_back(getUDivExpr(Op, RHS));
            return getAddRecExpr(Operands, AR->getLoop(), SCEV::FlagNW);
          }
          /// Get a canonical UDivExpr for a recurrence.
          /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
          const APInt *StartRem;
          if (!DivInt.urem(StepInt) && match(getURemExpr(AR->getStart(), Step),
                                             m_scev_APInt(StartRem))) {
            bool NoWrap =
                getZeroExtendExpr(AR, ExtTy) ==
                getAddRecExpr(getZeroExtendExpr(AR->getStart(), ExtTy),
                              getZeroExtendExpr(Step, ExtTy), AR->getLoop(),
                              SCEV::FlagAnyWrap);
 
            // With N <= C and both N, C as powers-of-2, the transformation
            // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
            // if wrapping occurs, as the division results remain equivalent for
            // all offsets in [[(X - X%N), X).
            bool CanFoldWithWrap = StepInt.ule(DivInt) && // N <= C
                                   StepInt.isPowerOf2() && DivInt.isPowerOf2();
            // Only fold if the subtraction can be folded in the start
            // expression.
            const SCEV *NewStart =
                getMinusSCEV(AR->getStart(), getConstant(*StartRem));
            if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
                !isa<SCEVAddExpr>(NewStart)) {
              const SCEV *NewLHS =
                  getAddRecExpr(NewStart, Step, AR->getLoop(),
                                NoWrap ? SCEV::FlagNW : SCEV::FlagAnyWrap);
              if (LHS != NewLHS)
                return getUDivExpr(NewLHS, RHS);
            }
          }
        }
      // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
      if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(LHS)) {
        if (M->hasNoUnsignedWrap()) {
          // Find an operand that's safely divisible.
          for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
            const SCEV *Op = M->getOperand(i);
            const SCEV *Div = getUDivExpr(Op, RHSC);
            if (!isa<SCEVUDivExpr>(Div) && getMulExpr(Div, RHSC) == Op) {
              SmallVector<SCEVUse, 4> Operands(M->operands());
              Operands[i] = Div;
              return getMulExpr(Operands);
            }
          }
 
          // Even if it's not divisible, try to remove a common factor.
          if (const auto *LHSC = dyn_cast<SCEVConstant>(M->getOperand(0))) {
            APInt Factor = APIntOps::GreatestCommonDivisor(LHSC->getAPInt(),
                                                           RHSC->getAPInt());
            if (!Factor.isIntN(1)) {
              SmallVector<SCEVUse, 2> NewOperands;
              NewOperands.push_back(getConstant(LHSC->getAPInt().udiv(Factor)));
              append_range(NewOperands, M->operands().drop_front());
              const SCEV *NewMul = getMulExpr(NewOperands);
              return getUDivExpr(NewMul,
                                 getConstant(RHSC->getAPInt().udiv(Factor)));
            }
          }
        }
      }
 
      // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
      if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(LHS)) {
        if (auto *DivisorConstant =
                dyn_cast<SCEVConstant>(OtherDiv->getRHS())) {
          bool Overflow = false;
          APInt NewRHS =
              DivisorConstant->getAPInt().umul_ov(RHSC->getAPInt(), Overflow);
          if (Overflow) {
            return getConstant(RHSC->getType(), 0, false);
          }
          return getUDivExpr(OtherDiv->getLHS(), getConstant(NewRHS));
        }
      }
 
      // (A+B)/C --> (A/C + B/C) if the add does not unsigned wrap and A/C and
      // B/C can be folded.
      if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(LHS)) {
        if (A->hasNoUnsignedWrap()) {
          SmallVector<SCEVUse, 4> Operands;
          for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
            const SCEV *Op = getUDivExpr(A->getOperand(i), RHS);
            if (isa<SCEVUDivExpr>(Op) ||
                getMulExpr(Op, RHS) != A->getOperand(i))
              break;
            Operands.push_back(Op);
          }
          if (Operands.size() == A->getNumOperands())
            return getAddExpr(Operands);
        }
      }
 
      // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
      // This is an idiom for rounding A up to the next multiple of N, where A
      // is aready known to be a multiple of M. In this case, instcombine can
      // see that some low bits of the added constant are unused, so can clear
      // them, but we want to canonicalise to set the low bits. This makes the
      // pattern easier to match, without needing to check for known bits in
      // A*M.
      const APInt &N = RHSC->getAPInt();
      const APInt *NMinusM, *M;
      const SCEV *A;
      if (match(LHS, m_scev_Add(m_scev_APInt(NMinusM),
                                m_scev_Mul(m_scev_APInt(M), m_SCEV(A))))) {
        if (N.isPowerOf2() && M->isPowerOf2() && M->ult(N) &&
            *NMinusM == N - *M) {
          return getUDivExpr(
              getAddExpr(getConstant(N - 1), getMulExpr(getConstant(*M), A)),
              RHS);
        }
      }
 
      // Fold if both operands are constant.
      if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(LHS))
        return getConstant(LHSC->getAPInt().udiv(RHSC->getAPInt()));
    }
  }
 
  // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
  const APInt *NegC, *C;
  if (match(LHS,
            m_scev_Add(m_scev_APInt(NegC),
                       m_scev_SMax(m_scev_APInt(C), m_scev_Specific(RHS)))) &&
      NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
    return getZero(LHS->getType());
 
  // (%a * %b)<nuw> / %b -> %a
  const auto *Mul = dyn_cast<SCEVMulExpr>(LHS);
  if (Mul && Mul->hasNoUnsignedWrap()) {
    for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
      if (Mul->getOperand(i) == RHS) {
        SmallVector<SCEVUse, 2> Operands;
        append_range(Operands, Mul->operands().take_front(i));
        append_range(Operands, Mul->operands().drop_front(i + 1));
        return getMulExpr(Operands);
      }
    }
  }
 
  // TODO: Generalize to handle any common factors.
  // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
  const SCEV *NewLHS, *NewRHS;
  if (match(LHS, m_scev_c_NUWMul(m_SCEV(NewLHS), m_SCEVVScale())) &&
      match(RHS, m_scev_c_NUWMul(m_SCEV(NewRHS), m_SCEVVScale())))
    return getUDivExpr(NewLHS, NewRHS);
 
  return getOrCreateUDivExpr(LHS, RHS);
}
 
/// Get a canonical unsigned division expression, or something simpler if
/// possible. There is no representation for an exact udiv in SCEV IR, but we
/// can attempt to optimize it prior to construction.
const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
  // Currently there is no exact specific logic.
 
  return getUDivExpr(LHS, RHS);
}
 
/// Get an add recurrence expression for the specified loop.  Simplify the
/// expression as much as possible.
const SCEV *ScalarEvolution::getAddRecExpr(SCEVUse Start, SCEVUse Step,
                                           const Loop *L,
                                           SCEV::NoWrapFlags Flags) {
  SmallVector<SCEVUse, 4> Operands;
  Operands.push_back(Start);
  if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Step))
    if (StepChrec->getLoop() == L) {
      append_range(Operands, StepChrec->operands());
      return getAddRecExpr(Operands, L, maskFlags(Flags, SCEV::FlagNW));
    }
 
  Operands.push_back(Step);
  return getAddRecExpr(Operands, L, Flags);
}
 
/// Get an add recurrence expression for the specified loop.  Simplify the
/// expression as much as possible.
const SCEV *ScalarEvolution::getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
                                           const Loop *L,
                                           SCEV::NoWrapFlags Flags) {
  if (Operands.size() == 1) return Operands[0];
#ifndef NDEBUG
  Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
  for (const SCEV *Op : llvm::drop_begin(Operands)) {
    assert(getEffectiveSCEVType(Op->getType()) == ETy &&
           "SCEVAddRecExpr operand types don't match!");
    assert(!Op->getType()->isPointerTy() && "Step must be integer");
  }
  for (const SCEV *Op : Operands)
    assert(isAvailableAtLoopEntry(Op, L) &&
           "SCEVAddRecExpr operand is not available at loop entry!");
#endif
 
  if (Operands.back()->isZero()) {
    Operands.pop_back();
    return getAddRecExpr(Operands, L, SCEV::FlagAnyWrap); // {X,+,0}  -->  X
  }
 
  // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
  // use that information to infer NUW and NSW flags. However, computing a
  // BE count requires calling getAddRecExpr, so we may not yet have a
  // meaningful BE count at this point (and if we don't, we'd be stuck
  // with a SCEVCouldNotCompute as the cached BE count).
 
  Flags = StrengthenNoWrapFlags(this, scAddRecExpr, Operands, Flags);
 
  // Canonicalize nested AddRecs in by nesting them in order of loop depth.
  if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Operands[0])) {
    const Loop *NestedLoop = NestedAR->getLoop();
    if (L->contains(NestedLoop)
            ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
            : (!NestedLoop->contains(L) &&
               DT.dominates(L->getHeader(), NestedLoop->getHeader()))) {
      SmallVector<SCEVUse, 4> NestedOperands(NestedAR->operands());
      Operands[0] = NestedAR->getStart();
      // AddRecs require their operands be loop-invariant with respect to their
      // loops. Don't perform this transformation if it would break this
      // requirement.
      bool AllInvariant = all_of(
          Operands, [&](const SCEV *Op) { return isLoopInvariant(Op, L); });
 
      if (AllInvariant) {
        // Create a recurrence for the outer loop with the same step size.
        //
        // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
        // inner recurrence has the same property.
        SCEV::NoWrapFlags OuterFlags =
          maskFlags(Flags, SCEV::FlagNW | NestedAR->getNoWrapFlags());
 
        NestedOperands[0] = getAddRecExpr(Operands, L, OuterFlags);
        AllInvariant = all_of(NestedOperands, [&](const SCEV *Op) {
          return isLoopInvariant(Op, NestedLoop);
        });
 
        if (AllInvariant) {
          // Ok, both add recurrences are valid after the transformation.
          //
          // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
          // the outer recurrence has the same property.
          SCEV::NoWrapFlags InnerFlags =
            maskFlags(NestedAR->getNoWrapFlags(), SCEV::FlagNW | Flags);
          return getAddRecExpr(NestedOperands, NestedLoop, InnerFlags);
        }
      }
      // Reset Operands to its original state.
      Operands[0] = NestedAR;
    }
  }
 
  // Okay, it looks like we really DO need an addrec expr.  Check to see if we
  // already have one, otherwise create a new one.
  return getOrCreateAddRecExpr(Operands, L, Flags);
}
 
const SCEV *ScalarEvolution::getGEPExpr(GEPOperator *GEP,
                                        ArrayRef<SCEVUse> IndexExprs) {
  const SCEV *BaseExpr = getSCEV(GEP->getPointerOperand());
  // getSCEV(Base)->getType() has the same address space as Base->getType()
  // because SCEV::getType() preserves the address space.
  GEPNoWrapFlags NW = GEP->getNoWrapFlags();
  if (NW != GEPNoWrapFlags::none()) {
    // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
    // but to do that, we have to ensure that said flag is valid in the entire
    // defined scope of the SCEV.
    // TODO: non-instructions have global scope.  We might be able to prove
    // some global scope cases
    auto *GEPI = dyn_cast<Instruction>(GEP);
    if (!GEPI || !isSCEVExprNeverPoison(GEPI))
      NW = GEPNoWrapFlags::none();
  }
 
  return getGEPExpr(BaseExpr, IndexExprs, GEP->getSourceElementType(), NW);
}
 
const SCEV *ScalarEvolution::getGEPExpr(SCEVUse BaseExpr,
                                        ArrayRef<SCEVUse> IndexExprs,
                                        Type *SrcElementTy, GEPNoWrapFlags NW) {
  SCEV::NoWrapFlags OffsetWrap = SCEV::FlagAnyWrap;
  if (NW.hasNoUnsignedSignedWrap())
    OffsetWrap = setFlags(OffsetWrap, SCEV::FlagNSW);
  if (NW.hasNoUnsignedWrap())
    OffsetWrap = setFlags(OffsetWrap, SCEV::FlagNUW);
 
  Type *CurTy = BaseExpr->getType();
  Type *IntIdxTy = getEffectiveSCEVType(BaseExpr->getType());
  bool FirstIter = true;
  SmallVector<SCEVUse, 4> Offsets;
  for (SCEVUse IndexExpr : IndexExprs) {
    // Compute the (potentially symbolic) offset in bytes for this index.
    if (StructType *STy = dyn_cast<StructType>(CurTy)) {
      // For a struct, add the member offset.
      ConstantInt *Index = cast<SCEVConstant>(IndexExpr)->getValue();
      unsigned FieldNo = Index->getZExtValue();
      const SCEV *FieldOffset = getOffsetOfExpr(IntIdxTy, STy, FieldNo);
      Offsets.push_back(FieldOffset);
 
      // Update CurTy to the type of the field at Index.
      CurTy = STy->getTypeAtIndex(Index);
    } else {
      // Update CurTy to its element type.
      if (FirstIter) {
        assert(isa<PointerType>(CurTy) &&
               "The first index of a GEP indexes a pointer");
        CurTy = SrcElementTy;
        FirstIter = false;
      } else {
        CurTy = GetElementPtrInst::getTypeAtIndex(CurTy, (uint64_t)0);
      }
      // For an array, add the element offset, explicitly scaled.
      const SCEV *ElementSize = getSizeOfExpr(IntIdxTy, CurTy);
      // Getelementptr indices are signed.
      IndexExpr = getTruncateOrSignExtend(IndexExpr, IntIdxTy);
 
      // Multiply the index by the element size to compute the element offset.
      const SCEV *LocalOffset = getMulExpr(IndexExpr, ElementSize, OffsetWrap);
      Offsets.push_back(LocalOffset);
    }
  }
 
  // Handle degenerate case of GEP without offsets.
  if (Offsets.empty())
    return BaseExpr;
 
  // Add the offsets together, assuming nsw if inbounds.
  const SCEV *Offset = getAddExpr(Offsets, OffsetWrap);
  // Add the base address and the offset. We cannot use the nsw flag, as the
  // base address is unsigned. However, if we know that the offset is
  // non-negative, we can use nuw.
  bool NUW = NW.hasNoUnsignedWrap() ||
             (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(Offset));
  SCEV::NoWrapFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagAnyWrap;
  auto *GEPExpr = getAddExpr(BaseExpr, Offset, BaseWrap);
  assert(BaseExpr->getType() == GEPExpr->getType() &&
         "GEP should not change type mid-flight.");
  return GEPExpr;
}
 
SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
                                               ArrayRef<SCEVUse> Ops) {
  FoldingSetNodeID ID;
  ID.AddInteger(SCEVType);
  for (SCEVUse Op : Ops)
    ID.AddPointer(Op.getOpaqueValue());
  FoldingSetInsertToken Token;
  return UniqueSCEVs.lookup(ID, Token);
}
 
const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
  SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
  return getSMaxExpr(Op, getNegativeSCEV(Op, Flags));
}
 
const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
                                           SmallVectorImpl<SCEVUse> &Ops) {
  assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
  assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
  if (Ops.size() == 1) return Ops[0];
#ifndef NDEBUG
  Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
  for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
    assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
           "Operand types don't match!");
    assert(Ops[0]->getType()->isPointerTy() ==
               Ops[i]->getType()->isPointerTy() &&
           "min/max should be consistently pointerish");
  }
#endif
 
  bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
  bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
 
  const SCEV *Folded = constantFoldAndGroupOps(
      *this, LI, DT, Ops,
      [&](const APInt &C1, const APInt &C2) {
        switch (Kind) {
        case scSMaxExpr:
          return APIntOps::smax(C1, C2);
        case scSMinExpr:
          return APIntOps::smin(C1, C2);
        case scUMaxExpr:
          return APIntOps::umax(C1, C2);
        case scUMinExpr:
          return APIntOps::umin(C1, C2);
        default:
          llvm_unreachable("Unknown SCEV min/max opcode");
        }
      },
      [&](const APInt &C) {
        // identity
        if (IsMax)
          return IsSigned ? C.isMinSignedValue() : C.isMinValue();
        else
          return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
      },
      [&](const APInt &C) {
        // absorber
        if (IsMax)
          return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
        else
          return IsSigned ? C.isMinSignedValue() : C.isMinValue();
      });
  if (Folded)
    return Folded;
 
  // Check if we have created the same expression before.
  if (const SCEV *S = findExistingSCEVInCache(Kind, Ops)) {
    return S;
  }
 
  // Find the first operation of the same kind
  unsigned Idx = 0;
  while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
    ++Idx;
 
  // Check to see if one of the operands is of the same kind. If so, expand its
  // operands onto our operand list, and recurse to simplify.
  if (Idx < Ops.size()) {
    bool DeletedAny = false;
    while (Ops[Idx]->getSCEVType() == Kind) {
      const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Ops[Idx]);
      Ops.erase(Ops.begin()+Idx);
      append_range(Ops, SMME->operands());
      DeletedAny = true;
    }
 
    if (DeletedAny)
      return getMinMaxExpr(Kind, Ops);
  }
 
  // Okay, check to see if the same value occurs in the operand list twice.  If
  // so, delete one.  Since we sorted the list, these values are required to
  // be adjacent.
  llvm::CmpInst::Predicate GEPred =
      IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
  llvm::CmpInst::Predicate LEPred =
      IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
  llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
  llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
  for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
    if (Ops[i] == Ops[i + 1] ||
        isKnownViaNonRecursiveReasoning(FirstPred, Ops[i], Ops[i + 1])) {
      //  X op Y op Y  -->  X op Y
      //  X op Y       -->  X, if we know X, Y are ordered appropriately
      Ops.erase(Ops.begin() + i + 1, Ops.begin() + i + 2);
      --i;
      --e;
    } else if (isKnownViaNonRecursiveReasoning(SecondPred, Ops[i],
                                               Ops[i + 1])) {
      //  X op Y       -->  Y, if we know X, Y are ordered appropriately
      Ops.erase(Ops.begin() + i, Ops.begin() + i + 1);
      --i;
      --e;
    }
  }
 
  if (Ops.size() == 1) return Ops[0];
 
  assert(!Ops.empty() && "Reduced smax down to nothing!");
 
  // Okay, it looks like we really DO need an expr.  Check to see if we
  // already have one, otherwise create a new one.
  FoldingSetNodeID ID;
  ID.AddInteger(Kind);
  for (SCEVUse Op : Ops)
    ID.AddPointer(Op.getOpaqueValue());
  FoldingSetInsertToken Token;
  const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
  if (ExistingSCEV)
    return ExistingSCEV;
  SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
  llvm::uninitialized_copy(Ops, O);
  SCEV *S = new (SCEVAllocator)
      SCEVMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
 
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  registerUser(S, Ops);
  return S;
}
 
namespace {
 
class SCEVSequentialMinMaxDeduplicatingVisitor final
    : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
                         std::optional<const SCEV *>> {
  using RetVal = std::optional<const SCEV *>;
 
  ScalarEvolution &SE;
  const SCEVTypes RootKind; // Must be a sequential min/max expression.
  const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
  SmallPtrSet<const SCEV *, 16> SeenOps;
 
  bool canRecurseInto(SCEVTypes Kind) const {
    // We can only recurse into the SCEV expression of the same effective type
    // as the type of our root SCEV expression.
    return RootKind == Kind || NonSequentialRootKind == Kind;
  };
 
  RetVal visit(const SCEV *S) {
    // Has the whole operand been seen already?
    if (!SeenOps.insert(S).second)
      return std::nullopt;
    if (isa<SCEVMinMaxExpr, SCEVSequentialMinMaxExpr>(S)) {
      SCEVTypes Kind = S->getSCEVType();
 
      if (!canRecurseInto(Kind))
        return S;
 
      auto *NAry = cast<SCEVNAryExpr>(S);
      SmallVector<SCEVUse> NewOps;
      bool Changed = visit(Kind, NAry->operands(), NewOps);
 
      if (!Changed)
        return S;
      if (NewOps.empty())
        return std::nullopt;
 
      return isa<SCEVSequentialMinMaxExpr>(S)
                 ? SE.getSequentialMinMaxExpr(Kind, NewOps)
                 : SE.getMinMaxExpr(Kind, NewOps);
    }
    return S;
  }
 
public:
  SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
                                           SCEVTypes RootKind)
      : SE(SE), RootKind(RootKind),
        NonSequentialRootKind(
            SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
                RootKind)) {}
 
  bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
                         SmallVectorImpl<SCEVUse> &NewOps) {
    bool Changed = false;
    SmallVector<SCEVUse> Ops;
    Ops.reserve(OrigOps.size());
 
    for (const SCEV *Op : OrigOps) {
      RetVal NewOp = visit(Op);
      if (NewOp != Op)
        Changed = true;
      if (NewOp)
        Ops.emplace_back(*NewOp);
    }
 
    if (Changed)
      NewOps = std::move(Ops);
    return Changed;
  }
};
 
} // namespace
 
static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
  switch (Kind) {
  case scConstant:
  case scVScale:
  case scTruncate:
  case scZeroExtend:
  case scSignExtend:
  case scPtrToAddr:
  case scAddExpr:
  case scMulExpr:
  case scUDivExpr:
  case scAddRecExpr:
  case scUMaxExpr:
  case scSMaxExpr:
  case scUMinExpr:
  case scSMinExpr:
  case scUnknown:
    // If any operand is poison, the whole expression is poison.
    return true;
  case scSequentialUMinExpr:
    // FIXME: if the *first* operand is poison, the whole expression is poison.
    return false; // Pessimistically, say that it does not propagate poison.
  case scCouldNotCompute:
    llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
  }
  llvm_unreachable("Unknown SCEV kind!");
}
 
namespace {
// The only way poison may be introduced in a SCEV expression is from a
// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
// not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not*
// introduce poison -- they encode guaranteed, non-speculated knowledge.
//
// Additionally, all SCEV nodes propagate poison from inputs to outputs,
// with the notable exception of umin_seq, where only poison from the first
// operand is (unconditionally) propagated.
struct SCEVPoisonCollector {
  bool LookThroughMaybePoisonBlocking;
  SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
  SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
      : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
 
  bool follow(const SCEV *S) {
    if (!LookThroughMaybePoisonBlocking &&
        !scevUnconditionallyPropagatesPoisonFromOperands(S->getSCEVType()))
      return false;
 
    if (auto *SU = dyn_cast<SCEVUnknown>(S)) {
      if (!isGuaranteedNotToBePoison(SU->getValue()))
        MaybePoison.insert(SU);
    }
    return true;
  }
  bool isDone() const { return false; }
};
} // namespace
 
/// Return true if V is poison given that AssumedPoison is already poison.
static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
  // First collect all SCEVs that might result in AssumedPoison to be poison.
  // We need to look through potentially poison-blocking operations here,
  // because we want to find all SCEVs that *might* result in poison, not only
  // those that are *required* to.
  SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
  visitAll(AssumedPoison, PC1);
 
  // AssumedPoison is never poison. As the assumption is false, the implication
  // is true. Don't bother walking the other SCEV in this case.
  if (PC1.MaybePoison.empty())
    return true;
 
  // Collect all SCEVs in S that, if poison, *will* result in S being poison
  // as well. We cannot look through potentially poison-blocking operations
  // here, as their arguments only *may* make the result poison.
  SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
  visitAll(S, PC2);
 
  // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
  // it will also make S poison by being part of PC2.MaybePoison.
  return llvm::set_is_subset(PC1.MaybePoison, PC2.MaybePoison);
}
 
void ScalarEvolution::getPoisonGeneratingValues(
    SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
  SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
  visitAll(S, PC);
  for (const SCEVUnknown *SU : PC.MaybePoison)
    Result.insert(SU->getValue());
}
 
bool ScalarEvolution::canReuseInstruction(
    const SCEV *S, Instruction *I,
    SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
  // If the instruction cannot be poison, it's always safe to reuse.
  if (programUndefinedIfPoison(I))
    return true;
 
  // Otherwise, it is possible that I is more poisonous that S. Collect the
  // poison-contributors of S, and then check whether I has any additional
  // poison-contributors. Poison that is contributed through poison-generating
  // flags is handled by dropping those flags instead.
  SmallPtrSet<const Value *, 8> PoisonVals;
  getPoisonGeneratingValues(PoisonVals, S);
 
  SmallVector<Value *> Worklist;
  SmallPtrSet<Value *, 8> Visited;
  Worklist.push_back(I);
  while (!Worklist.empty()) {
    Value *V = Worklist.pop_back_val();
    if (!Visited.insert(V).second)
      continue;
 
    // Avoid walking large instruction graphs.
    if (Visited.size() > 16)
      return false;
 
    // Either the value can't be poison, or the S would also be poison if it
    // is.
    if (PoisonVals.contains(V) || ::isGuaranteedNotToBePoison(V))
      continue;
 
    auto *I = dyn_cast<Instruction>(V);
    if (!I)
      return false;
 
    // Disjoint or instructions are interpreted as adds by SCEV. However, we
    // can't replace an arbitrary add with disjoint or, even if we drop the
    // flag. We would need to convert the or into an add.
    if (auto *PDI = dyn_cast<PossiblyDisjointInst>(I))
      if (PDI->isDisjoint())
        return false;
 
    // FIXME: Ignore vscale, even though it technically could be poison. Do this
    // because SCEV currently assumes it can't be poison. Remove this special
    // case once we proper model when vscale can be poison.
    if (auto *II = dyn_cast<IntrinsicInst>(I);
        II && II->getIntrinsicID() == Intrinsic::vscale)
      continue;
 
    if (canCreatePoison(cast<Operator>(I), /*ConsiderFlagsAndMetadata*/ false))
      return false;
 
    // If the instruction can't create poison, we can recurse to its operands.
    if (I->hasPoisonGeneratingAnnotations())
      DropPoisonGeneratingInsts.push_back(I);
 
    llvm::append_range(Worklist, I->operands());
  }
  return true;
}
 
const SCEV *
ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
                                         SmallVectorImpl<SCEVUse> &Ops) {
  assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
         "Not a SCEVSequentialMinMaxExpr!");
  assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
  if (Ops.size() == 1)
    return Ops[0];
#ifndef NDEBUG
  Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
  for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
    assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
           "Operand types don't match!");
    assert(Ops[0]->getType()->isPointerTy() ==
               Ops[i]->getType()->isPointerTy() &&
           "min/max should be consistently pointerish");
  }
#endif
 
  // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
  // so we can *NOT* do any kind of sorting of the expressions!
 
  // Check if we have created the same expression before.
  if (const SCEV *S = findExistingSCEVInCache(Kind, Ops))
    return S;
 
  // FIXME: there are *some* simplifications that we can do here.
 
  // Keep only the first instance of an operand.
  {
    SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
    bool Changed = Deduplicator.visit(Kind, Ops, Ops);
    if (Changed)
      return getSequentialMinMaxExpr(Kind, Ops);
  }
 
  // Check to see if one of the operands is of the same kind. If so, expand its
  // operands onto our operand list, and recurse to simplify.
  {
    unsigned Idx = 0;
    bool DeletedAny = false;
    while (Idx < Ops.size()) {
      if (Ops[Idx]->getSCEVType() != Kind) {
        ++Idx;
        continue;
      }
      const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Ops[Idx]);
      Ops.erase(Ops.begin() + Idx);
      Ops.insert(Ops.begin() + Idx, SMME->operands().begin(),
                 SMME->operands().end());
      DeletedAny = true;
    }
 
    if (DeletedAny)
      return getSequentialMinMaxExpr(Kind, Ops);
  }
 
  const SCEV *SaturationPoint;
  ICmpInst::Predicate Pred;
  switch (Kind) {
  case scSequentialUMinExpr:
    SaturationPoint = getZero(Ops[0]->getType());
    Pred = ICmpInst::ICMP_ULE;
    break;
  default:
    llvm_unreachable("Not a sequential min/max type.");
  }
 
  for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
    if (!isGuaranteedNotToCauseUB(Ops[i]))
      continue;
    // We can replace %x umin_seq %y with %x umin %y if either:
    //  * %y being poison implies %x is also poison.
    //  * %x cannot be the saturating value (e.g. zero for umin).
    if (::impliesPoison(Ops[i], Ops[i - 1]) ||
        isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_NE, Ops[i - 1],
                                        SaturationPoint)) {
      SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
      Ops[i - 1] = getMinMaxExpr(
          SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Kind),
          SeqOps);
      Ops.erase(Ops.begin() + i);
      return getSequentialMinMaxExpr(Kind, Ops);
    }
    // Fold %x umin_seq %y to %x if %x ule %y.
    // TODO: We might be able to prove the predicate for a later operand.
    if (isKnownViaNonRecursiveReasoning(Pred, Ops[i - 1], Ops[i])) {
      Ops.erase(Ops.begin() + i);
      return getSequentialMinMaxExpr(Kind, Ops);
    }
  }
 
  // Okay, it looks like we really DO need an expr.  Check to see if we
  // already have one, otherwise create a new one.
  FoldingSetNodeID ID;
  ID.AddInteger(Kind);
  for (SCEVUse Op : Ops)
    ID.AddPointer(Op.getOpaqueValue());
  FoldingSetInsertToken Token;
  const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
  if (ExistingSCEV)
    return ExistingSCEV;
 
  SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Ops.size());
  llvm::uninitialized_copy(Ops, O);
  SCEV *S = new (SCEVAllocator)
      SCEVSequentialMinMaxExpr(ID.Intern(SCEVAllocator), Kind, O, Ops.size());
 
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  registerUser(S, Ops);
  return S;
}
 
const SCEV *ScalarEvolution::getSMaxExpr(SCEVUse LHS, SCEVUse RHS) {
  SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
  return getMinMaxExpr(scSMaxExpr, Ops);
}
 
const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
  return getMinMaxExpr(scSMaxExpr, Ops);
}
 
const SCEV *ScalarEvolution::getUMaxExpr(SCEVUse LHS, SCEVUse RHS) {
  SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
  return getMinMaxExpr(scUMaxExpr, Ops);
}
 
const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
  return getMinMaxExpr(scUMaxExpr, Ops);
}
 
const SCEV *ScalarEvolution::getSMinExpr(SCEVUse LHS, SCEVUse RHS) {
  SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
  return getMinMaxExpr(scSMinExpr, Ops);
}
 
const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<SCEVUse> &Ops) {
  return getMinMaxExpr(scSMinExpr, Ops);
}
 
const SCEV *ScalarEvolution::getUMinExpr(SCEVUse LHS, SCEVUse RHS,
                                         bool Sequential) {
  SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
  return getUMinExpr(Ops, Sequential);
}
 
const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<SCEVUse> &Ops,
                                         bool Sequential) {
  return Sequential ? getSequentialMinMaxExpr(scSequentialUMinExpr, Ops)
                    : getMinMaxExpr(scUMinExpr, Ops);
}
 
const SCEV *
ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
  const SCEV *Res = getConstant(IntTy, Size.getKnownMinValue());
  if (Size.isScalable())
    Res = getMulExpr(Res, getVScale(IntTy));
  return Res;
}
 
const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
  return getSizeOfExpr(IntTy, getDataLayout().getTypeAllocSize(AllocTy));
}
 
const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
  return getSizeOfExpr(IntTy, getDataLayout().getTypeStoreSize(StoreTy));
}
 
const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
                                             StructType *STy,
                                             unsigned FieldNo) {
  // We can bypass creating a target-independent constant expression and then
  // folding it back into a ConstantInt. This is just a compile-time
  // optimization.
  const StructLayout *SL = getDataLayout().getStructLayout(STy);
  assert(!SL->getSizeInBits().isScalable() &&
         "Cannot get offset for structure containing scalable vector types");
  return getConstant(IntTy, SL->getElementOffset(FieldNo));
}
 
const SCEV *ScalarEvolution::getUnknown(Value *V) {
  // Don't attempt to do anything other than create a SCEVUnknown object
  // here.  createSCEV only calls getUnknown after checking for all other
  // interesting possibilities, and any other code that calls getUnknown
  // is doing so in order to hide a value from SCEV canonicalization.
 
  FoldingSetNodeID ID;
  ID.AddInteger(scUnknown);
  ID.AddPointer(V);
  FoldingSetInsertToken Token;
  if (SCEV *S = UniqueSCEVs.lookup(ID, Token)) {
    assert(cast<SCEVUnknown>(S)->getValue() == V &&
           "Stale SCEVUnknown in uniquing map!");
    return S;
  }
  SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(SCEVAllocator), V, this,
                                            FirstUnknown);
  FirstUnknown = cast<SCEVUnknown>(S);
  UniqueSCEVs.insert(S, Token);
  S->computeAndSetCanonical(*this);
  return S;
}
 
//===----------------------------------------------------------------------===//
//            Basic SCEV Analysis and PHI Idiom Recognition Code
//
 
/// Test if values of the given type are analyzable within the SCEV
/// framework. This primarily includes integer types, and it can optionally
/// include pointer types if the ScalarEvolution class has access to
/// target-specific information.
bool ScalarEvolution::isSCEVable(Type *Ty) const {
  // Integers and pointers are always SCEVable.
  return Ty->isIntOrPtrTy();
}
 
/// Return the size in bits of the specified type, for which isSCEVable must
/// return true.
uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
  assert(isSCEVable(Ty) && "Type is not SCEVable!");
  if (Ty->isPointerTy())
    return getDataLayout().getIndexTypeSizeInBits(Ty);
  return getDataLayout().getTypeSizeInBits(Ty);
}
 
/// Return a type with the same bitwidth as the given type and which represents
/// how SCEV will treat the given type, for which isSCEVable must return
/// true. For pointer types, this is the pointer index sized integer type.
Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
  assert(isSCEVable(Ty) && "Type is not SCEVable!");
 
  if (Ty->isIntegerTy())
    return Ty;
 
  // The only other support type is pointer.
  assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
  return getDataLayout().getIndexType(Ty);
}
 
Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
  return  getTypeSizeInBits(T1) >= getTypeSizeInBits(T2) ? T1 : T2;
}
 
bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
                                                        const SCEV *B) {
  /// For a valid use point to exist, the defining scope of one operand
  /// must dominate the other.
  bool PreciseA, PreciseB;
  auto *ScopeA = getDefiningScopeBound({A}, PreciseA);
  auto *ScopeB = getDefiningScopeBound({B}, PreciseB);
  if (!PreciseA || !PreciseB)
    // Can't tell.
    return false;
  return (ScopeA == ScopeB) || DT.dominates(ScopeA, ScopeB) ||
    DT.dominates(ScopeB, ScopeA);
}
 
const SCEV *ScalarEvolution::getCouldNotCompute() {
  return CouldNotCompute.get();
}
 
bool ScalarEvolution::checkValidity(const SCEV *S) const {
  bool ContainsNulls = SCEVExprContains(S, [](const SCEV *S) {
    auto *SU = dyn_cast<SCEVUnknown>(S);
    return SU && SU->getValue() == nullptr;
  });
 
  return !ContainsNulls;
}
 
bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
  HasRecMapType::iterator I = HasRecMap.find(S);
  if (I != HasRecMap.end())
    return I->second;
 
  bool FoundAddRec =
      SCEVExprContains(S, [](const SCEV *S) { return isa<SCEVAddRecExpr>(S); });
  HasRecMap.insert({S, FoundAddRec});
  return FoundAddRec;
}
 
/// Return the ValueOffsetPair set for \p S. \p S can be represented
/// by the value and offset from any ValueOffsetPair in the set.
ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
  ExprValueMapType::iterator SI = ExprValueMap.find_as(S);
  if (SI == ExprValueMap.end())
    return {};
  return SI->second.getArrayRef();
}
 
/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
/// cannot be used separately. eraseValueFromMap should be used to remove
/// V from ValueExprMap and ExprValueMap at the same time.
void ScalarEvolution::eraseValueFromMap(Value *V) {
  ValueExprMapType::iterator I = ValueExprMap.find_as(V);
  if (I != ValueExprMap.end()) {
    auto EVIt = ExprValueMap.find(I->second);
    bool Removed = EVIt->second.remove(V);
    (void) Removed;
    assert(Removed && "Value not in ExprValueMap?");
    ValueExprMap.erase(I);
  }
}
 
void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
  // A recursive query may have already computed the SCEV. It should be
  // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
  // inferred nowrap flags.
  auto It = ValueExprMap.find_as(V);
  if (It == ValueExprMap.end()) {
    ValueExprMap.insert({SCEVCallbackVH(V, this), S});
    ExprValueMap[S].insert(V);
  }
}
 
/// Return an existing SCEV if it exists, otherwise analyze the expression and
/// create a new one.
const SCEV *ScalarEvolution::getSCEV(Value *V) {
  assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
 
  if (const SCEV *S = getExistingSCEV(V))
    return S;
  return createSCEVIter(V);
}
 
const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
  assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
 
  ValueExprMapType::iterator I = ValueExprMap.find_as(V);
  if (I != ValueExprMap.end()) {
    const SCEV *S = I->second;
    assert(checkValidity(S) &&
           "existing SCEV has not been properly invalidated");
    return S;
  }
  return nullptr;
}
 
/// Return a SCEV corresponding to -V = -1*V
const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V,
                                             SCEV::NoWrapFlags Flags) {
  if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
    return getConstant(
               cast<ConstantInt>(ConstantExpr::getNeg(VC->getValue())));
 
  Type *Ty = V->getType();
  Ty = getEffectiveSCEVType(Ty);
  return getMulExpr(V, getMinusOne(Ty), Flags);
}
 
/// If Expr computes ~A, return A else return nullptr
static const SCEV *MatchNotExpr(const SCEV *Expr) {
  const SCEV *MulOp;
  if (match(Expr, m_scev_Add(m_scev_AllOnes(),
                             m_scev_Mul(m_scev_AllOnes(), m_SCEV(MulOp)))))
    return MulOp;
  return nullptr;
}
 
/// Return a SCEV corresponding to ~V = -1-V
const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
  assert(!V->getType()->isPointerTy() && "Can't negate pointer");
 
  if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(V))
    return getConstant(
                cast<ConstantInt>(ConstantExpr::getNot(VC->getValue())));
 
  // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
  if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(V)) {
    auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
      SmallVector<SCEVUse, 2> MatchedOperands;
      for (const SCEV *Operand : MME->operands()) {
        const SCEV *Matched = MatchNotExpr(Operand);
        if (!Matched)
          return (const SCEV *)nullptr;
        MatchedOperands.push_back(Matched);
      }
      return getMinMaxExpr(SCEVMinMaxExpr::negate(MME->getSCEVType()),
                           MatchedOperands);
    };
    if (const SCEV *Replaced = MatchMinMaxNegation(MME))
      return Replaced;
  }
 
  Type *Ty = V->getType();
  Ty = getEffectiveSCEVType(Ty);
  return getMinusSCEV(getMinusOne(Ty), V);
}
 
const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
  assert(P->getType()->isPointerTy());
 
  if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(P)) {
    // The base of an AddRec is the first operand.
    SmallVector<SCEVUse> Ops{AddRec->operands()};
    Ops[0] = removePointerBase(Ops[0]);
    // Don't try to transfer nowrap flags for now. We could in some cases
    // (for example, if pointer operand of the AddRec is a SCEVUnknown).
    return getAddRecExpr(Ops, AddRec->getLoop(), SCEV::FlagAnyWrap);
  }
  if (auto *Add = dyn_cast<SCEVAddExpr>(P)) {
    // The base of an Add is the pointer operand.
    SmallVector<SCEVUse> Ops{Add->operands()};
    SCEVUse *PtrOp = nullptr;
    for (SCEVUse &AddOp : Ops) {
      if (AddOp->getType()->isPointerTy()) {
        assert(!PtrOp && "Cannot have multiple pointer ops");
        PtrOp = &AddOp;
      }
    }
    *PtrOp = removePointerBase(*PtrOp);
    // Don't try to transfer nowrap flags for now. We could in some cases
    // (for example, if the pointer operand of the Add is a SCEVUnknown).
    return getAddExpr(Ops);
  }
  // Any other expression must be a pointer base.
  return getZero(P->getType());
}
 
const SCEV *ScalarEvolution::getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
                                          SCEV::NoWrapFlags Flags,
                                          unsigned Depth) {
  // Fast path: X - X --> 0.
  if (LHS == RHS)
    return getZero(LHS->getType());
 
  // If we subtract two pointers with different pointer bases, bail.
  // Eventually, we're going to add an assertion to getMulExpr that we
  // can't multiply by a pointer.
  if (RHS->getType()->isPointerTy()) {
    if (!LHS->getType()->isPointerTy() ||
        getPointerBase(LHS) != getPointerBase(RHS))
      return getCouldNotCompute();
    LHS = removePointerBase(LHS);
    RHS = removePointerBase(RHS);
  }
 
  // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
  // makes it so that we cannot make much use of NUW.
  auto AddFlags = SCEV::FlagAnyWrap;
  const bool RHSIsNotMinSigned =
      !getSignedRangeMin(RHS).isMinSignedValue();
  if (hasFlags(Flags, SCEV::FlagNSW)) {
    // Let M be the minimum representable signed value. Then (-1)*RHS
    // signed-wraps if and only if RHS is M. That can happen even for
    // a NSW subtraction because e.g. (-1)*M signed-wraps even though
    // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
    // (-1)*RHS, we need to prove that RHS != M.
    //
    // If LHS is non-negative and we know that LHS - RHS does not
    // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
    // either by proving that RHS > M or that LHS >= 0.
    if (RHSIsNotMinSigned || isKnownNonNegative(LHS)) {
      AddFlags = SCEV::FlagNSW;
    }
  }
 
  // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
  // RHS is NSW and LHS >= 0.
  //
  // The difficulty here is that the NSW flag may have been proven
  // relative to a loop that is to be found in a recurrence in LHS and
  // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
  // larger scope than intended.
  auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
 
  return getAddExpr(LHS, getNegativeSCEV(RHS, NegFlags), AddFlags, Depth);
}
 
const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
                                                     unsigned Depth) {
  Type *SrcTy = V->getType();
  assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
         "Cannot truncate or zero extend with non-integer arguments!");
  if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
    return V;  // No conversion
  if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
    return getTruncateExpr(V, Ty, Depth);
  return getZeroExtendExpr(V, Ty, Depth);
}
 
const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
                                                     unsigned Depth) {
  Type *SrcTy = V->getType();
  assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
         "Cannot truncate or zero extend with non-integer arguments!");
  if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
    return V;  // No conversion
  if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty))
    return getTruncateExpr(V, Ty, Depth);
  return getSignExtendExpr(V, Ty, Depth);
}
 
const SCEV *ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
  Type *SrcTy = V->getType();
  assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
         "Cannot noop or zero extend with non-integer arguments!");
  assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
         "getNoopOrZeroExtend cannot truncate!");
  if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
    return V;  // No conversion
  return getZeroExtendExpr(V, Ty);
}
 
const SCEV *ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
  Type *SrcTy = V->getType();
  assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
         "Cannot noop or sign extend with non-integer arguments!");
  assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
         "getNoopOrSignExtend cannot truncate!");
  if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
    return V;  // No conversion
  return getSignExtendExpr(V, Ty);
}
 
const SCEV *ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
  Type *SrcTy = V->getType();
  assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
         "Cannot noop or any extend with non-integer arguments!");
  assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
         "getNoopOrAnyExtend cannot truncate!");
  if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
    return V;  // No conversion
  return getAnyExtendExpr(V, Ty);
}
 
const SCEV *ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
  Type *SrcTy = V->getType();
  assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
         "Cannot truncate or noop with non-integer arguments!");
  assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
         "getTruncateOrNoop cannot extend!");
  if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty))
    return V;  // No conversion
  return getTruncateExpr(V, Ty);
}
 
const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
                                                        const SCEV *RHS) {
  const SCEV *PromotedLHS = LHS;
  const SCEV *PromotedRHS = RHS;
 
  if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(RHS->getType()))
    PromotedRHS = getZeroExtendExpr(RHS, LHS->getType());
  else
    PromotedLHS = getNoopOrZeroExtend(LHS, RHS->getType());
 
  return getUMaxExpr(PromotedLHS, PromotedRHS);
}
 
const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
                                                        const SCEV *RHS,
                                                        bool Sequential) {
  SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
  return getUMinFromMismatchedTypes(Ops, Sequential);
}
 
const SCEV *
ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
                                            bool Sequential) {
  assert(!Ops.empty() && "At least one operand must be!");
  // Trivial case.
  if (Ops.size() == 1)
    return Ops[0];
 
  // Find the max type first.
  Type *MaxType = nullptr;
  for (SCEVUse S : Ops)
    if (MaxType)
      MaxType = getWiderType(MaxType, S->getType());
    else
      MaxType = S->getType();
  assert(MaxType && "Failed to find maximum type!");
 
  // Extend all ops to max type.
  SmallVector<SCEVUse, 2> PromotedOps;
  for (SCEVUse S : Ops)
    PromotedOps.push_back(getNoopOrZeroExtend(S, MaxType));
 
  // Generate umin.
  return getUMinExpr(PromotedOps, Sequential);
}
 
const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
  // A pointer operand may evaluate to a nonpointer expression, such as null.
  if (!V->getType()->isPointerTy())
    return V;
 
  while (true) {
    if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(V)) {
      V = AddRec->getStart();
    } else if (auto *Add = dyn_cast<SCEVAddExpr>(V)) {
      const SCEV *PtrOp = nullptr;
      for (const SCEV *AddOp : Add->operands()) {
        if (AddOp->getType()->isPointerTy()) {
          assert(!PtrOp && "Cannot have multiple pointer ops");
          PtrOp = AddOp;
        }
      }
      assert(PtrOp && "Must have pointer op");
      V = PtrOp;
    } else // Not something we can look further into.
      return V;
  }
}
 
/// Push users of the given Instruction onto the given Worklist.
static void PushDefUseChildren(Instruction *I,
                               SmallVectorImpl<Instruction *> &Worklist,
                               SmallPtrSetImpl<Instruction *> &Visited) {
  // Push the def-use children onto the Worklist stack.
  for (User *U : I->users()) {
    auto *UserInsn = cast<Instruction>(U);
    if (Visited.insert(UserInsn).second)
      Worklist.push_back(UserInsn);
  }
}
 
namespace {
 
/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
/// expression in case its Loop is L. If it is not L then
/// if IgnoreOtherLoops is true then use AddRec itself
/// otherwise rewrite cannot be done.
/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
public:
  static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
                             bool IgnoreOtherLoops = true) {
    SCEVInitRewriter Rewriter(L, SE);
    const SCEV *Result = Rewriter.visit(S);
    if (Rewriter.hasSeenLoopVariantSCEVUnknown())
      return SE.getCouldNotCompute();
    return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
               ? SE.getCouldNotCompute()
               : Result;
  }
 
  const SCEV *visitUnknown(const SCEVUnknown *Expr) {
    if (!SE.isLoopInvariant(Expr, L))
      SeenLoopVariantSCEVUnknown = true;
    return Expr;
  }
 
  const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
    // Only re-write AddRecExprs for this loop.
    if (Expr->getLoop() == L)
      return Expr->getStart();
    SeenOtherLoops = true;
    return Expr;
  }
 
  bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
 
  bool hasSeenOtherLoops() { return SeenOtherLoops; }
 
private:
  explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
      : SCEVRewriteVisitor(SE), L(L) {}
 
  const Loop *L;
  bool SeenLoopVariantSCEVUnknown = false;
  bool SeenOtherLoops = false;
};
 
/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
/// increment expression in case its Loop is L. If it is not L then
/// use AddRec itself.
/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
public:
  static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
    SCEVPostIncRewriter Rewriter(L, SE);
    const SCEV *Result = Rewriter.visit(S);
    return Rewriter.hasSeenLoopVariantSCEVUnknown()
        ? SE.getCouldNotCompute()
        : Result;
  }
 
  const SCEV *visitUnknown(const SCEVUnknown *Expr) {
    if (!SE.isLoopInvariant(Expr, L))
      SeenLoopVariantSCEVUnknown = true;
    return Expr;
  }
 
  const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
    // Only re-write AddRecExprs for this loop.
    if (Expr->getLoop() == L)
      return Expr->getPostIncExpr(SE);
    SeenOtherLoops = true;
    return Expr;
  }
 
  bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
 
  bool hasSeenOtherLoops() { return SeenOtherLoops; }
 
private:
  explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
      : SCEVRewriteVisitor(SE), L(L) {}
 
  const Loop *L;
  bool SeenLoopVariantSCEVUnknown = false;
  bool SeenOtherLoops = false;
};
 
/// This class evaluates the compare condition by matching it against the
/// condition of loop latch. If there is a match we assume a true value
/// for the condition while building SCEV nodes.
class SCEVBackedgeConditionFolder
    : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
public:
  static const SCEV *rewrite(const SCEV *S, const Loop *L,
                             ScalarEvolution &SE) {
    bool IsPosBECond = false;
    Value *BECond = nullptr;
    if (BasicBlock *Latch = L->getLoopLatch()) {
      if (CondBrInst *BI = dyn_cast<CondBrInst>(Latch->getTerminator())) {
        assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
               "Both outgoing branches should not target same header!");
        BECond = BI->getCondition();
        IsPosBECond = BI->getSuccessor(0) == L->getHeader();
      } else {
        return S;
      }
    }
    SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
    return Rewriter.visit(S);
  }
 
  const SCEV *visitUnknown(const SCEVUnknown *Expr) {
    const SCEV *Result = Expr;
    bool InvariantF = SE.isLoopInvariant(Expr, L);
 
    if (!InvariantF) {
      Instruction *I = cast<Instruction>(Expr->getValue());
      switch (I->getOpcode()) {
      case Instruction::Select: {
        SelectInst *SI = cast<SelectInst>(I);
        std::optional<const SCEV *> Res =
            compareWithBackedgeCondition(SI->getCondition());
        if (Res) {
          bool IsOne = cast<SCEVConstant>(*Res)->getValue()->isOne();
          Result = SE.getSCEV(IsOne ? SI->getTrueValue() : SI->getFalseValue());
        }
        break;
      }
      default: {
        std::optional<const SCEV *> Res = compareWithBackedgeCondition(I);
        if (Res)
          Result = *Res;
        break;
      }
      }
    }
    return Result;
  }
 
private:
  explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
                                       bool IsPosBECond, ScalarEvolution &SE)
      : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
        IsPositiveBECond(IsPosBECond) {}
 
  std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
 
  const Loop *L;
  /// Loop back condition.
  Value *BackedgeCond = nullptr;
  /// Set to true if loop back is on positive branch condition.
  bool IsPositiveBECond;
};
 
std::optional<const SCEV *>
SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
 
  // If value matches the backedge condition for loop latch,
  // then return a constant evolution node based on loopback
  // branch taken.
  if (BackedgeCond == IC)
    return IsPositiveBECond ? SE.getOne(Type::getInt1Ty(SE.getContext()))
                            : SE.getZero(Type::getInt1Ty(SE.getContext()));
  return std::nullopt;
}
 
class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
public:
  static const SCEV *rewrite(const SCEV *S, const Loop *L,
                             ScalarEvolution &SE) {
    SCEVShiftRewriter Rewriter(L, SE);
    const SCEV *Result = Rewriter.visit(S);
    return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
  }
 
  const SCEV *visitUnknown(const SCEVUnknown *Expr) {
    // Only allow AddRecExprs for this loop.
    if (!SE.isLoopInvariant(Expr, L))
      Valid = false;
    return Expr;
  }
 
  const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
    if (Expr->getLoop() == L && Expr->isAffine())
      return SE.getMinusSCEV(Expr, Expr->getStepRecurrence(SE));
    Valid = false;
    return Expr;
  }
 
  bool isValid() { return Valid; }
 
private:
  explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
      : SCEVRewriteVisitor(SE), L(L) {}
 
  const Loop *L;
  bool Valid = true;
};
 
} // end anonymous namespace
 
void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
  if (!AR->isAffine())
    return;
 
  // Force computation of ranges, which will also perform range-based flag
  // inference.
  if (!AR->hasNoSignedWrap())
    (void)getSignedRange(AR);
 
  if (!AR->hasNoUnsignedWrap())
    (void)getUnsignedRange(AR);
 
  if (!AR->hasNoSelfWrap()) {
    const SCEV *BECount = getConstantMaxBackedgeTakenCount(AR->getLoop());
    if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(BECount)) {
      ConstantRange StepCR = getSignedRange(AR->getStepRecurrence(*this));
      const APInt &BECountAP = BECountMax->getAPInt();
      unsigned NoOverflowBitWidth =
        BECountAP.getActiveBits() + StepCR.getMinSignedBits();
      if (NoOverflowBitWidth <= getTypeSizeInBits(AR->getType()))
        const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
    }
  }
}
 
SCEV::NoWrapFlags
ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
  SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
 
  if (AR->hasNoSignedWrap())
    return Result;
 
  if (!AR->isAffine())
    return Result;
 
  // This function can be expensive, only try to prove NSW once per AddRec.
  if (!SignedWrapViaInductionTried.insert(AR).second)
    return Result;
 
  const SCEV *Step = AR->getStepRecurrence(*this);
  const Loop *L = AR->getLoop();
 
  // Check whether the backedge-taken count is SCEVCouldNotCompute.
  // Note that this serves two purposes: It filters out loops that are
  // simply not analyzable, and it covers the case where this code is
  // being called from within backedge-taken count analysis, such that
  // attempting to ask for the backedge-taken count would likely result
  // in infinite recursion. In the later case, the analysis code will
  // cope with a conservative value, and it will take care to purge
  // that value once it has finished.
  const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
 
  // Normally, in the cases we can prove no-overflow via a
  // backedge guarding condition, we can also compute a backedge
  // taken count for the loop.  The exceptions are assumptions and
  // guards present in the loop -- SCEV is not great at exploiting
  // these to compute max backedge taken counts, but can still use
  // these to prove lack of overflow.  Use this fact to avoid
  // doing extra work that may not pay off.
 
  if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
      AC.assumptions().empty())
    return Result;
 
  // If the backedge is guarded by a comparison with the pre-inc  value the
  // addrec is safe. Also, if the entry is guarded by a comparison with the
  // start value and the backedge is guarded by a comparison with the post-inc
  // value, the addrec is safe.
  ICmpInst::Predicate Pred;
  const SCEV *OverflowLimit =
    getSignedOverflowLimitForStep(Step, &Pred, this);
  if (OverflowLimit &&
      (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
       isKnownOnEveryIteration(Pred, AR, OverflowLimit))) {
    Result = setFlags(Result, SCEV::FlagNSW);
  }
  return Result;
}
SCEV::NoWrapFlags
ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
  SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
 
  if (AR->hasNoUnsignedWrap())
    return Result;
 
  if (!AR->isAffine())
    return Result;
 
  // This function can be expensive, only try to prove NUW once per AddRec.
  if (!UnsignedWrapViaInductionTried.insert(AR).second)
    return Result;
 
  const SCEV *Step = AR->getStepRecurrence(*this);
  const Loop *L = AR->getLoop();
 
  // Check whether the backedge-taken count is SCEVCouldNotCompute.
  // Note that this serves two purposes: It filters out loops that are
  // simply not analyzable, and it covers the case where this code is
  // being called from within backedge-taken count analysis, such that
  // attempting to ask for the backedge-taken count would likely result
  // in infinite recursion. In the later case, the analysis code will
  // cope with a conservative value, and it will take care to purge
  // that value once it has finished.
  const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
 
  // Normally, in the cases we can prove no-overflow via a
  // backedge guarding condition, we can also compute a backedge
  // taken count for the loop.  The exceptions are assumptions and
  // guards present in the loop -- SCEV is not great at exploiting
  // these to compute max backedge taken counts, but can still use
  // these to prove lack of overflow.  Use this fact to avoid
  // doing extra work that may not pay off.
 
  if (isa<SCEVCouldNotCompute>(MaxBECount) && !HasGuards &&
      AC.assumptions().empty())
    return Result;
 
  // If the backedge is guarded by a comparison with the pre-inc  value the
  // addrec is safe. Also, if the entry is guarded by a comparison with the
  // start value and the backedge is guarded by a comparison with the post-inc
  // value, the addrec is safe.
  if (isKnownPositive(Step)) {
    ICmpInst::Predicate Pred;
    const SCEV *OverflowLimit =
        getUnsignedOverflowLimitForStep(Step, &Pred, this);
    if (isLoopBackedgeGuardedByCond(L, Pred, AR, OverflowLimit) ||
        isKnownOnEveryIteration(Pred, AR, OverflowLimit))
      Result = setFlags(Result, SCEV::FlagNUW);
  }
  return Result;
}
 
namespace {
 
/// Represents an abstract binary operation.  This may exist as a
/// normal instruction or constant expression, or may have been
/// derived from an expression tree.
struct BinaryOp {
  unsigned Opcode;
  Value *LHS;
  Value *RHS;
  bool IsNSW = false;
  bool IsNUW = false;
 
  /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
  /// constant expression.
  Operator *Op = nullptr;
 
  explicit BinaryOp(Operator *Op)
      : Opcode(Op->getOpcode()), LHS(Op->getOperand(0)), RHS(Op->getOperand(1)),
        Op(Op) {
    if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Op)) {
      IsNSW = OBO->hasNoSignedWrap();
      IsNUW = OBO->hasNoUnsignedWrap();
    }
  }
 
  explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
                    bool IsNUW = false)
      : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
};
 
} // end anonymous namespace
 
/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
                                             AssumptionCache &AC,
                                             const DominatorTree &DT,
                                             const Instruction *CxtI) {
  auto *Op = dyn_cast<Operator>(V);
  if (!Op)
    return std::nullopt;
 
  // Implementation detail: all the cleverness here should happen without
  // creating new SCEV expressions -- our caller knowns tricks to avoid creating
  // SCEV expressions when possible, and we should not break that.
 
  switch (Op->getOpcode()) {
  case Instruction::Add:
  case Instruction::Sub:
  case Instruction::Mul:
  case Instruction::UDiv:
  case Instruction::URem:
  case Instruction::And:
  case Instruction::AShr:
  case Instruction::Shl:
    return BinaryOp(Op);
 
  case Instruction::Or: {
    // Convert or disjoint into add nuw nsw.
    if (cast<PossiblyDisjointInst>(Op)->isDisjoint()) {
      BinaryOp BinOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1),
                     /*IsNSW=*/true, /*IsNUW=*/true);
      // Keep the reference to the original instruction so that we can later
      // check whether it can produce poison value or not.
      BinOp.Op = Op;
      return BinOp;
    }
    return BinaryOp(Op);
  }
 
  case Instruction::Xor:
    if (auto *RHSC = dyn_cast<ConstantInt>(Op->getOperand(1)))
      // If the RHS of the xor is a signmask, then this is just an add.
      // Instcombine turns add of signmask into xor as a strength reduction step.
      if (RHSC->getValue().isSignMask())
        return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1));
    // Binary `xor` is a bit-wise `add`.
    if (V->getType()->isIntegerTy(1))
      return BinaryOp(Instruction::Add, Op->getOperand(0), Op->getOperand(1));
    return BinaryOp(Op);
 
  case Instruction::LShr:
    // Turn logical shift right of a constant into a unsigned divide.
    if (ConstantInt *SA = dyn_cast<ConstantInt>(Op->getOperand(1))) {
      uint32_t BitWidth = cast<IntegerType>(Op->getType())->getBitWidth();
 
      // If the shift count is not less than the bitwidth, the result of
      // the shift is undefined. Don't try to analyze it, because the
      // resolution chosen here may differ from the resolution chosen in
      // other parts of the compiler.
      if (SA->getValue().ult(BitWidth)) {
        Constant *X =
            ConstantInt::get(SA->getContext(),
                             APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
        return BinaryOp(Instruction::UDiv, Op->getOperand(0), X);
      }
    }
    return BinaryOp(Op);
 
  case Instruction::ExtractValue: {
    auto *EVI = cast<ExtractValueInst>(Op);
    if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
      break;
 
    auto *WO = dyn_cast<WithOverflowInst>(EVI->getAggregateOperand());
    if (!WO)
      break;
 
    Instruction::BinaryOps BinOp = WO->getBinaryOp();
    bool Signed = WO->isSigned();
    // TODO: Should add nuw/nsw flags for mul as well.
    if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
      return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
 
    // Now that we know that all uses of the arithmetic-result component of
    // CI are guarded by the overflow check, we can go ahead and pretend
    // that the arithmetic is non-overflowing.
    return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
                    /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
  }
 
  default:
    break;
  }
 
  // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
  // semantics as a Sub, return a binary sub expression.
  if (auto *II = dyn_cast<IntrinsicInst>(V))
    if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
      return BinaryOp(Instruction::Sub, II->getOperand(0), II->getOperand(1));
 
  return std::nullopt;
}
 
/// Helper function to createAddRecFromPHIWithCasts. We have a phi
/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
/// follows one of the following patterns:
/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
/// If the SCEV expression of \p Op conforms with one of the expected patterns
/// we return the type of the truncation operation, and indicate whether the
/// truncated type should be treated as signed/unsigned by setting
/// \p Signed to true/false, respectively.
static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
                               bool &Signed, ScalarEvolution &SE) {
  // The case where Op == SymbolicPHI (that is, with no type conversions on
  // the way) is handled by the regular add recurrence creating logic and
  // would have already been triggered in createAddRecForPHI. Reaching it here
  // means that createAddRecFromPHI had failed for this PHI before (e.g.,
  // because one of the other operands of the SCEVAddExpr updating this PHI is
  // not invariant).
  //
  // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
  // this case predicates that allow us to prove that Op == SymbolicPHI will
  // be added.
  if (Op == SymbolicPHI)
    return nullptr;
 
  unsigned SourceBits = SE.getTypeSizeInBits(SymbolicPHI->getType());
  unsigned NewBits = SE.getTypeSizeInBits(Op->getType());
  if (SourceBits != NewBits)
    return nullptr;
 
  if (match(Op, m_scev_SExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) {
    Signed = true;
    return cast<SCEVCastExpr>(Op)->getOperand()->getType();
  }
  if (match(Op, m_scev_ZExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) {
    Signed = false;
    return cast<SCEVCastExpr>(Op)->getOperand()->getType();
  }
  return nullptr;
}
 
static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
  if (!PN->getType()->isIntegerTy())
    return nullptr;
  const Loop *L = LI.getLoopFor(PN->getParent());
  if (!L || L->getHeader() != PN->getParent())
    return nullptr;
  return L;
}
 
// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
// computation that updates the phi follows the following pattern:
//   (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
// which correspond to a phi->trunc->sext/zext->add->phi update chain.
// If so, try to see if it can be rewritten as an AddRecExpr under some
// Predicates. If successful, return them as a pair. Also cache the results
// of the analysis.
//
// Example usage scenario:
//    Say the Rewriter is called for the following SCEV:
//         8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
//    where:
//         %X = phi i64 (%Start, %BEValue)
//    It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
//    and call this function with %SymbolicPHI = %X.
//
//    The analysis will find that the value coming around the backedge has
//    the following SCEV:
//         BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
//    Upon concluding that this matches the desired pattern, the function
//    will return the pair {NewAddRec, SmallPredsVec} where:
//         NewAddRec = {%Start,+,%Step}
//         SmallPredsVec = {P1, P2, P3} as follows:
//           P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
//           P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
//           P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
//    The returned pair means that SymbolicPHI can be rewritten into NewAddRec
//    under the predicates {P1,P2,P3}.
//    This predicated rewrite will be cached in PredicatedSCEVRewrites:
//         PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
//
// TODO's:
//
// 1) Extend the Induction descriptor to also support inductions that involve
//    casts: When needed (namely, when we are called in the context of the
//    vectorizer induction analysis), a Set of cast instructions will be
//    populated by this method, and provided back to isInductionPHI. This is
//    needed to allow the vectorizer to properly record them to be ignored by
//    the cost model and to avoid vectorizing them (otherwise these casts,
//    which are redundant under the runtime overflow checks, will be
//    vectorized, which can be costly).
//
// 2) Support additional induction/PHISCEV patterns: We also want to support
//    inductions where the sext-trunc / zext-trunc operations (partly) occur
//    after the induction update operation (the induction increment):
//
//      (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
//    which correspond to a phi->add->trunc->sext/zext->phi update chain.
//
//      (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
//    which correspond to a phi->trunc->add->sext/zext->phi update chain.
//
// 3) Outline common code with createAddRecFromPHI to avoid duplication.
std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
  SmallVector<const SCEVPredicate *, 3> Predicates;
 
  // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
  // return an AddRec expression under some predicate.
 
  auto *PN = cast<PHINode>(SymbolicPHI->getValue());
  const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
  assert(L && "Expecting an integer loop header phi");
 
  // The loop may have multiple entrances or multiple exits; we can analyze
  // this phi as an addrec if it has a unique entry value and a unique
  // backedge value.
  Value *BEValueV = nullptr, *StartValueV = nullptr;
  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
    Value *V = PN->getIncomingValue(i);
    if (L->contains(PN->getIncomingBlock(i))) {
      if (!BEValueV) {
        BEValueV = V;
      } else if (BEValueV != V) {
        BEValueV = nullptr;
        break;
      }
    } else if (!StartValueV) {
      StartValueV = V;
    } else if (StartValueV != V) {
      StartValueV = nullptr;
      break;
    }
  }
  if (!BEValueV || !StartValueV)
    return std::nullopt;
 
  const SCEV *BEValue = getSCEV(BEValueV);
 
  // If the value coming around the backedge is an add with the symbolic
  // value we just inserted, possibly with casts that we can ignore under
  // an appropriate runtime guard, then we found a simple induction variable!
  const auto *Add = dyn_cast<SCEVAddExpr>(BEValue);
  if (!Add)
    return std::nullopt;
 
  // If there is a single occurrence of the symbolic value, possibly
  // casted, replace it with a recurrence.
  unsigned FoundIndex = Add->getNumOperands();
  Type *TruncTy = nullptr;
  bool Signed;
  for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
    if ((TruncTy =
             isSimpleCastedPHI(Add->getOperand(i), SymbolicPHI, Signed, *this)))
      if (FoundIndex == e) {
        FoundIndex = i;
        break;
      }
 
  if (FoundIndex == Add->getNumOperands())
    return std::nullopt;
 
  // Create an add with everything but the specified operand.
  SmallVector<SCEVUse, 8> Ops;
  for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
    if (i != FoundIndex)
      Ops.push_back(Add->getOperand(i));
  const SCEV *Accum = getAddExpr(Ops);
 
  // The runtime checks will not be valid if the step amount is
  // varying inside the loop.
  if (!isLoopInvariant(Accum, L))
    return std::nullopt;
 
  // *** Part2: Create the predicates
 
  // Analysis was successful: we have a phi-with-cast pattern for which we
  // can return an AddRec expression under the following predicates:
  //
  // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
  //     fits within the truncated type (does not overflow) for i = 0 to n-1.
  // P2: An Equal predicate that guarantees that
  //     Start = (Ext ix (Trunc iy (Start) to ix) to iy)
  // P3: An Equal predicate that guarantees that
  //     Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
  //
  // As we next prove, the above predicates guarantee that:
  //     Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
  //
  //
  // More formally, we want to prove that:
  //     Expr(i+1) = Start + (i+1) * Accum
  //               = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
  //
  // Given that:
  // 1) Expr(0) = Start
  // 2) Expr(1) = Start + Accum
  //            = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
  // 3) Induction hypothesis (step i):
  //    Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
  //
  // Proof:
  //  Expr(i+1) =
  //   = Start + (i+1)*Accum
  //   = (Start + i*Accum) + Accum
  //   = Expr(i) + Accum
  //   = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
  //                                                             :: from step i
  //
  //   = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
  //
  //   = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
  //     + (Ext ix (Trunc iy (Accum) to ix) to iy)
  //     + Accum                                                     :: from P3
  //
  //   = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
  //     + Accum                            :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
  //
  //   = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
  //   = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
  //
  // By induction, the same applies to all iterations 1<=i<n:
  //
 
  // Create a truncated addrec for which we will add a no overflow check (P1).
  const SCEV *StartVal = getSCEV(StartValueV);
  const SCEV *PHISCEV =
      getAddRecExpr(getTruncateExpr(StartVal, TruncTy),
                    getTruncateExpr(Accum, TruncTy), L, SCEV::FlagAnyWrap);
 
  // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
  // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
  // will be constant.
  //
  //  If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
  // add P1.
  if (const auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV)) {
    SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
        Signed ? SCEVWrapPredicate::IncrementNSSW
               : SCEVWrapPredicate::IncrementNUSW;
    const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
    Predicates.push_back(AddRecPred);
  }
 
  // Create the Equal Predicates P2,P3:
 
  // It is possible that the predicates P2 and/or P3 are computable at
  // compile time due to StartVal and/or Accum being constants.
  // If either one is, then we can check that now and escape if either P2
  // or P3 is false.
 
  // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
  // for each of StartVal and Accum
  auto getExtendedExpr = [&](const SCEV *Expr,
                             bool CreateSignExtend) -> const SCEV * {
    assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
    const SCEV *TruncatedExpr = getTruncateExpr(Expr, TruncTy);
    const SCEV *ExtendedExpr =
        CreateSignExtend ? getSignExtendExpr(TruncatedExpr, Expr->getType())
                         : getZeroExtendExpr(TruncatedExpr, Expr->getType());
    return ExtendedExpr;
  };
 
  // Given:
  //  ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
  //               = getExtendedExpr(Expr)
  // Determine whether the predicate P: Expr == ExtendedExpr
  // is known to be false at compile time
  auto PredIsKnownFalse = [&](const SCEV *Expr,
                              const SCEV *ExtendedExpr) -> bool {
    return Expr != ExtendedExpr &&
           isKnownPredicate(ICmpInst::ICMP_NE, Expr, ExtendedExpr);
  };
 
  const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
  if (PredIsKnownFalse(StartVal, StartExtended)) {
    LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
    return std::nullopt;
  }
 
  // The Step is always Signed (because the overflow checks are either
  // NSSW or NUSW)
  const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
  if (PredIsKnownFalse(Accum, AccumExtended)) {
    LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
    return std::nullopt;
  }
 
  auto AppendPredicate = [&](const SCEV *Expr,
                             const SCEV *ExtendedExpr) -> void {
    if (Expr != ExtendedExpr &&
        !isKnownPredicate(ICmpInst::ICMP_EQ, Expr, ExtendedExpr)) {
      const SCEVPredicate *Pred = getEqualPredicate(Expr, ExtendedExpr);
      LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
      Predicates.push_back(Pred);
    }
  };
 
  AppendPredicate(StartVal, StartExtended);
  AppendPredicate(Accum, AccumExtended);
 
  // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
  // which the casts had been folded away. The caller can rewrite SymbolicPHI
  // into NewAR if it will also add the runtime overflow checks specified in
  // Predicates.
  auto *NewAR = getAddRecExpr(StartVal, Accum, L, SCEV::FlagAnyWrap);
 
  std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
      std::make_pair(NewAR, Predicates);
  // Remember the result of the analysis for this SCEV at this locayyytion.
  PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
  return PredRewrite;
}
 
std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
  auto *PN = cast<PHINode>(SymbolicPHI->getValue());
  const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
  if (!L)
    return std::nullopt;
 
  // Check to see if we already analyzed this PHI.
  auto I = PredicatedSCEVRewrites.find({SymbolicPHI, L});
  if (I != PredicatedSCEVRewrites.end()) {
    std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
        I->second;
    // Analysis was done before and failed to create an AddRec:
    if (Rewrite.first == SymbolicPHI)
      return std::nullopt;
    // Analysis was done before and succeeded to create an AddRec under
    // a predicate:
    assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
    assert(!(Rewrite.second).empty() && "Expected to find Predicates");
    return Rewrite;
  }
 
  std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
    Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
 
  // Record in the cache that the analysis failed
  if (!Rewrite) {
    SmallVector<const SCEVPredicate *, 3> Predicates;
    PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
    return std::nullopt;
  }
 
  return Rewrite;
}
 
// FIXME: This utility is currently required because the Rewriter currently
// does not rewrite this expression:
// {0, +, (sext ix (trunc iy to ix) to iy)}
// into {0, +, %step},
// even when the following Equal predicate exists:
// "%step == (sext ix (trunc iy to ix) to iy)".
bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
    const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
    ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
  if (AR1 == AR2)
    return true;
 
  SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
  SCEVUnionPredicate AllPreds = Preds->getUnionWith(&NoWrapUnionPred, SE);
  auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
    if (Expr1 != Expr2 &&
        !AllPreds.implies(SE.getEqualPredicate(Expr1, Expr2), SE) &&
        !AllPreds.implies(SE.getEqualPredicate(Expr2, Expr1), SE))
      return false;
    return true;
  };
 
  if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
      !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
    return false;
  return true;
}
 
/// A helper function for createAddRecFromPHI to handle simple cases.
///
/// This function tries to find an AddRec expression for the simplest (yet most
/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
/// If it fails, createAddRecFromPHI will use a more general, but slow,
/// technique for finding the AddRec expression.
const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
                                                      Value *BEValueV,
                                                      Value *StartValueV) {
  const Loop *L = LI.getLoopFor(PN->getParent());
  assert(L && L->getHeader() == PN->getParent());
  assert(BEValueV && StartValueV);
 
  auto BO = MatchBinaryOp(BEValueV, getDataLayout(), AC, DT, PN);
  if (!BO)
    return nullptr;
 
  if (BO->Opcode != Instruction::Add)
    return nullptr;
 
  const SCEV *Accum = nullptr;
  if (BO->LHS == PN && L->isLoopInvariant(BO->RHS))
    Accum = getSCEV(BO->RHS);
  else if (BO->RHS == PN && L->isLoopInvariant(BO->LHS))
    Accum = getSCEV(BO->LHS);
 
  if (!Accum)
    return nullptr;
 
  SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
  if (BO->IsNUW)
    Flags = setFlags(Flags, SCEV::FlagNUW);
  if (BO->IsNSW)
    Flags = setFlags(Flags, SCEV::FlagNSW);
 
  const SCEV *StartVal = getSCEV(StartValueV);
  const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
  insertValueToMap(PN, PHISCEV);
 
  if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV))
    inferNoWrapViaConstantRanges(AR);
 
  // We can add Flags to the post-inc expression only if we
  // know that it is *undefined behavior* for BEValueV to
  // overflow.
  if (auto *BEInst = dyn_cast<Instruction>(BEValueV)) {
    assert(isLoopInvariant(Accum, L) &&
           "Accum is defined outside L, but is not invariant?");
    if (isAddRecNeverPoison(BEInst, L))
      (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
  }
 
  return PHISCEV;
}
 
const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
  const Loop *L = LI.getLoopFor(PN->getParent());
  if (!L || L->getHeader() != PN->getParent())
    return nullptr;
 
  // The loop may have multiple entrances or multiple exits; we can analyze
  // this phi as an addrec if it has a unique entry value and a unique
  // backedge value.
  Value *BEValueV = nullptr, *StartValueV = nullptr;
  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
    Value *V = PN->getIncomingValue(i);
    if (L->contains(PN->getIncomingBlock(i))) {
      if (!BEValueV) {
        BEValueV = V;
      } else if (BEValueV != V) {
        BEValueV = nullptr;
        break;
      }
    } else if (!StartValueV) {
      StartValueV = V;
    } else if (StartValueV != V) {
      StartValueV = nullptr;
      break;
    }
  }
  if (!BEValueV || !StartValueV)
    return nullptr;
 
  assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
         "PHI node already processed?");
 
  // First, try to find AddRec expression without creating a fictituos symbolic
  // value for PN.
  if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
    return S;
 
  // Handle PHI node value symbolically.
  const SCEV *SymbolicName = getUnknown(PN);
  insertValueToMap(PN, SymbolicName);
 
  // Using this symbolic name for the PHI, analyze the value coming around
  // the back-edge.
  const SCEV *BEValue = getSCEV(BEValueV);
 
  // NOTE: If BEValue is loop invariant, we know that the PHI node just
  // has a special value for the first iteration of the loop.
 
  // If the value coming around the backedge is an add with the symbolic
  // value we just inserted, then we found a simple induction variable!
  if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(BEValue)) {
    // If there is a single occurrence of the symbolic value, replace it
    // with a recurrence.
    unsigned FoundIndex = Add->getNumOperands();
    for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
      if (Add->getOperand(i) == SymbolicName)
        if (FoundIndex == e) {
          FoundIndex = i;
          break;
        }
 
    if (FoundIndex != Add->getNumOperands()) {
      // Create an add with everything but the specified operand.
      SmallVector<SCEVUse, 8> Ops;
      for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
        if (i != FoundIndex)
          Ops.push_back(SCEVBackedgeConditionFolder::rewrite(Add->getOperand(i),
                                                             L, *this));
      const SCEV *Accum = getAddExpr(Ops);
 
      // This is not a valid addrec if the step amount is varying each
      // loop iteration, but is not itself an addrec in this loop.
      if (isLoopInvariant(Accum, L) ||
          (isa<SCEVAddRecExpr>(Accum) &&
           cast<SCEVAddRecExpr>(Accum)->getLoop() == L)) {
        SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
 
        if (auto BO = MatchBinaryOp(BEValueV, getDataLayout(), AC, DT, PN)) {
          if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
            if (BO->IsNUW)
              Flags = setFlags(Flags, SCEV::FlagNUW);
            if (BO->IsNSW)
              Flags = setFlags(Flags, SCEV::FlagNSW);
          }
        } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(BEValueV)) {
          if (GEP->getOperand(0) == PN) {
            GEPNoWrapFlags NW = GEP->getNoWrapFlags();
            // If the increment has any nowrap flags, then we know the address
            // space cannot be wrapped around.
            if (NW != GEPNoWrapFlags::none())
              Flags = setFlags(Flags, SCEV::FlagNW);
            // If the GEP is nuw or nusw with non-negative offset, we know that
            // no unsigned wrap occurs. We cannot set the nsw flag as only the
            // offset is treated as signed, while the base is unsigned.
            if (NW.hasNoUnsignedWrap() ||
                (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(Accum)))
              Flags = setFlags(Flags, SCEV::FlagNUW);
          }
 
          // We cannot transfer nuw and nsw flags from subtraction
          // operations -- sub nuw X, Y is not the same as add nuw X, -Y
          // for instance.
        }
 
        const SCEV *StartVal = getSCEV(StartValueV);
        const SCEV *PHISCEV = getAddRecExpr(StartVal, Accum, L, Flags);
 
        // Okay, for the entire analysis of this edge we assumed the PHI
        // to be symbolic.  We now need to go back and purge all of the
        // entries for the scalars that use the symbolic expression.
        forgetMemoizedResults({SymbolicName});
        insertValueToMap(PN, PHISCEV);
 
        if (auto *AR = dyn_cast<SCEVAddRecExpr>(PHISCEV))
          inferNoWrapViaConstantRanges(AR);
 
        // We can add Flags to the post-inc expression only if we
        // know that it is *undefined behavior* for BEValueV to
        // overflow.
        if (auto *BEInst = dyn_cast<Instruction>(BEValueV))
          if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L))
            (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
 
        return PHISCEV;
      }
    }
  } else {
    // Otherwise, this could be a loop like this:
    //     i = 0;  for (j = 1; ..; ++j) { ....  i = j; }
    // In this case, j = {1,+,1}  and BEValue is j.
    // Because the other in-value of i (0) fits the evolution of BEValue
    // i really is an addrec evolution.
    //
    // We can generalize this saying that i is the shifted value of BEValue
    // by one iteration:
    //   PHI(f(0), f({1,+,1})) --> f({0,+,1})
 
    // Do not allow refinement in rewriting of BEValue.
    const SCEV *Shifted = SCEVShiftRewriter::rewrite(BEValue, L, *this);
    const SCEV *Start = SCEVInitRewriter::rewrite(Shifted, L, *this, false);
    if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
        isGuaranteedNotToCauseUB(Shifted) && ::impliesPoison(Shifted, Start)) {
      const SCEV *StartVal = getSCEV(StartValueV);
      if (Start == StartVal) {
        // Okay, for the entire analysis of this edge we assumed the PHI
        // to be symbolic.  We now need to go back and purge all of the
        // entries for the scalars that use the symbolic expression.
        forgetMemoizedResults({SymbolicName});
        insertValueToMap(PN, Shifted);
        return Shifted;
      }
    }
  }
 
  // Remove the temporary PHI node SCEV that has been inserted while intending
  // to create an AddRecExpr for this PHI node. We can not keep this temporary
  // as it will prevent later (possibly simpler) SCEV expressions to be added
  // to the ValueExprMap.
  eraseValueFromMap(PN);
 
  return nullptr;
}
 
// Try to match a control flow sequence that branches out at BI and merges back
// at Merge into a "C ? LHS : RHS" select pattern.  Return true on a successful
// match.
static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge,
                          Value *&C, Value *&LHS, Value *&RHS) {
  C = BI->getCondition();
 
  BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(0));
  BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(1));
 
  Use &LeftUse = Merge->getOperandUse(0);
  Use &RightUse = Merge->getOperandUse(1);
 
  if (DT.dominates(LeftEdge, LeftUse) && DT.dominates(RightEdge, RightUse)) {
    LHS = LeftUse;
    RHS = RightUse;
    return true;
  }
 
  if (DT.dominates(LeftEdge, RightUse) && DT.dominates(RightEdge, LeftUse)) {
    LHS = RightUse;
    RHS = LeftUse;
    return true;
  }
 
  return false;
}
 
static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN,
                                        Value *&Cond, Value *&LHS,
                                        Value *&RHS) {
  auto IsReachable =
      [&](BasicBlock *BB) { return DT.isReachableFromEntry(BB); };
  if (PN->getNumIncomingValues() == 2 && all_of(PN->blocks(), IsReachable)) {
    // Try to match
    //
    //  br %cond, label %left, label %right
    // left:
    //  br label %merge
    // right:
    //  br label %merge
    // merge:
    //  V = phi [ %x, %left ], [ %y, %right ]
    //
    // as "select %cond, %x, %y"
 
    BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
    assert(IDom && "At least the entry block should dominate PN");
 
    auto *BI = dyn_cast<CondBrInst>(IDom->getTerminator());
    return BI && BrPHIToSelect(DT, BI, PN, Cond, LHS, RHS);
  }
  return false;
}
 
const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
  Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
  if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
      properlyDominates(getSCEV(LHS), PN->getParent()) &&
      properlyDominates(getSCEV(RHS), PN->getParent()))
    return createNodeForSelectOrPHI(PN, Cond, LHS, RHS);
 
  return nullptr;
}
 
static BinaryOperator *getCommonInstForPHI(PHINode *PN) {
  BinaryOperator *CommonInst = nullptr;
  // Check if instructions are identical.
  for (Value *Incoming : PN->incoming_values()) {
    auto *IncomingInst = dyn_cast<BinaryOperator>(Incoming);
    if (!IncomingInst)
      return nullptr;
    if (CommonInst) {
      if (!CommonInst->isIdenticalToWhenDefined(IncomingInst))
        return nullptr; // Not identical, give up
    } else {
      // Remember binary operator
      CommonInst = IncomingInst;
    }
  }
  return CommonInst;
}
 
/// Returns SCEV for the first operand of a phi if all phi operands have
/// identical opcodes and operands
/// eg.
/// a: %add = %a + %b
///    br %c
/// b: %add1 = %a + %b
///    br %c
/// c: %phi = phi [%add, a], [%add1, b]
/// scev(%phi) => scev(%add)
const SCEV *
ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
  BinaryOperator *CommonInst = getCommonInstForPHI(PN);
  if (!CommonInst)
    return nullptr;
 
  // Check if SCEV exprs for instructions are identical.
  const SCEV *CommonSCEV = getSCEV(CommonInst);
  bool SCEVExprsIdentical =
      all_of(drop_begin(PN->incoming_values()),
             [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
  return SCEVExprsIdentical ? CommonSCEV : nullptr;
}
 
const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
  if (const SCEV *S = createAddRecFromPHI(PN))
    return S;
 
  // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
  // phi node for X.
  if (Value *V = simplifyInstruction(
          PN, {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
               /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
    return getSCEV(V);
 
  if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
    return S;
 
  if (const SCEV *S = createNodeFromSelectLikePHI(PN))
    return S;
 
  // If it's not a loop phi, we can't handle it yet.
  return getUnknown(PN);
}
 
bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
                            SCEVTypes RootKind) {
  struct FindClosure {
    const SCEV *OperandToFind;
    const SCEVTypes RootKind; // Must be a sequential min/max expression.
    const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
 
    bool Found = false;
 
    bool canRecurseInto(SCEVTypes Kind) const {
      // We can only recurse into the SCEV expression of the same effective type
      // as the type of our root SCEV expression, and into zero-extensions.
      return RootKind == Kind || NonSequentialRootKind == Kind ||
             scZeroExtend == Kind;
    };
 
    FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
        : OperandToFind(OperandToFind), RootKind(RootKind),
          NonSequentialRootKind(
              SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
                  RootKind)) {}
 
    bool follow(const SCEV *S) {
      Found = S == OperandToFind;
 
      return !isDone() && canRecurseInto(S->getSCEVType());
    }
 
    bool isDone() const { return Found; }
  };
 
  FindClosure FC(OperandToFind, RootKind);
  visitAll(Root, FC);
  return FC.Found;
}
 
std::optional<const SCEV *>
ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
                                                              ICmpInst *Cond,
                                                              Value *TrueVal,
                                                              Value *FalseVal) {
  // Try to match some simple smax or umax patterns.
  auto *ICI = Cond;
 
  Value *LHS = ICI->getOperand(0);
  Value *RHS = ICI->getOperand(1);
 
  switch (ICI->getPredicate()) {
  case ICmpInst::ICMP_SLT:
  case ICmpInst::ICMP_SLE:
  case ICmpInst::ICMP_ULT:
  case ICmpInst::ICMP_ULE:
    std::swap(LHS, RHS);
    [[fallthrough]];
  case ICmpInst::ICMP_SGT:
  case ICmpInst::ICMP_SGE:
  case ICmpInst::ICMP_UGT:
  case ICmpInst::ICMP_UGE:
    // a > b ? a+x : b+x  ->  max(a, b)+x
    // a > b ? b+x : a+x  ->  min(a, b)+x
    if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(Ty)) {
      bool Signed = ICI->isSigned();
      const SCEV *LA = getSCEV(TrueVal);
      const SCEV *RA = getSCEV(FalseVal);
      const SCEV *LS = getSCEV(LHS);
      const SCEV *RS = getSCEV(RHS);
      if (LA->getType()->isPointerTy()) {
        // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
        // Need to make sure we can't produce weird expressions involving
        // negated pointers.
        if (LA == LS && RA == RS)
          return Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS);
        if (LA == RS && RA == LS)
          return Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS);
      }
      auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
        if (Op->getType()->isPointerTy()) {
          Op = getPtrToAddrExpr(Op);
          if (isa<SCEVCouldNotCompute>(Op))
            return Op;
        }
        if (Signed)
          Op = getNoopOrSignExtend(Op, Ty);
        else
          Op = getNoopOrZeroExtend(Op, Ty);
        return Op;
      };
      LS = CoerceOperand(LS);
      RS = CoerceOperand(RS);
      if (isa<SCEVCouldNotCompute>(LS) || isa<SCEVCouldNotCompute>(RS))
        break;
      const SCEV *LDiff = getMinusSCEV(LA, LS);
      const SCEV *RDiff = getMinusSCEV(RA, RS);
      if (LDiff == RDiff)
        return getAddExpr(Signed ? getSMaxExpr(LS, RS) : getUMaxExpr(LS, RS),
                          LDiff);
      LDiff = getMinusSCEV(LA, RS);
      RDiff = getMinusSCEV(RA, LS);
      if (LDiff == RDiff)
        return getAddExpr(Signed ? getSMinExpr(LS, RS) : getUMinExpr(LS, RS),
                          LDiff);
    }
    break;
  case ICmpInst::ICMP_NE:
    // x != 0 ? x+y : C+y  ->  x == 0 ? C+y : x+y
    std::swap(TrueVal, FalseVal);
    [[fallthrough]];
  case ICmpInst::ICMP_EQ:
    // x == 0 ? C+y : x+y  ->  umax(x, C)+y   iff C u<= 1
    if (getTypeSizeInBits(LHS->getType()) <= getTypeSizeInBits(Ty) &&
        isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()) {
      const SCEV *X = getNoopOrZeroExtend(getSCEV(LHS), Ty);
      const SCEV *TrueValExpr = getSCEV(TrueVal);    // C+y
      const SCEV *FalseValExpr = getSCEV(FalseVal);  // x+y
      const SCEV *Y = getMinusSCEV(FalseValExpr, X); // y = (x+y)-x
      const SCEV *C = getMinusSCEV(TrueValExpr, Y);  // C = (C+y)-y
      if (isa<SCEVConstant>(C) && cast<SCEVConstant>(C)->getAPInt().ule(1))
        return getAddExpr(getUMaxExpr(X, C), Y);
    }
    // x == 0 ? 0 : umin    (..., x, ...)  ->  umin_seq(x, umin    (...))
    // x == 0 ? 0 : umin_seq(..., x, ...)  ->  umin_seq(x, umin_seq(...))
    // x == 0 ? 0 : umin    (..., umin_seq(..., x, ...), ...)
    //                    ->  umin_seq(x, umin (..., umin_seq(...), ...))
    if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero() &&
        isa<ConstantInt>(TrueVal) && cast<ConstantInt>(TrueVal)->isZero()) {
      const SCEV *X = getSCEV(LHS);
      while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(X))
        X = ZExt->getOperand();
      if (getTypeSizeInBits(X->getType()) <= getTypeSizeInBits(Ty)) {
        const SCEV *FalseValExpr = getSCEV(FalseVal);
        if (SCEVMinMaxExprContains(FalseValExpr, X, scSequentialUMinExpr))
          return getUMinExpr(getNoopOrZeroExtend(X, Ty), FalseValExpr,
                             /*Sequential=*/true);
      }
    }
    break;
  default:
    break;
  }
 
  return std::nullopt;
}
 
static std::optional<const SCEV *>
createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
                              const SCEV *TrueExpr, const SCEV *FalseExpr) {
  assert(CondExpr->getType()->isIntegerTy(1) &&
         TrueExpr->getType() == FalseExpr->getType() &&
         TrueExpr->getType()->isIntegerTy(1) &&
         "Unexpected operands of a select.");
 
  // i1 cond ? i1 x : i1 C  -->  C + (i1  cond ? (i1 x - i1 C) : i1 0)
  //                        -->  C + (umin_seq  cond, x - C)
  //
  // i1 cond ? i1 C : i1 x  -->  C + (i1  cond ? i1 0 : (i1 x - i1 C))
  //                        -->  C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
  //                        -->  C + (umin_seq ~cond, x - C)
 
  // FIXME: while we can't legally model the case where both of the hands
  // are fully variable, we only require that the *difference* is constant.
  if (!isa<SCEVConstant>(TrueExpr) && !isa<SCEVConstant>(FalseExpr))
    return std::nullopt;
 
  const SCEV *X, *C;
  if (isa<SCEVConstant>(TrueExpr)) {
    CondExpr = SE->getNotSCEV(CondExpr);
    X = FalseExpr;
    C = TrueExpr;
  } else {
    X = TrueExpr;
    C = FalseExpr;
  }
  return SE->getAddExpr(C, SE->getUMinExpr(CondExpr, SE->getMinusSCEV(X, C),
                                           /*Sequential=*/true));
}
 
static std::optional<const SCEV *>
createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
                              Value *FalseVal) {
  if (!isa<ConstantInt>(TrueVal) && !isa<ConstantInt>(FalseVal))
    return std::nullopt;
 
  const auto *SECond = SE->getSCEV(Cond);
  const auto *SETrue = SE->getSCEV(TrueVal);
  const auto *SEFalse = SE->getSCEV(FalseVal);
  return createNodeForSelectViaUMinSeq(SE, SECond, SETrue, SEFalse);
}
 
const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
    Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
  assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
  assert(TrueVal->getType() == FalseVal->getType() &&
         V->getType() == TrueVal->getType() &&
         "Types of select hands and of the result must match.");
 
  // For now, only deal with i1-typed `select`s.
  if (!V->getType()->isIntegerTy(1))
    return getUnknown(V);
 
  if (std::optional<const SCEV *> S =
          createNodeForSelectViaUMinSeq(this, Cond, TrueVal, FalseVal))
    return *S;
 
  return getUnknown(V);
}
 
const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
                                                      Value *TrueVal,
                                                      Value *FalseVal) {
  // Handle "constant" branch or select. This can occur for instance when a
  // loop pass transforms an inner loop and moves on to process the outer loop.
  if (auto *CI = dyn_cast<ConstantInt>(Cond))
    return getSCEV(CI->isOne() ? TrueVal : FalseVal);
 
  if (auto *I = dyn_cast<Instruction>(V)) {
    if (auto *ICI = dyn_cast<ICmpInst>(Cond)) {
      if (std::optional<const SCEV *> S =
              createNodeForSelectOrPHIInstWithICmpInstCond(I->getType(), ICI,
                                                           TrueVal, FalseVal))
        return *S;
    }
  }
 
  return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
}
 
/// Expand GEP instructions into add and multiply operations. This allows them
/// to be analyzed by regular SCEV code.
const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
  assert(GEP->getSourceElementType()->isSized() &&
         "GEP source element type must be sized");
 
  SmallVector<SCEVUse, 4> IndexExprs;
  for (Value *Index : GEP->indices())
    IndexExprs.push_back(getSCEV(Index));
  return getGEPExpr(GEP, IndexExprs);
}
 
APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
                                               const Instruction *CtxI) {
  uint64_t BitWidth = getTypeSizeInBits(S->getType());
  auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
    return TrailingZeros >= BitWidth
               ? APInt::getZero(BitWidth)
               : APInt::getOneBitSet(BitWidth, TrailingZeros);
  };
  auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
    // The result is GCD of all operands results.
    APInt Res = getConstantMultiple(N->getOperand(0), CtxI);
    for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
      Res = APIntOps::GreatestCommonDivisor(
          Res, getConstantMultiple(N->getOperand(I), CtxI));
    return Res;
  };
 
  switch (S->getSCEVType()) {
  case scConstant:
    return cast<SCEVConstant>(S)->getAPInt();
  case scPtrToAddr:
    return getConstantMultiple(cast<SCEVCastExpr>(S)->getOperand());
  case scUDivExpr:
  case scVScale:
    return APInt(BitWidth, 1);
  case scTruncate: {
    // Only multiples that are a power of 2 will hold after truncation.
    const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(S);
    uint32_t TZ = getMinTrailingZeros(T->getOperand(), CtxI);
    return GetShiftedByZeros(TZ);
  }
  case scZeroExtend: {
    const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(S);
    return getConstantMultiple(Z->getOperand(), CtxI).zext(BitWidth);
  }
  case scSignExtend: {
    // Only multiples that are a power of 2 will hold after sext.
    const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(S);
    uint32_t TZ = getMinTrailingZeros(E->getOperand(), CtxI);
    return GetShiftedByZeros(TZ);
  }
  case scMulExpr: {
    const SCEVMulExpr *M = cast<SCEVMulExpr>(S);
    if (M->hasNoUnsignedWrap()) {
      // The result is the product of all operand results.
      APInt Res = getConstantMultiple(M->getOperand(0), CtxI);
      for (const SCEV *Operand : M->operands().drop_front())
        Res = Res * getConstantMultiple(Operand, CtxI);
      return Res;
    }
 
    // If there are no wrap guarentees, find the trailing zeros, which is the
    // sum of trailing zeros for all its operands.
    uint32_t TZ = 0;
    for (const SCEV *Operand : M->operands())
      TZ += getMinTrailingZeros(Operand, CtxI);
    return GetShiftedByZeros(TZ);
  }
  case scAddExpr:
  case scAddRecExpr: {
    const SCEVNAryExpr *N = cast<SCEVNAryExpr>(S);
    if (N->hasNoUnsignedWrap())
        return GetGCDMultiple(N);
    // Find the trailing bits, which is the minimum of its operands.
    uint32_t TZ = getMinTrailingZeros(N->getOperand(0), CtxI);
    for (const SCEV *Operand : N->operands().drop_front())
      TZ = std::min(TZ, getMinTrailingZeros(Operand, CtxI));
    return GetShiftedByZeros(TZ);
  }
  case scUMaxExpr:
  case scSMaxExpr:
  case scUMinExpr:
  case scSMinExpr:
  case scSequentialUMinExpr:
    return GetGCDMultiple(cast<SCEVNAryExpr>(S));
  case scUnknown: {
    // Ask ValueTracking for known bits. SCEVUnknown only become available at
    // the point their underlying IR instruction has been defined. If CtxI was
    // not provided, use:
    // * the first instruction in the entry block if it is an argument
    // * the instruction itself otherwise.
    const SCEVUnknown *U = cast<SCEVUnknown>(S);
    if (!CtxI) {
      if (isa<Argument>(U->getValue()))
        CtxI = &*F.getEntryBlock().begin();
      else if (auto *I = dyn_cast<Instruction>(U->getValue()))
        CtxI = I;
    }
    unsigned Known =
        computeKnownBits(U->getValue(),
                         SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
                             .allowEphemerals(true))
            .countMinTrailingZeros();
    return GetShiftedByZeros(Known);
  }
  case scCouldNotCompute:
    llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
  }
  llvm_unreachable("Unknown SCEV kind!");
}
 
APInt ScalarEvolution::getConstantMultiple(const SCEV *S,
                                           const Instruction *CtxI) {
  // Skip looking up and updating the cache if there is a context instruction,
  // as the result will only be valid in the specified context.
  if (CtxI)
    return getConstantMultipleImpl(S, CtxI);
 
  auto I = ConstantMultipleCache.find(S);
  if (I != ConstantMultipleCache.end())
    return I->second;
 
  APInt Result = getConstantMultipleImpl(S, CtxI);
  auto InsertPair = ConstantMultipleCache.insert({S, Result});
  assert(InsertPair.second && "Should insert a new key");
  return InsertPair.first->second;
}
 
APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
  APInt Multiple = getConstantMultiple(S);
  return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
}
 
uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S,
                                              const Instruction *CtxI) {
  return std::min(getConstantMultiple(S, CtxI).countTrailingZeros(),
                  (unsigned)getTypeSizeInBits(S->getType()));
}
 
/// Helper method to assign a range to V from metadata present in the IR.
static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
  if (Instruction *I = dyn_cast<Instruction>(V)) {
    if (MDNode *MD = I->getMetadata(LLVMContext::MD_range))
      return getConstantRangeFromMetadata(*MD);
    if (const auto *CB = dyn_cast<CallBase>(V))
      if (std::optional<ConstantRange> Range = CB->getRange())
        return Range;
  }
  if (auto *A = dyn_cast<Argument>(V))
    if (std::optional<ConstantRange> Range = A->getRange())
      return Range;
 
  return std::nullopt;
}
 
void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
                                     SCEV::NoWrapFlags Flags) {
  if (AddRec->getNoWrapFlags(Flags) != Flags) {
    AddRec->setNoWrapFlags(Flags);
    UnsignedRanges.erase(AddRec);
    SignedRanges.erase(AddRec);
    ConstantMultipleCache.erase(AddRec);
  }
}
 
ConstantRange ScalarEvolution::
getRangeForUnknownRecurrence(const SCEVUnknown *U) {
  const DataLayout &DL = getDataLayout();
 
  unsigned BitWidth = getTypeSizeInBits(U->getType());
  const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
 
  // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
  // use information about the trip count to improve our available range.  Note
  // that the trip count independent cases are already handled by known bits.
  // WARNING: The definition of recurrence used here is subtly different than
  // the one used by AddRec (and thus most of this file).  Step is allowed to
  // be arbitrarily loop varying here, where AddRec allows only loop invariant
  // and other addrecs in the same loop (for non-affine addrecs).  The code
  // below intentionally handles the case where step is not loop invariant.
  auto *P = dyn_cast<PHINode>(U->getValue());
  if (!P)
    return FullSet;
 
  // Make sure that no Phi input comes from an unreachable block. Otherwise,
  // even the values that are not available in these blocks may come from them,
  // and this leads to false-positive recurrence test.
  for (auto *Pred : predecessors(P->getParent()))
    if (!DT.isReachableFromEntry(Pred))
      return FullSet;
 
  BinaryOperator *BO;
  Value *Start, *Step;
  if (!matchSimpleRecurrence(P, BO, Start, Step))
    return FullSet;
 
  // If we found a recurrence in reachable code, we must be in a loop. Note
  // that BO might be in some subloop of L, and that's completely okay.
  auto *L = LI.getLoopFor(P->getParent());
  assert(L && L->getHeader() == P->getParent());
  if (!L->contains(BO->getParent()))
    // NOTE: This bailout should be an assert instead.  However, asserting
    // the condition here exposes a case where LoopFusion is querying SCEV
    // with malformed loop information during the midst of the transform.
    // There doesn't appear to be an obvious fix, so for the moment bailout
    // until the caller issue can be fixed.  PR49566 tracks the bug.
    return FullSet;
 
  // TODO: Extend to other opcodes such as mul, and div
  switch (BO->getOpcode()) {
  default:
    return FullSet;
  case Instruction::AShr:
  case Instruction::LShr:
  case Instruction::Shl:
    break;
  };
 
  if (BO->getOperand(0) != P)
    // TODO: Handle the power function forms some day.
    return FullSet;
 
  unsigned TC = getSmallConstantMaxTripCount(L);
  if (!TC || TC >= BitWidth)
    return FullSet;
 
  auto KnownStart = computeKnownBits(Start, DL, &AC, nullptr, &DT);
  auto KnownStep = computeKnownBits(Step, DL, &AC, nullptr, &DT);
  assert(KnownStart.getBitWidth() == BitWidth &&
         KnownStep.getBitWidth() == BitWidth);
 
  // Compute total shift amount, being careful of overflow and bitwidths.
  auto MaxShiftAmt = KnownStep.getMaxValue();
  APInt TCAP(BitWidth, TC-1);
  bool Overflow = false;
  auto TotalShift = MaxShiftAmt.umul_ov(TCAP, Overflow);
  if (Overflow)
    return FullSet;
 
  switch (BO->getOpcode()) {
  default:
    llvm_unreachable("filtered out above");
  case Instruction::AShr: {
    // For each ashr, three cases:
    //   shift = 0 => unchanged value
    //   saturation => 0 or -1
    //   other => a value closer to zero (of the same sign)
    // Thus, the end value is closer to zero than the start.
    auto KnownEnd = KnownBits::ashr(KnownStart,
                                    KnownBits::makeConstant(TotalShift));
    if (KnownStart.isNonNegative())
      // Analogous to lshr (simply not yet canonicalized)
      return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
                                        KnownStart.getMaxValue() + 1);
    if (KnownStart.isNegative())
      // End >=u Start && End <=s Start
      return ConstantRange::getNonEmpty(KnownStart.getMinValue(),
                                        KnownEnd.getMaxValue() + 1);
    break;
  }
  case Instruction::LShr: {
    // For each lshr, three cases:
    //   shift = 0 => unchanged value
    //   saturation => 0
    //   other => a smaller positive number
    // Thus, the low end of the unsigned range is the last value produced.
    auto KnownEnd = KnownBits::lshr(KnownStart,
                                    KnownBits::makeConstant(TotalShift));
    return ConstantRange::getNonEmpty(KnownEnd.getMinValue(),
                                      KnownStart.getMaxValue() + 1);
  }
  case Instruction::Shl: {
    // Iff no bits are shifted out, value increases on every shift.
    auto KnownEnd = KnownBits::shl(KnownStart,
                                   KnownBits::makeConstant(TotalShift));
    if (TotalShift.ult(KnownStart.countMinLeadingZeros()))
      return ConstantRange(KnownStart.getMinValue(),
                           KnownEnd.getMaxValue() + 1);
    break;
  }
  };
  return FullSet;
}
 
// The goal of this function is to check if recursively visiting the operands
// of this PHI might lead to an infinite loop. If we do see such a loop,
// there's no good way to break it, so we avoid analyzing such cases.
//
// getRangeRef previously used a visited set to avoid infinite loops, but this
// caused other issues: the result was dependent on the order of getRangeRef
// calls, and the interaction with createSCEVIter could cause a stack overflow
// in some cases (see issue #148253).
//
// FIXME: The way this is implemented is overly conservative; this checks
// for a few obviously safe patterns, but anything that doesn't lead to
// recursion is fine.
static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI) {
  Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
  if (getOperandsForSelectLikePHI(DT, PHI, Cond, LHS, RHS))
    return true;
 
  if (all_of(PHI->operands(),
             [&](Value *Operand) { return DT.dominates(Operand, PHI); }))
    return true;
 
  return false;
}
 
const ConstantRange &
ScalarEvolution::getRangeRefIter(const SCEV *S,
                                 ScalarEvolution::RangeSignHint SignHint) {
  DenseMap<const SCEV *, ConstantRange> &Cache =
      SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
                                                       : SignedRanges;
  SmallVector<SCEVUse> WorkList;
  SmallPtrSet<const SCEV *, 8> Seen;
 
  // Add Expr to the worklist, if Expr is either an N-ary expression or a
  // SCEVUnknown PHI node.
  auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
    if (!Seen.insert(Expr).second)
      return;
    if (Cache.contains(Expr))
      return;
    switch (Expr->getSCEVType()) {
    case scUnknown:
      if (!isa<PHINode>(cast<SCEVUnknown>(Expr)->getValue()))
        break;
      [[fallthrough]];
    case scConstant:
    case scVScale:
    case scTruncate:
    case scZeroExtend:
    case scSignExtend:
    case scPtrToAddr:
    case scAddExpr:
    case scMulExpr:
    case scUDivExpr:
    case scAddRecExpr:
    case scUMaxExpr:
    case scSMaxExpr:
    case scUMinExpr:
    case scSMinExpr:
    case scSequentialUMinExpr:
      WorkList.push_back(Expr);
      break;
    case scCouldNotCompute:
      llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
    }
  };
  AddToWorklist(S);
 
  // Build worklist by queuing operands of N-ary expressions and phi nodes.
  for (unsigned I = 0; I != WorkList.size(); ++I) {
    const SCEV *P = WorkList[I];
    auto *UnknownS = dyn_cast<SCEVUnknown>(P);
    // If it is not a `SCEVUnknown`, just recurse into operands.
    if (!UnknownS) {
      for (const SCEV *Op : P->operands())
        AddToWorklist(Op);
      continue;
    }
    // `SCEVUnknown`'s require special treatment.
    if (PHINode *P = dyn_cast<PHINode>(UnknownS->getValue())) {
      if (!RangeRefPHIAllowedOperands(DT, P))
        continue;
      for (auto &Op : reverse(P->operands()))
        AddToWorklist(getSCEV(Op));
    }
  }
 
  if (!WorkList.empty()) {
    // Use getRangeRef to compute ranges for items in the worklist in reverse
    // order. This will force ranges for earlier operands to be computed before
    // their users in most cases.
    for (const SCEV *P : reverse(drop_begin(WorkList))) {
      getRangeRef(P, SignHint);
    }
  }
 
  return getRangeRef(S, SignHint, 0);
}
 
const APInt *ScalarEvolution::getConstantAPIntOrNull(const SCEV *S) {
  if (const auto *C = dyn_cast<SCEVConstant>(S))
    return &C->getAPInt();
  return nullptr;
}
 
/// Determine the range for a particular SCEV.  If SignHint is
/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
/// with a "cleaner" unsigned (resp. signed) representation.
const ConstantRange &ScalarEvolution::getRangeRef(
    const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
  DenseMap<const SCEV *, ConstantRange> &Cache =
      SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
                                                       : SignedRanges;
  ConstantRange::PreferredRangeType RangeType =
      SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
                                                       : ConstantRange::Signed;
 
  // See if we've computed this range already.
  auto I = Cache.find(S);
  if (I != Cache.end())
    return I->second;
 
  if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S))
    return setRange(C, SignHint, ConstantRange(C->getAPInt()));
 
  // Switch to iteratively computing the range for S, if it is part of a deeply
  // nested expression.
  if (Depth > RangeIterThreshold)
    return getRangeRefIter(S, SignHint);
 
  unsigned BitWidth = getTypeSizeInBits(S->getType());
  ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
  using OBO = OverflowingBinaryOperator;
 
  // If the value has known zeros, the maximum value will have those known zeros
  // as well.
  if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
    APInt Multiple = getNonZeroConstantMultiple(S);
    APInt Remainder = APInt::getMaxValue(BitWidth).urem(Multiple);
    if (!Remainder.isZero())
      ConservativeResult =
          ConstantRange(APInt::getMinValue(BitWidth),
                        APInt::getMaxValue(BitWidth) - Remainder + 1);
  }
  else {
    uint32_t TZ = getMinTrailingZeros(S);
    if (TZ != 0) {
      ConservativeResult = ConstantRange(
          APInt::getSignedMinValue(BitWidth),
          APInt::getSignedMaxValue(BitWidth).ashr(TZ).shl(TZ) + 1);
    }
  }
 
  switch (S->getSCEVType()) {
  case scConstant:
    llvm_unreachable("Already handled above.");
  case scVScale:
    return setRange(S, SignHint, getVScaleRange(&F, BitWidth));
  case scTruncate: {
    const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(S);
    ConstantRange X = getRangeRef(Trunc->getOperand(), SignHint, Depth + 1);
    return setRange(
        Trunc, SignHint,
        ConservativeResult.intersectWith(X.truncate(BitWidth), RangeType));
  }
  case scZeroExtend: {
    const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(S);
    ConstantRange X = getRangeRef(ZExt->getOperand(), SignHint, Depth + 1);
    return setRange(
        ZExt, SignHint,
        ConservativeResult.intersectWith(X.zeroExtend(BitWidth), RangeType));
  }
  case scSignExtend: {
    const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(S);
    ConstantRange X = getRangeRef(SExt->getOperand(), SignHint, Depth + 1);
    return setRange(
        SExt, SignHint,
        ConservativeResult.intersectWith(X.signExtend(BitWidth), RangeType));
  }
  case scPtrToAddr: {
    const SCEVCastExpr *Cast = cast<SCEVCastExpr>(S);
    ConstantRange X = getRangeRef(Cast->getOperand(), SignHint, Depth + 1);
    return setRange(Cast, SignHint, X);
  }
  case scAddExpr: {
    const SCEVAddExpr *Add = cast<SCEVAddExpr>(S);
    // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
    const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
    if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
        match(S, m_scev_URem(m_SCEV(URemLHS), m_SCEV(URemRHS), *this))) {
      ConstantRange LHSRange = getRangeRef(URemLHS, SignHint, Depth + 1);
      ConstantRange RHSRange = getRangeRef(URemRHS, SignHint, Depth + 1);
      ConservativeResult =
          ConservativeResult.intersectWith(LHSRange.urem(RHSRange), RangeType);
    }
    ConstantRange X = getRangeRef(Add->getOperand(0), SignHint, Depth + 1);
    unsigned WrapType = OBO::AnyWrap;
    if (Add->hasNoSignedWrap())
      WrapType |= OBO::NoSignedWrap;
    if (Add->hasNoUnsignedWrap())
      WrapType |= OBO::NoUnsignedWrap;
    for (const SCEV *Op : drop_begin(Add->operands()))
      X = X.addWithNoWrap(getRangeRef(Op, SignHint, Depth + 1), WrapType,
                          RangeType);
    return setRange(Add, SignHint,
                    ConservativeResult.intersectWith(X, RangeType));
  }
  case scMulExpr: {
    const SCEVMulExpr *Mul = cast<SCEVMulExpr>(S);
    ConstantRange X = getRangeRef(Mul->getOperand(0), SignHint, Depth + 1);
    for (const SCEV *Op : drop_begin(Mul->operands()))
      X = X.multiply(getRangeRef(Op, SignHint, Depth + 1));
    return setRange(Mul, SignHint,
                    ConservativeResult.intersectWith(X, RangeType));
  }
  case scUDivExpr: {
    const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(S);
    ConstantRange X = getRangeRef(UDiv->getLHS(), SignHint, Depth + 1);
    ConstantRange Y = getRangeRef(UDiv->getRHS(), SignHint, Depth + 1);
    return setRange(UDiv, SignHint,
                    ConservativeResult.intersectWith(X.udiv(Y), RangeType));
  }
  case scAddRecExpr: {
    const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(S);
    // If there's no unsigned wrap, the value will never be less than its
    // initial value.
    if (AddRec->hasNoUnsignedWrap()) {
      APInt UnsignedMinValue = getUnsignedRangeMin(AddRec->getStart());
      if (!UnsignedMinValue.isZero())
        ConservativeResult = ConservativeResult.intersectWith(
            ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), RangeType);
    }
 
    // If there's no signed wrap, and all the operands except initial value have
    // the same sign or zero, the value won't ever be:
    // 1: smaller than initial value if operands are non negative,
    // 2: bigger than initial value if operands are non positive.
    // For both cases, value can not cross signed min/max boundary.
    if (AddRec->hasNoSignedWrap()) {
      bool AllNonNeg = true;
      bool AllNonPos = true;
      for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
        if (!isKnownNonNegative(AddRec->getOperand(i)))
          AllNonNeg = false;
        if (!isKnownNonPositive(AddRec->getOperand(i)))
          AllNonPos = false;
      }
      if (AllNonNeg)
        ConservativeResult = ConservativeResult.intersectWith(
            ConstantRange::getNonEmpty(getSignedRangeMin(AddRec->getStart()),
                                       APInt::getSignedMinValue(BitWidth)),
            RangeType);
      else if (AllNonPos)
        ConservativeResult = ConservativeResult.intersectWith(
            ConstantRange::getNonEmpty(APInt::getSignedMinValue(BitWidth),
                                       getSignedRangeMax(AddRec->getStart()) +
                                           1),
            RangeType);
    }
 
    // TODO: non-affine addrec
    if (AddRec->isAffine()) {
      const SCEV *MaxBEScev =
          getConstantMaxBackedgeTakenCount(AddRec->getLoop());
      if (!isa<SCEVCouldNotCompute>(MaxBEScev)) {
        APInt MaxBECount = cast<SCEVConstant>(MaxBEScev)->getAPInt();
 
        // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
        // MaxBECount's active bits are all <= AddRec's bit width.
        if (MaxBECount.getBitWidth() > BitWidth &&
            MaxBECount.getActiveBits() <= BitWidth)
          MaxBECount = MaxBECount.trunc(BitWidth);
        else if (MaxBECount.getBitWidth() < BitWidth)
          MaxBECount = MaxBECount.zext(BitWidth);
 
        if (MaxBECount.getBitWidth() == BitWidth) {
          auto [RangeFromAffine, Flags] = getRangeForAffineAR(
              AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount);
          ConservativeResult =
              ConservativeResult.intersectWith(RangeFromAffine, RangeType);
          const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
 
          auto RangeFromFactoring = getRangeViaFactoring(
              AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount);
          ConservativeResult =
              ConservativeResult.intersectWith(RangeFromFactoring, RangeType);
        }
      }
 
      // Now try symbolic BE count and more powerful methods.
      if (UseExpensiveRangeSharpening) {
        const SCEV *SymbolicMaxBECount =
            getSymbolicMaxBackedgeTakenCount(AddRec->getLoop());
        if (!isa<SCEVCouldNotCompute>(SymbolicMaxBECount) &&
            getTypeSizeInBits(MaxBEScev->getType()) <= BitWidth &&
            AddRec->hasNoSelfWrap()) {
          auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
              AddRec, SymbolicMaxBECount, BitWidth, SignHint);
          ConservativeResult =
              ConservativeResult.intersectWith(RangeFromAffineNew, RangeType);
        }
      }
    }
 
    return setRange(AddRec, SignHint, std::move(ConservativeResult));
  }
  case scUMaxExpr:
  case scSMaxExpr:
  case scUMinExpr:
  case scSMinExpr:
  case scSequentialUMinExpr: {
    Intrinsic::ID ID;
    switch (S->getSCEVType()) {
    case scUMaxExpr:
      ID = Intrinsic::umax;
      break;
    case scSMaxExpr:
      ID = Intrinsic::smax;
      break;
    case scUMinExpr:
    case scSequentialUMinExpr:
      ID = Intrinsic::umin;
      break;
    case scSMinExpr:
      ID = Intrinsic::smin;
      break;
    default:
      llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
    }
 
    const auto *NAry = cast<SCEVNAryExpr>(S);
    ConstantRange X = getRangeRef(NAry->getOperand(0), SignHint, Depth + 1);
    for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
      X = X.intrinsic(
          ID, {X, getRangeRef(NAry->getOperand(i), SignHint, Depth + 1)});
    return setRange(S, SignHint,
                    ConservativeResult.intersectWith(X, RangeType));
  }
  case scUnknown: {
    const SCEVUnknown *U = cast<SCEVUnknown>(S);
    Value *V = U->getValue();
 
    // Check if the IR explicitly contains !range metadata.
    std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
    if (MDRange)
      ConservativeResult =
          ConservativeResult.intersectWith(*MDRange, RangeType);
 
    // Use facts about recurrences in the underlying IR.  Note that add
    // recurrences are AddRecExprs and thus don't hit this path.  This
    // primarily handles shift recurrences.
    auto CR = getRangeForUnknownRecurrence(U);
    ConservativeResult = ConservativeResult.intersectWith(CR);
 
    // See if ValueTracking can give us a useful range.
    const DataLayout &DL = getDataLayout();
    KnownBits Known = computeKnownBits(V, DL, &AC, nullptr, &DT);
    if (Known.getBitWidth() != BitWidth)
      Known = Known.zextOrTrunc(BitWidth);
 
    // ValueTracking may be able to compute a tighter result for the number of
    // sign bits than for the value of those sign bits.
    unsigned NS = ComputeNumSignBits(V, DL, &AC, nullptr, &DT);
    if (U->getType()->isPointerTy()) {
      // If the pointer size is larger than the index size type, this can cause
      // NS to be larger than BitWidth. So compensate for this.
      unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
      int ptrIdxDiff = ptrSize - BitWidth;
      if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
        NS -= ptrIdxDiff;
    }
 
    if (NS > 1) {
      // If we know any of the sign bits, we know all of the sign bits.
      if (!Known.Zero.getHiBits(NS).isZero())
        Known.Zero.setHighBits(NS);
      if (!Known.One.getHiBits(NS).isZero())
        Known.One.setHighBits(NS);
    }
 
    if (Known.getMinValue() != Known.getMaxValue() + 1)
      ConservativeResult = ConservativeResult.intersectWith(
          ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
          RangeType);
    if (NS > 1)
      ConservativeResult = ConservativeResult.intersectWith(
          ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1),
                        APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1),
          RangeType);
 
    if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
      // Strengthen the range if the underlying IR value is a
      // global/alloca/heap allocation using the size of the object.
      bool CanBeNull;
      uint64_t DerefBytes = V->getPointerDereferenceableBytes(
          DL, CanBeNull, /*CanBeFreed=*/nullptr);
      if (DerefBytes > 1 && isUIntN(BitWidth, DerefBytes)) {
        // The highest address the object can start is DerefBytes bytes before
        // the end (unsigned max value). If this value is not a multiple of the
        // alignment, the last possible start value is the next lowest multiple
        // of the alignment. Note: The computations below cannot overflow,
        // because if they would there's no possible start address for the
        // object.
        APInt MaxVal =
            APInt::getMaxValue(BitWidth) - APInt(BitWidth, DerefBytes);
        uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
        uint64_t Rem = MaxVal.urem(Align);
        MaxVal -= APInt(BitWidth, Rem);
        APInt MinVal = APInt::getZero(BitWidth);
        if (llvm::isKnownNonZero(V, DL))
          MinVal = Align;
        ConservativeResult = ConservativeResult.intersectWith(
            ConstantRange::getNonEmpty(MinVal, MaxVal + 1), RangeType);
      }
    }
 
    // A range of Phi is a subset of union of all ranges of its input.
    if (PHINode *Phi = dyn_cast<PHINode>(V)) {
      // SCEVExpander sometimes creates SCEVUnknowns that are secretly
      // AddRecs; return the range for the corresponding AddRec.
      if (auto *AR = dyn_cast<SCEVAddRecExpr>(getSCEV(V)))
        return getRangeRef(AR, SignHint, Depth + 1);
 
      // Make sure that we do not run over cycled Phis.
      if (RangeRefPHIAllowedOperands(DT, Phi)) {
        ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
 
        for (const auto &Op : Phi->operands()) {
          auto OpRange = getRangeRef(getSCEV(Op), SignHint, Depth + 1);
          RangeFromOps = RangeFromOps.unionWith(OpRange);
          // No point to continue if we already have a full set.
          if (RangeFromOps.isFullSet())
            break;
        }
        ConservativeResult =
            ConservativeResult.intersectWith(RangeFromOps, RangeType);
      }
    }
 
    // vscale can't be equal to zero
    if (const auto *II = dyn_cast<IntrinsicInst>(V))
      if (II->getIntrinsicID() == Intrinsic::vscale) {
        ConstantRange Disallowed = APInt::getZero(BitWidth);
        ConservativeResult = ConservativeResult.difference(Disallowed);
      }
 
    return setRange(U, SignHint, std::move(ConservativeResult));
  }
  case scCouldNotCompute:
    llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
  }
 
  return setRange(S, SignHint, std::move(ConservativeResult));
}
 
// Given a StartRange, Step and MaxBECount for an expression compute a range of
// values that the expression can take. Initially, the expression has a value
// from StartRange and then is changed by Step up to MaxBECount times. Signed
// argument defines if we treat Step as signed or unsigned. The second return
// value indicates that no wrapping occurred.
static std::pair<ConstantRange, bool>
getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange,
                          const APInt &MaxBECount, bool Signed) {
  unsigned BitWidth = Step.getBitWidth();
  assert(BitWidth == StartRange.getBitWidth() &&
         BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
  // If either Step or MaxBECount is 0, then the expression won't change, and we
  // just need to return the initial range.
  if (Step == 0 || MaxBECount == 0)
    return {StartRange, true};
 
  // If we don't know anything about the initial value (i.e. StartRange is
  // FullRange), then we don't know anything about the final range either.
  // Return FullRange.
  if (StartRange.isFullSet())
    return {ConstantRange::getFull(BitWidth), false};
 
  // If Step is signed and negative, then we use its absolute value, but we also
  // note that we're moving in the opposite direction.
  bool Descending = Signed && Step.isNegative();
 
  if (Signed)
    // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
    // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
    // This equations hold true due to the well-defined wrap-around behavior of
    // APInt.
    Step = Step.abs();
 
  // Check if Offset is more than full span of BitWidth. If it is, the
  // expression is guaranteed to overflow.
  if (APInt::getMaxValue(StartRange.getBitWidth()).udiv(Step).ult(MaxBECount))
    return {ConstantRange::getFull(BitWidth), false};
 
  // Offset is by how much the expression can change. Checks above guarantee no
  // overflow here.
  APInt Offset = Step * MaxBECount;
 
  // Minimum value of the final range will match the minimal value of StartRange
  // if the expression is increasing and will be decreased by Offset otherwise.
  // Maximum value of the final range will match the maximal value of StartRange
  // if the expression is decreasing and will be increased by Offset otherwise.
  APInt StartLower = StartRange.getLower();
  APInt StartUpper = StartRange.getUpper() - 1;
  bool Overflow;
  APInt MovedBoundary;
  if (Signed) {
    // This does not use sadd_ov, as we want to check overflow for a signed
    // start with an unsigned offset.
    if (Descending) {
      MovedBoundary = StartLower - std::move(Offset);
      Overflow = MovedBoundary.sgt(StartLower) || StartRange.isSignWrappedSet();
    } else {
      MovedBoundary = StartUpper + std::move(Offset);
      Overflow = MovedBoundary.slt(StartUpper) || StartRange.isSignWrappedSet();
    }
  } else {
    MovedBoundary = StartUpper.uadd_ov(std::move(Offset), Overflow);
    Overflow |= StartRange.isWrappedSet();
  }
 
  // It's possible that the new minimum/maximum value will fall into the initial
  // range (due to wrap around). This means that the expression can take any
  // value in this bitwidth, and we have to return full range.
  if (StartRange.contains(MovedBoundary))
    return {ConstantRange::getFull(BitWidth), false};
 
  APInt NewLower =
      Descending ? std::move(MovedBoundary) : std::move(StartLower);
  APInt NewUpper =
      Descending ? std::move(StartUpper) : std::move(MovedBoundary);
  NewUpper += 1;
 
  // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
  return {ConstantRange::getNonEmpty(std::move(NewLower), std::move(NewUpper)),
          !Overflow};
}
 
std::pair<ConstantRange, SCEV::NoWrapFlags>
ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
                                     const APInt &MaxBECount) {
  assert(getTypeSizeInBits(Start->getType()) ==
             getTypeSizeInBits(Step->getType()) &&
         getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
         "mismatched bit widths");
 
  // First, consider step signed.
  ConstantRange StartSRange = getSignedRange(Start);
  ConstantRange StepSRange = getSignedRange(Step);
 
  // If Step can be both positive and negative, we need to find ranges for the
  // maximum absolute step values in both directions and union them.
  auto [SR1, NSW1] = getRangeForAffineARHelper(
      StepSRange.getSignedMin(), StartSRange, MaxBECount, /*Signed=*/true);
  auto [SR2, NSW2] = getRangeForAffineARHelper(StepSRange.getSignedMax(),
                                               StartSRange, MaxBECount,
                                               /*Signed=*/true);
  ConstantRange SR = SR1.unionWith(SR2);
 
  // Next, consider step unsigned.
  auto [UR, NUW] = getRangeForAffineARHelper(
      getUnsignedRangeMax(Step), getUnsignedRange(Start), MaxBECount,
      /*Signed=*/false);
 
  SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
  if (NUW)
    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
  if (NSW1 && NSW2)
    Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
 
  // Finally, intersect signed and unsigned ranges.
  return {SR.intersectWith(UR, ConstantRange::Smallest), Flags};
}
 
ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
    const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
    ScalarEvolution::RangeSignHint SignHint) {
  assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
  assert(AddRec->hasNoSelfWrap() &&
         "This only works for non-self-wrapping AddRecs!");
  const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
  const SCEV *Step = AddRec->getStepRecurrence(*this);
  // Only deal with constant step to save compile time.
  if (!isa<SCEVConstant>(Step))
    return ConstantRange::getFull(BitWidth);
  // Let's make sure that we can prove that we do not self-wrap during
  // MaxBECount iterations. We need this because MaxBECount is a maximum
  // iteration count estimate, and we might infer nw from some exit for which we
  // do not know max exit count (or any other side reasoning).
  // TODO: Turn into assert at some point.
  if (getTypeSizeInBits(MaxBECount->getType()) >
      getTypeSizeInBits(AddRec->getType()))
    return ConstantRange::getFull(BitWidth);
  MaxBECount = getNoopOrZeroExtend(MaxBECount, AddRec->getType());
  const SCEV *RangeWidth = getMinusOne(AddRec->getType());
  const SCEV *StepAbs = getUMinExpr(Step, getNegativeSCEV(Step));
  const SCEV *MaxItersWithoutWrap = getUDivExpr(RangeWidth, StepAbs);
  if (!isKnownPredicateViaConstantRanges(ICmpInst::ICMP_ULE, MaxBECount,
                                         MaxItersWithoutWrap))
    return ConstantRange::getFull(BitWidth);
 
  ICmpInst::Predicate LEPred =
      IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
  ICmpInst::Predicate GEPred =
      IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
  const SCEV *End = AddRec->evaluateAtIteration(MaxBECount, *this);
 
  // We know that there is no self-wrap. Let's take Start and End values and
  // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
  // the iteration. They either lie inside the range [Min(Start, End),
  // Max(Start, End)] or outside it:
  //
  // Case 1:   RangeMin    ...    Start V1 ... VN End ...           RangeMax;
  // Case 2:   RangeMin Vk ... V1 Start    ...    End Vn ... Vk + 1 RangeMax;
  //
  // No self wrap flag guarantees that the intermediate values cannot be BOTH
  // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
  // knowledge, let's try to prove that we are dealing with Case 1. It is so if
  // Start <= End and step is positive, or Start >= End and step is negative.
  const SCEV *Start = applyLoopGuards(AddRec->getStart(), AddRec->getLoop());
  ConstantRange StartRange = getRangeRef(Start, SignHint);
  ConstantRange EndRange = getRangeRef(End, SignHint);
  ConstantRange RangeBetween = StartRange.unionWith(EndRange);
  // If they already cover full iteration space, we will know nothing useful
  // even if we prove what we want to prove.
  if (RangeBetween.isFullSet())
    return RangeBetween;
  // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
  bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
                               : RangeBetween.isWrappedSet();
  if (IsWrappedSet)
    return ConstantRange::getFull(BitWidth);
 
  if (isKnownPositive(Step) &&
      isKnownPredicateViaConstantRanges(LEPred, Start, End))
    return RangeBetween;
  if (isKnownNegative(Step) &&
           isKnownPredicateViaConstantRanges(GEPred, Start, End))
    return RangeBetween;
  return ConstantRange::getFull(BitWidth);
}
 
ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
                                                    const SCEV *Step,
                                                    const APInt &MaxBECount) {
  //    RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
  // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
 
  unsigned BitWidth = MaxBECount.getBitWidth();
  assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
         getTypeSizeInBits(Step->getType()) == BitWidth &&
         "mismatched bit widths");
 
  struct SelectPattern {
    Value *Condition = nullptr;
    APInt TrueValue;
    APInt FalseValue;
 
    explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
                           const SCEV *S) {
      std::optional<unsigned> CastOp;
      APInt Offset(BitWidth, 0);
 
      assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
             "Should be!");
 
      // Peel off a constant offset. In the future we could consider being
      // smarter here and handle {Start+Step,+,Step} too.
      const APInt *Off;
      if (match(S, m_scev_Add(m_scev_APInt(Off), m_SCEV(S))))
        Offset = *Off;
 
      // Peel off a cast operation
      if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(S)) {
        CastOp = SCast->getSCEVType();
        S = SCast->getOperand();
      }
 
      using namespace llvm::PatternMatch;
 
      auto *SU = dyn_cast<SCEVUnknown>(S);
      const APInt *TrueVal, *FalseVal;
      if (!SU ||
          !match(SU->getValue(), m_Select(m_Value(Condition), m_APInt(TrueVal),
                                          m_APInt(FalseVal)))) {
        Condition = nullptr;
        return;
      }
 
      TrueValue = *TrueVal;
      FalseValue = *FalseVal;
 
      // Re-apply the cast we peeled off earlier
      if (CastOp)
        switch (*CastOp) {
        default:
          llvm_unreachable("Unknown SCEV cast type!");
 
        case scTruncate:
          TrueValue = TrueValue.trunc(BitWidth);
          FalseValue = FalseValue.trunc(BitWidth);
          break;
        case scZeroExtend:
          TrueValue = TrueValue.zext(BitWidth);
          FalseValue = FalseValue.zext(BitWidth);
          break;
        case scSignExtend:
          TrueValue = TrueValue.sext(BitWidth);
          FalseValue = FalseValue.sext(BitWidth);
          break;
        }
 
      // Re-apply the constant offset we peeled off earlier
      TrueValue += Offset;
      FalseValue += Offset;
    }
 
    bool isRecognized() { return Condition != nullptr; }
  };
 
  SelectPattern StartPattern(*this, BitWidth, Start);
  if (!StartPattern.isRecognized())
    return ConstantRange::getFull(BitWidth);
 
  SelectPattern StepPattern(*this, BitWidth, Step);
  if (!StepPattern.isRecognized())
    return ConstantRange::getFull(BitWidth);
 
  if (StartPattern.Condition != StepPattern.Condition) {
    // We don't handle this case today; but we could, by considering four
    // possibilities below instead of two. I'm not sure if there are cases where
    // that will help over what getRange already does, though.
    return ConstantRange::getFull(BitWidth);
  }
 
  // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
  // construct arbitrary general SCEV expressions here.  This function is called
  // from deep in the call stack, and calling getSCEV (on a sext instruction,
  // say) can end up caching a suboptimal value.
 
  // FIXME: without the explicit `this` receiver below, MSVC errors out with
  // C2352 and C2512 (otherwise it isn't needed).
 
  const SCEV *TrueStart = this->getConstant(StartPattern.TrueValue);
  const SCEV *TrueStep = this->getConstant(StepPattern.TrueValue);
  const SCEV *FalseStart = this->getConstant(StartPattern.FalseValue);
  const SCEV *FalseStep = this->getConstant(StepPattern.FalseValue);
 
  ConstantRange TrueRange =
      this->getRangeForAffineAR(TrueStart, TrueStep, MaxBECount).first;
  ConstantRange FalseRange =
      this->getRangeForAffineAR(FalseStart, FalseStep, MaxBECount).first;
 
  return TrueRange.unionWith(FalseRange);
}
 
SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
  if (isa<ConstantExpr>(V)) return SCEV::FlagAnyWrap;
  const BinaryOperator *BinOp = cast<BinaryOperator>(V);
 
  // Return early if there are no flags to propagate to the SCEV.
  SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
  if (auto *PDI = dyn_cast<PossiblyDisjointInst>(BinOp);
      PDI && PDI->isDisjoint()) {
    Flags = ScalarEvolution::setFlags(SCEV::FlagNUW, SCEV::FlagNSW);
  } else {
    if (BinOp->hasNoUnsignedWrap())
      Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNUW);
    if (BinOp->hasNoSignedWrap())
      Flags = ScalarEvolution::setFlags(Flags, SCEV::FlagNSW);
  }
  if (Flags == SCEV::FlagAnyWrap)
    return SCEV::FlagAnyWrap;
 
  return isSCEVExprNeverPoison(BinOp) ? Flags : SCEV::FlagAnyWrap;
}
 
const Instruction *
ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
  if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S))
    return &*AddRec->getLoop()->getHeader()->begin();
  if (auto *U = dyn_cast<SCEVUnknown>(S))
    if (auto *I = dyn_cast<Instruction>(U->getValue()))
      return I;
  return nullptr;
}
 
const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
                                                          bool &Precise) {
  Precise = true;
  // Do a bounded search of the def relation of the requested SCEVs.
  SmallPtrSet<const SCEV *, 16> Visited;
  SmallVector<SCEVUse> Worklist;
  auto pushOp = [&](const SCEV *S) {
    if (!Visited.insert(S).second)
      return;
    // Threshold of 30 here is arbitrary.
    if (Visited.size() > 30) {
      Precise = false;
      return;
    }
    Worklist.push_back(S);
  };
 
  for (SCEVUse S : Ops)
    pushOp(S);
 
  const Instruction *Bound = nullptr;
  while (!Worklist.empty()) {
    SCEVUse S = Worklist.pop_back_val();
    if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
      if (!Bound || DT.dominates(Bound, DefI))
        Bound = DefI;
    } else {
      for (SCEVUse Op : S->operands())
        pushOp(Op);
    }
  }
  return Bound ? Bound : &*F.getEntryBlock().begin();
}
 
const Instruction *
ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
  bool Discard;
  return getDefiningScopeBound(Ops, Discard);
}
 
bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
                                                        const Instruction *B) {
  if (A->getParent() == B->getParent() &&
      isGuaranteedToTransferExecutionToSuccessor(A->getIterator(),
                                                 B->getIterator()))
    return true;
 
  auto *BLoop = LI.getLoopFor(B->getParent());
  if (BLoop && BLoop->getHeader() == B->getParent() &&
      BLoop->getLoopPreheader() == A->getParent() &&
      isGuaranteedToTransferExecutionToSuccessor(A->getIterator(),
                                                 A->getParent()->end()) &&
      isGuaranteedToTransferExecutionToSuccessor(B->getParent()->begin(),
                                                 B->getIterator()))
    return true;
  return false;
}
 
bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
  SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
  visitAll(Op, PC);
  return PC.MaybePoison.empty();
}
 
bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
  return !SCEVExprContains(Op, [this](const SCEV *S) {
    const SCEV *Op1;
    bool M = match(S, m_scev_UDiv(m_SCEV(), m_SCEV(Op1)));
    // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
    // is a non-zero constant, we have to assume the UDiv may be UB.
    return M && (!isKnownNonZero(Op1) || !isGuaranteedNotToBePoison(Op1));
  });
}
 
bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
  // Only proceed if we can prove that I does not yield poison.
  if (!programUndefinedIfPoison(I))
    return false;
 
  // At this point we know that if I is executed, then it does not wrap
  // according to at least one of NSW or NUW. If I is not executed, then we do
  // not know if the calculation that I represents would wrap. Multiple
  // instructions can map to the same SCEV. If we apply NSW or NUW from I to
  // the SCEV, we must guarantee no wrapping for that SCEV also when it is
  // derived from other instructions that map to the same SCEV. We cannot make
  // that guarantee for cases where I is not executed. So we need to find a
  // upper bound on the defining scope for the SCEV, and prove that I is
  // executed every time we enter that scope.  When the bounding scope is a
  // loop (the common case), this is equivalent to proving I executes on every
  // iteration of that loop.
  SmallVector<SCEVUse> SCEVOps;
  for (const Use &Op : I->operands()) {
    // I could be an extractvalue from a call to an overflow intrinsic.
    // TODO: We can do better here in some cases.
    if (isSCEVable(Op->getType()))
      SCEVOps.push_back(getSCEV(Op));
  }
  auto *DefI = getDefiningScopeBound(SCEVOps);
  return isGuaranteedToTransferExecutionTo(DefI, I);
}
 
bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
  // If we know that \c I can never be poison period, then that's enough.
  if (isSCEVExprNeverPoison(I))
    return true;
 
  // If the loop only has one exit, then we know that, if the loop is entered,
  // any instruction dominating that exit will be executed. If any such
  // instruction would result in UB, the addrec cannot be poison.
  //
  // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
  // also handles uses outside the loop header (they just need to dominate the
  // single exit).
 
  auto *ExitingBB = L->getExitingBlock();
  if (!ExitingBB || !loopHasNoAbnormalExits(L))
    return false;
 
  SmallPtrSet<const Value *, 16> KnownPoison;
  SmallVector<const Instruction *, 8> Worklist;
 
  // We start by assuming \c I, the post-inc add recurrence, is poison.  Only
  // things that are known to be poison under that assumption go on the
  // Worklist.
  KnownPoison.insert(I);
  Worklist.push_back(I);
 
  while (!Worklist.empty()) {
    const Instruction *Poison = Worklist.pop_back_val();
 
    for (const Use &U : Poison->uses()) {
      const Instruction *PoisonUser = cast<Instruction>(U.getUser());
      if (mustTriggerUB(PoisonUser, KnownPoison) &&
          DT.dominates(PoisonUser->getParent(), ExitingBB))
        return true;
 
      if (propagatesPoison(U) && L->contains(PoisonUser))
        if (KnownPoison.insert(PoisonUser).second)
          Worklist.push_back(PoisonUser);
    }
  }
 
  return false;
}
 
ScalarEvolution::LoopProperties
ScalarEvolution::getLoopProperties(const Loop *L) {
  using LoopProperties = ScalarEvolution::LoopProperties;
 
  auto Itr = LoopPropertiesCache.find(L);
  if (Itr == LoopPropertiesCache.end()) {
    auto HasSideEffects = [](Instruction *I) {
      if (auto *SI = dyn_cast<StoreInst>(I))
        return !SI->isSimple();
 
      if (I->mayThrow())
        return true;
 
      // Non-volatile memset / memcpy do not count as side-effect for forward
      // progress.
      if (isa<MemIntrinsic>(I) && !I->isVolatile())
        return false;
 
      return I->mayWriteToMemory();
    };
 
    LoopProperties LP = {/* HasNoAbnormalExits */ true,
                         /*HasNoSideEffects*/ true};
 
    for (auto *BB : L->getBlocks())
      for (auto &I : *BB) {
        if (!isGuaranteedToTransferExecutionToSuccessor(&I))
          LP.HasNoAbnormalExits = false;
        if (HasSideEffects(&I))
          LP.HasNoSideEffects = false;
        if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
          break; // We're already as pessimistic as we can get.
      }
 
    auto InsertPair = LoopPropertiesCache.insert({L, LP});
    assert(InsertPair.second && "We just checked!");
    Itr = InsertPair.first;
  }
 
  return Itr->second;
}
 
bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
  // A mustprogress loop without side effects must be finite.
  // TODO: The check used here is very conservative.  It's only *specific*
  // side effects which are well defined in infinite loops.
  return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
}
 
const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
  // Worklist item with a Value and a bool indicating whether all operands have
  // been visited already.
  using PointerTy = PointerIntPair<Value *, 1, bool>;
  SmallVector<PointerTy> Stack;
 
  Stack.emplace_back(V, false);
  while (!Stack.empty()) {
    auto E = Stack.back();
    Value *CurV = E.getPointer();
 
    if (getExistingSCEV(CurV)) {
      Stack.pop_back();
      continue;
    }
 
    SmallVector<Value *> Ops;
    const SCEV *CreatedSCEV = nullptr;
    // If all operands have been visited already, create the SCEV.
    if (E.getInt()) {
      CreatedSCEV = createSCEV(CurV);
    } else {
      // Otherwise get the operands we need to create SCEV's for before creating
      // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
      // just use it.
      CreatedSCEV = getOperandsToCreate(CurV, Ops);
    }
 
    if (CreatedSCEV) {
      insertValueToMap(CurV, CreatedSCEV);
      Stack.pop_back();
    } else {
      Stack.back().setInt(true);
      // Queue its operands which need to be constructed.
      for (Value *Op : Ops)
        Stack.emplace_back(Op, false);
    }
  }
 
  return getExistingSCEV(V);
}
 
const SCEV *
ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
  if (!isSCEVable(V->getType()))
    return getUnknown(V);
 
  if (Instruction *I = dyn_cast<Instruction>(V)) {
    // Don't attempt to analyze instructions in blocks that aren't
    // reachable. Such instructions don't matter, and they aren't required
    // to obey basic rules for definitions dominating uses which this
    // analysis depends on.
    if (!DT.isReachableFromEntry(I->getParent()))
      return getUnknown(PoisonValue::get(V->getType()));
  } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
    return getConstant(CI);
  else if (isa<GlobalAlias>(V))
    return getUnknown(V);
  else if (!isa<ConstantExpr>(V))
    return getUnknown(V);
 
  Operator *U = cast<Operator>(V);
  if (auto BO =
          MatchBinaryOp(U, getDataLayout(), AC, DT, dyn_cast<Instruction>(V))) {
    bool IsConstArg = isa<ConstantInt>(BO->RHS);
    switch (BO->Opcode) {
    case Instruction::Add:
    case Instruction::Mul: {
      // For additions and multiplications, traverse add/mul chains for which we
      // can potentially create a single SCEV, to reduce the number of
      // get{Add,Mul}Expr calls.
      do {
        if (BO->Op) {
          if (BO->Op != V && getExistingSCEV(BO->Op)) {
            Ops.push_back(BO->Op);
            break;
          }
        }
        Ops.push_back(BO->RHS);
        auto NewBO = MatchBinaryOp(BO->LHS, getDataLayout(), AC, DT,
                                   dyn_cast<Instruction>(V));
        if (!NewBO ||
            (BO->Opcode == Instruction::Add &&
             (NewBO->Opcode != Instruction::Add &&
              NewBO->Opcode != Instruction::Sub)) ||
            (BO->Opcode == Instruction::Mul &&
             NewBO->Opcode != Instruction::Mul)) {
          Ops.push_back(BO->LHS);
          break;
        }
        // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
        // requires a SCEV for the LHS.
        if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
          auto *I = dyn_cast<Instruction>(BO->Op);
          if (I && programUndefinedIfPoison(I)) {
            Ops.push_back(BO->LHS);
            break;
          }
        }
        BO = NewBO;
      } while (true);
      return nullptr;
    }
    case Instruction::Sub:
    case Instruction::UDiv:
    case Instruction::URem:
      break;
    case Instruction::AShr:
    case Instruction::Shl:
    case Instruction::Xor:
      if (!IsConstArg)
        return nullptr;
      break;
    case Instruction::And:
    case Instruction::Or:
      if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(1))
        return nullptr;
      break;
    case Instruction::LShr:
      return getUnknown(V);
    default:
      llvm_unreachable("Unhandled binop");
      break;
    }
 
    Ops.push_back(BO->LHS);
    Ops.push_back(BO->RHS);
    return nullptr;
  }
 
  switch (U->getOpcode()) {
  case Instruction::Trunc:
  case Instruction::ZExt:
  case Instruction::SExt:
  case Instruction::PtrToAddr:
  case Instruction::PtrToInt:
    Ops.push_back(U->getOperand(0));
    return nullptr;
 
  case Instruction::BitCast:
    if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType())) {
      Ops.push_back(U->getOperand(0));
      return nullptr;
    }
    return getUnknown(V);
 
  case Instruction::SDiv:
  case Instruction::SRem:
    Ops.push_back(U->getOperand(0));
    Ops.push_back(U->getOperand(1));
    return nullptr;
 
  case Instruction::GetElementPtr:
    assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
           "GEP source element type must be sized");
    llvm::append_range(Ops, U->operands());
    return nullptr;
 
  case Instruction::IntToPtr:
    return getUnknown(V);
 
  case Instruction::PHI:
    // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
    // relevant nodes for each of them.
    //
    // The first is just to call simplifyInstruction, and get something back
    // that isn't a PHI.
    if (Value *V = simplifyInstruction(
            cast<PHINode>(U),
            {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
             /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
      assert(V);
      Ops.push_back(V);
      return nullptr;
    }
    // The second is createNodeForPHIWithIdenticalOperands: this looks for
    // operands which all perform the same operation, but haven't been
    // CSE'ed for whatever reason.
    if (BinaryOperator *BO = getCommonInstForPHI(cast<PHINode>(U))) {
      assert(BO);
      Ops.push_back(BO);
      return nullptr;
    }
    // The third is createNodeFromSelectLikePHI; this takes a PHI which
    // is equivalent to a select, and analyzes it like a select.
    {
      Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
      if (getOperandsForSelectLikePHI(DT, cast<PHINode>(U), Cond, LHS, RHS)) {
        assert(Cond);
        assert(LHS);
        assert(RHS);
        if (auto *CondICmp = dyn_cast<ICmpInst>(Cond)) {
          Ops.push_back(CondICmp->getOperand(0));
          Ops.push_back(CondICmp->getOperand(1));
        }
        Ops.push_back(Cond);
        Ops.push_back(LHS);
        Ops.push_back(RHS);
        return nullptr;
      }
    }
    // The fourth way is createAddRecFromPHI. It's complicated to handle here,
    // so just construct it recursively.
    //
    // In addition to getNodeForPHI, also construct nodes which might be needed
    // by getRangeRef.
    if (RangeRefPHIAllowedOperands(DT, cast<PHINode>(U))) {
      for (Value *V : cast<PHINode>(U)->operands())
        Ops.push_back(V);
      return nullptr;
    }
    return nullptr;
 
  case Instruction::Select: {
    // Check if U is a select that can be simplified to a SCEVUnknown.
    auto CanSimplifyToUnknown = [this, U]() {
      if (U->getType()->isIntegerTy(1) || isa<ConstantInt>(U->getOperand(0)))
        return false;
 
      auto *ICI = dyn_cast<ICmpInst>(U->getOperand(0));
      if (!ICI)
        return false;
      Value *LHS = ICI->getOperand(0);
      Value *RHS = ICI->getOperand(1);
      if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
          ICI->getPredicate() == CmpInst::ICMP_NE) {
        if (!(isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isZero()))
          return true;
      } else if (getTypeSizeInBits(LHS->getType()) >
                 getTypeSizeInBits(U->getType()))
        return true;
      return false;
    };
    if (CanSimplifyToUnknown())
      return getUnknown(U);
 
    llvm::append_range(Ops, U->operands());
    return nullptr;
    break;
  }
  case Instruction::Call:
  case Instruction::Invoke:
    if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand()) {
      Ops.push_back(RV);
      return nullptr;
    }
 
    if (auto *II = dyn_cast<IntrinsicInst>(U)) {
      switch (II->getIntrinsicID()) {
      case Intrinsic::abs:
        Ops.push_back(II->getArgOperand(0));
        return nullptr;
      case Intrinsic::umax:
      case Intrinsic::umin:
      case Intrinsic::smax:
      case Intrinsic::smin:
      case Intrinsic::usub_sat:
      case Intrinsic::uadd_sat:
        Ops.push_back(II->getArgOperand(0));
        Ops.push_back(II->getArgOperand(1));
        return nullptr;
      case Intrinsic::start_loop_iterations:
      case Intrinsic::annotation:
      case Intrinsic::ptr_annotation:
        Ops.push_back(II->getArgOperand(0));
        return nullptr;
      default:
        break;
      }
    }
    break;
  }
 
  return nullptr;
}
 
const SCEV *ScalarEvolution::createSCEV(Value *V) {
  if (!isSCEVable(V->getType()))
    return getUnknown(V);
 
  if (Instruction *I = dyn_cast<Instruction>(V)) {
    // Don't attempt to analyze instructions in blocks that aren't
    // reachable. Such instructions don't matter, and they aren't required
    // to obey basic rules for definitions dominating uses which this
    // analysis depends on.
    if (!DT.isReachableFromEntry(I->getParent()))
      return getUnknown(PoisonValue::get(V->getType()));
  } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
    return getConstant(CI);
  else if (isa<GlobalAlias>(V))
    return getUnknown(V);
  else if (!isa<ConstantExpr>(V))
    return getUnknown(V);
 
  const SCEV *LHS;
  const SCEV *RHS;
 
  Operator *U = cast<Operator>(V);
  if (auto BO =
          MatchBinaryOp(U, getDataLayout(), AC, DT, dyn_cast<Instruction>(V))) {
    switch (BO->Opcode) {
    case Instruction::Add: {
      // The simple thing to do would be to just call getSCEV on both operands
      // and call getAddExpr with the result. However if we're looking at a
      // bunch of things all added together, this can be quite inefficient,
      // because it leads to N-1 getAddExpr calls for N ultimate operands.
      // Instead, gather up all the operands and make a single getAddExpr call.
      // LLVM IR canonical form means we need only traverse the left operands.
      SmallVector<SCEVUse, 4> AddOps;
      do {
        if (BO->Op) {
          if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
            AddOps.push_back(OpSCEV);
            break;
          }
 
          // If a NUW or NSW flag can be applied to the SCEV for this
          // addition, then compute the SCEV for this addition by itself
          // with a separate call to getAddExpr. We need to do that
          // instead of pushing the operands of the addition onto AddOps,
          // since the flags are only known to apply to this particular
          // addition - they may not apply to other additions that can be
          // formed with operands from AddOps.
          const SCEV *RHS = getSCEV(BO->RHS);
          SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op);
          if (Flags != SCEV::FlagAnyWrap) {
            const SCEV *LHS = getSCEV(BO->LHS);
            if (BO->Opcode == Instruction::Sub)
              AddOps.push_back(getMinusSCEV(LHS, RHS, Flags));
            else
              AddOps.push_back(getAddExpr(LHS, RHS, Flags));
            break;
          }
        }
 
        if (BO->Opcode == Instruction::Sub)
          AddOps.push_back(getNegativeSCEV(getSCEV(BO->RHS)));
        else
          AddOps.push_back(getSCEV(BO->RHS));
 
        auto NewBO = MatchBinaryOp(BO->LHS, getDataLayout(), AC, DT,
                                   dyn_cast<Instruction>(V));
        if (!NewBO || (NewBO->Opcode != Instruction::Add &&
                       NewBO->Opcode != Instruction::Sub)) {
          AddOps.push_back(getSCEV(BO->LHS));
          break;
        }
        BO = NewBO;
      } while (true);
 
      return getAddExpr(AddOps);
    }
 
    case Instruction::Mul: {
      SmallVector<SCEVUse, 4> MulOps;
      do {
        if (BO->Op) {
          if (auto *OpSCEV = getExistingSCEV(BO->Op)) {
            MulOps.push_back(OpSCEV);
            break;
          }
 
          SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(BO->Op);
          if (Flags != SCEV::FlagAnyWrap) {
            LHS = getSCEV(BO->LHS);
            RHS = getSCEV(BO->RHS);
            MulOps.push_back(getMulExpr(LHS, RHS, Flags));
            break;
          }
        }
 
        MulOps.push_back(getSCEV(BO->RHS));
        auto NewBO = MatchBinaryOp(BO->LHS, getDataLayout(), AC, DT,
                                   dyn_cast<Instruction>(V));
        if (!NewBO || NewBO->Opcode != Instruction::Mul) {
          MulOps.push_back(getSCEV(BO->LHS));
          break;
        }
        BO = NewBO;
      } while (true);
 
      return getMulExpr(MulOps);
    }
    case Instruction::UDiv:
      LHS = getSCEV(BO->LHS);
      RHS = getSCEV(BO->RHS);
      return getUDivExpr(LHS, RHS);
    case Instruction::URem:
      LHS = getSCEV(BO->LHS);
      RHS = getSCEV(BO->RHS);
      return getURemExpr(LHS, RHS);
    case Instruction::Sub: {
      SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
      if (BO->Op)
        Flags = getNoWrapFlagsFromUB(BO->Op);
 
      // Try to use ptrtoaddr for subtracts with at least one ptrtoint
      // operand. While we don't model ptrtoint directly in SCEV, the
      // difference between two pointer addresses is well-defined.
      Value *PtrLHS = nullptr, *PtrRHS = nullptr;
      bool HasPtrLHS = match(BO->LHS, m_PtrToInt(m_Value(PtrLHS)));
      bool HasPtrRHS = match(BO->RHS, m_PtrToInt(m_Value(PtrRHS)));
      if (HasPtrLHS || HasPtrRHS) {
        // Convert a ptrtoint operand (OrigOp) to ptrtoaddr of its pointer
        // PtrOp. When only one side is ptrtoint (BothPtr is false), skip
        // SCEVUnknown pointers since wrapping them in ptrtoaddr adds no
        // useful structure.
        auto GetOp = [&](bool HasPtr, Value *PtrOp, Value *OrigOp,
                         bool BothPtr) -> const SCEV * {
          if (!HasPtr)
            return getSCEV(OrigOp);
          const SCEV *PtrSCEV = getSCEV(PtrOp);
          if (BothPtr || !isa<SCEVUnknown>(PtrSCEV)) {
            const SCEV *Addr = getPtrToAddrExpr(PtrSCEV);
            if (!isa<SCEVCouldNotCompute>(Addr) &&
                getTypeSizeInBits(OrigOp->getType()) <=
                    getTypeSizeInBits(Addr->getType()))
              return getTruncateOrNoop(Addr, OrigOp->getType());
          }
          return getSCEV(OrigOp);
        };
        const SCEV *L = GetOp(HasPtrLHS, PtrLHS, BO->LHS, HasPtrRHS);
        const SCEV *R = GetOp(HasPtrRHS, PtrRHS, BO->RHS, HasPtrLHS);
        return getMinusSCEV(L, R, Flags);
      }
 
      LHS = getSCEV(BO->LHS);
      RHS = getSCEV(BO->RHS);
      return getMinusSCEV(LHS, RHS, Flags);
    }
    case Instruction::And:
      // For an expression like x&255 that merely masks off the high bits,
      // use zext(trunc(x)) as the SCEV expression.
      if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
        if (CI->isZero())
          return getSCEV(BO->RHS);
        if (CI->isMinusOne())
          return getSCEV(BO->LHS);
        const APInt &A = CI->getValue();
 
        // Instcombine's ShrinkDemandedConstant may strip bits out of
        // constants, obscuring what would otherwise be a low-bits mask.
        // Use computeKnownBits to compute what ShrinkDemandedConstant
        // knew about to reconstruct a low-bits mask value.
        unsigned LZ = A.countl_zero();
        unsigned TZ = A.countr_zero();
        unsigned BitWidth = A.getBitWidth();
        KnownBits Known(BitWidth);
        computeKnownBits(BO->LHS, Known, getDataLayout(), &AC, nullptr, &DT);
 
        APInt EffectiveMask =
            APInt::getLowBitsSet(BitWidth, BitWidth - LZ - TZ).shl(TZ);
        if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
          const SCEV *MulCount = getConstant(APInt::getOneBitSet(BitWidth, TZ));
          const SCEV *LHS = getSCEV(BO->LHS);
          const SCEV *ShiftedLHS = nullptr;
          if (auto *LHSMul = dyn_cast<SCEVMulExpr>(LHS)) {
            if (auto *OpC = dyn_cast<SCEVConstant>(LHSMul->getOperand(0))) {
              // For an expression like (x * 8) & 8, simplify the multiply.
              unsigned MulZeros = OpC->getAPInt().countr_zero();
              unsigned GCD = std::min(MulZeros, TZ);
              APInt DivAmt = APInt::getOneBitSet(BitWidth, TZ - GCD);
              SmallVector<SCEVUse, 4> MulOps;
              MulOps.push_back(getConstant(OpC->getAPInt().ashr(GCD)));
              append_range(MulOps, LHSMul->operands().drop_front());
              auto *NewMul = getMulExpr(MulOps, LHSMul->getNoWrapFlags());
              ShiftedLHS = getUDivExpr(NewMul, getConstant(DivAmt));
            }
          }
          if (!ShiftedLHS)
            ShiftedLHS = getUDivExpr(LHS, MulCount);
          return getMulExpr(
              getZeroExtendExpr(
                  getTruncateExpr(ShiftedLHS,
                      IntegerType::get(getContext(), BitWidth - LZ - TZ)),
                  BO->LHS->getType()),
              MulCount);
        }
      }
      // Binary `and` is a bit-wise `umin`.
      if (BO->LHS->getType()->isIntegerTy(1)) {
        LHS = getSCEV(BO->LHS);
        RHS = getSCEV(BO->RHS);
        return getUMinExpr(LHS, RHS);
      }
      break;
 
    case Instruction::Or:
      // Binary `or` is a bit-wise `umax`.
      if (BO->LHS->getType()->isIntegerTy(1)) {
        LHS = getSCEV(BO->LHS);
        RHS = getSCEV(BO->RHS);
        return getUMaxExpr(LHS, RHS);
      }
      break;
 
    case Instruction::Xor:
      if (ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS)) {
        // If the RHS of xor is -1, then this is a not operation.
        if (CI->isMinusOne())
          return getNotSCEV(getSCEV(BO->LHS));
 
        // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
        // This is a variant of the check for xor with -1, and it handles
        // the case where instcombine has trimmed non-demanded bits out
        // of an xor with -1.
        if (auto *LBO = dyn_cast<BinaryOperator>(BO->LHS))
          if (ConstantInt *LCI = dyn_cast<ConstantInt>(LBO->getOperand(1)))
            if (LBO->getOpcode() == Instruction::And &&
                LCI->getValue() == CI->getValue())
              if (const SCEVZeroExtendExpr *Z =
                      dyn_cast<SCEVZeroExtendExpr>(getSCEV(BO->LHS))) {
                Type *UTy = BO->LHS->getType();
                const SCEV *Z0 = Z->getOperand();
                Type *Z0Ty = Z0->getType();
                unsigned Z0TySize = getTypeSizeInBits(Z0Ty);
 
                // If C is a low-bits mask, the zero extend is serving to
                // mask off the high bits. Complement the operand and
                // re-apply the zext.
                if (CI->getValue().isMask(Z0TySize))
                  return getZeroExtendExpr(getNotSCEV(Z0), UTy);
 
                // If C is a single bit, it may be in the sign-bit position
                // before the zero-extend. In this case, represent the xor
                // using an add, which is equivalent, and re-apply the zext.
                APInt Trunc = CI->getValue().trunc(Z0TySize);
                if (Trunc.zext(getTypeSizeInBits(UTy)) == CI->getValue() &&
                    Trunc.isSignMask())
                  return getZeroExtendExpr(getAddExpr(Z0, getConstant(Trunc)),
                                           UTy);
              }
      }
      break;
 
    case Instruction::Shl:
      // Turn shift left of a constant amount into a multiply.
      if (ConstantInt *SA = dyn_cast<ConstantInt>(BO->RHS)) {
        uint32_t BitWidth = cast<IntegerType>(SA->getType())->getBitWidth();
 
        // If the shift count is not less than the bitwidth, the result of
        // the shift is undefined. Don't try to analyze it, because the
        // resolution chosen here may differ from the resolution chosen in
        // other parts of the compiler.
        if (SA->getValue().uge(BitWidth))
          break;
 
        // We can safely preserve the nuw flag in all cases. It's also safe to
        // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
        // requires special handling. It can be preserved as long as we're not
        // left shifting by bitwidth - 1.
        auto Flags = SCEV::FlagAnyWrap;
        if (BO->Op) {
          auto MulFlags = getNoWrapFlagsFromUB(BO->Op);
          if (any(MulFlags & SCEV::FlagNSW) &&
              (any(MulFlags & SCEV::FlagNUW) ||
               SA->getValue().ult(BitWidth - 1)))
            Flags = Flags | SCEV::FlagNSW;
          if (any(MulFlags & SCEV::FlagNUW))
            Flags = Flags | SCEV::FlagNUW;
        }
 
        ConstantInt *X = ConstantInt::get(
            getContext(), APInt::getOneBitSet(BitWidth, SA->getZExtValue()));
        return getMulExpr(getSCEV(BO->LHS), getConstant(X), Flags);
      }
      break;
 
    case Instruction::AShr:
      // AShr X, C, where C is a constant.
      ConstantInt *CI = dyn_cast<ConstantInt>(BO->RHS);
      if (!CI)
        break;
 
      Type *OuterTy = BO->LHS->getType();
      uint64_t BitWidth = getTypeSizeInBits(OuterTy);
      // If the shift count is not less than the bitwidth, the result of
      // the shift is undefined. Don't try to analyze it, because the
      // resolution chosen here may differ from the resolution chosen in
      // other parts of the compiler.
      if (CI->getValue().uge(BitWidth))
        break;
 
      if (CI->isZero())
        return getSCEV(BO->LHS); // shift by zero --> noop
 
      uint64_t AShrAmt = CI->getZExtValue();
      Type *TruncTy = IntegerType::get(getContext(), BitWidth - AShrAmt);
 
      Operator *L = dyn_cast<Operator>(BO->LHS);
      const SCEV *AddTruncateExpr = nullptr;
      ConstantInt *ShlAmtCI = nullptr;
      const SCEV *AddConstant = nullptr;
 
      if (L && L->getOpcode() == Instruction::Add) {
        // X = Shl A, n
        // Y = Add X, c
        // Z = AShr Y, m
        // n, c and m are constants.
 
        Operator *LShift = dyn_cast<Operator>(L->getOperand(0));
        ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(L->getOperand(1));
        if (LShift && LShift->getOpcode() == Instruction::Shl) {
          if (AddOperandCI) {
            const SCEV *ShlOp0SCEV = getSCEV(LShift->getOperand(0));
            ShlAmtCI = dyn_cast<ConstantInt>(LShift->getOperand(1));
            // since we truncate to TruncTy, the AddConstant should be of the
            // same type, so create a new Constant with type same as TruncTy.
            // Also, the Add constant should be shifted right by AShr amount.
            APInt AddOperand = AddOperandCI->getValue().ashr(AShrAmt);
            AddConstant = getConstant(AddOperand.trunc(BitWidth - AShrAmt));
            // we model the expression as sext(add(trunc(A), c << n)), since the
            // sext(trunc) part is already handled below, we create a
            // AddExpr(TruncExp) which will be used later.
            AddTruncateExpr = getTruncateExpr(ShlOp0SCEV, TruncTy);
          }
        }
      } else if (L && L->getOpcode() == Instruction::Shl) {
        // X = Shl A, n
        // Y = AShr X, m
        // Both n and m are constant.
 
        const SCEV *ShlOp0SCEV = getSCEV(L->getOperand(0));
        ShlAmtCI = dyn_cast<ConstantInt>(L->getOperand(1));
        AddTruncateExpr = getTruncateExpr(ShlOp0SCEV, TruncTy);
      }
 
      if (AddTruncateExpr && ShlAmtCI) {
        // We can merge the two given cases into a single SCEV statement,
        // incase n = m, the mul expression will be 2^0, so it gets resolved to
        // a simpler case. The following code handles the two cases:
        //
        // 1) For a two-shift sext-inreg, i.e. n = m,
        //    use sext(trunc(x)) as the SCEV expression.
        //
        // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
        //    expression. We already checked that ShlAmt < BitWidth, so
        //    the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
        //    ShlAmt - AShrAmt < Amt.
        const APInt &ShlAmt = ShlAmtCI->getValue();
        if (ShlAmt.ult(BitWidth) && ShlAmt.uge(AShrAmt)) {
          APInt Mul = APInt::getOneBitSet(BitWidth - AShrAmt,
                                          ShlAmtCI->getZExtValue() - AShrAmt);
          const SCEV *CompositeExpr =
              getMulExpr(AddTruncateExpr, getConstant(Mul));
          if (L->getOpcode() != Instruction::Shl)
            CompositeExpr = getAddExpr(CompositeExpr, AddConstant);
 
          return getSignExtendExpr(CompositeExpr, OuterTy);
        }
      }
      break;
    }
  }
 
  switch (U->getOpcode()) {
  case Instruction::Trunc:
    return getTruncateExpr(getSCEV(U->getOperand(0)), U->getType());
 
  case Instruction::ZExt:
    return getZeroExtendExpr(getSCEV(U->getOperand(0)), U->getType());
 
  case Instruction::SExt:
    if (auto BO = MatchBinaryOp(U->getOperand(0), getDataLayout(), AC, DT,
                                dyn_cast<Instruction>(V))) {
      // The NSW flag of a subtract does not always survive the conversion to
      // A + (-1)*B.  By pushing sign extension onto its operands we are much
      // more likely to preserve NSW and allow later AddRec optimisations.
      //
      // NOTE: This is effectively duplicating this logic from getSignExtend:
      //   sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
      // but by that point the NSW information has potentially been lost.
      if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
        Type *Ty = U->getType();
        auto *V1 = getSignExtendExpr(getSCEV(BO->LHS), Ty);
        auto *V2 = getSignExtendExpr(getSCEV(BO->RHS), Ty);
        return getMinusSCEV(V1, V2, SCEV::FlagNSW);
      }
    }
    return getSignExtendExpr(getSCEV(U->getOperand(0)), U->getType());
 
  case Instruction::BitCast:
    // BitCasts are no-op casts so we just eliminate the cast.
    if (isSCEVable(U->getType()) && isSCEVable(U->getOperand(0)->getType()))
      return getSCEV(U->getOperand(0));
    break;
 
  case Instruction::PtrToAddr: {
    const SCEV *IntOp = getPtrToAddrExpr(getSCEV(U->getOperand(0)));
    if (isa<SCEVCouldNotCompute>(IntOp))
      return getUnknown(V);
    return IntOp;
  }
 
  case Instruction::PtrToInt:
    // SCEV only models ptrtoaddr.
    return getUnknown(V);
 
  case Instruction::IntToPtr:
    // Just don't deal with inttoptr casts.
    return getUnknown(V);
 
  case Instruction::SDiv:
    // If both operands are non-negative, this is just an udiv.
    if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
        isKnownNonNegative(getSCEV(U->getOperand(1))))
      return getUDivExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
    break;
 
  case Instruction::SRem:
    // If both operands are non-negative, this is just an urem.
    if (isKnownNonNegative(getSCEV(U->getOperand(0))) &&
        isKnownNonNegative(getSCEV(U->getOperand(1))))
      return getURemExpr(getSCEV(U->getOperand(0)), getSCEV(U->getOperand(1)));
    break;
 
  case Instruction::GetElementPtr:
    return createNodeForGEP(cast<GEPOperator>(U));
 
  case Instruction::PHI:
    return createNodeForPHI(cast<PHINode>(U));
 
  case Instruction::Select:
    return createNodeForSelectOrPHI(U, U->getOperand(0), U->getOperand(1),
                                    U->getOperand(2));
 
  case Instruction::Call:
  case Instruction::Invoke:
    if (Value *RV = cast<CallBase>(U)->getReturnedArgOperand())
      return getSCEV(RV);
 
    if (auto *II = dyn_cast<IntrinsicInst>(U)) {
      switch (II->getIntrinsicID()) {
      case Intrinsic::abs:
        return getAbsExpr(
            getSCEV(II->getArgOperand(0)),
            /*IsNSW=*/cast<ConstantInt>(II->getArgOperand(1))->isOne());
      case Intrinsic::umax:
        LHS = getSCEV(II->getArgOperand(0));
        RHS = getSCEV(II->getArgOperand(1));
        return getUMaxExpr(LHS, RHS);
      case Intrinsic::umin:
        LHS = getSCEV(II->getArgOperand(0));
        RHS = getSCEV(II->getArgOperand(1));
        return getUMinExpr(LHS, RHS);
      case Intrinsic::smax:
        LHS = getSCEV(II->getArgOperand(0));
        RHS = getSCEV(II->getArgOperand(1));
        return getSMaxExpr(LHS, RHS);
      case Intrinsic::smin:
        LHS = getSCEV(II->getArgOperand(0));
        RHS = getSCEV(II->getArgOperand(1));
        return getSMinExpr(LHS, RHS);
      case Intrinsic::usub_sat: {
        const SCEV *X = getSCEV(II->getArgOperand(0));
        const SCEV *Y = getSCEV(II->getArgOperand(1));
        const SCEV *ClampedY = getUMinExpr(X, Y);
        return getMinusSCEV(X, ClampedY, SCEV::FlagNUW);
      }
      case Intrinsic::uadd_sat: {
        const SCEV *X = getSCEV(II->getArgOperand(0));
        const SCEV *Y = getSCEV(II->getArgOperand(1));
        const SCEV *ClampedX = getUMinExpr(X, getNotSCEV(Y));
        return getAddExpr(ClampedX, Y, SCEV::FlagNUW);
      }
      case Intrinsic::start_loop_iterations:
      case Intrinsic::annotation:
      case Intrinsic::ptr_annotation:
        // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
        // just eqivalent to the first operand for SCEV purposes.
        return getSCEV(II->getArgOperand(0));
      case Intrinsic::vscale:
        return getVScale(II->getType());
      default:
        break;
      }
    }
    break;
  }
 
  return getUnknown(V);
}
 
//===----------------------------------------------------------------------===//
//                   Iteration Count Computation Code
//
 
const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
  if (isa<SCEVCouldNotCompute>(ExitCount))
    return getCouldNotCompute();
 
  auto *ExitCountType = ExitCount->getType();
  assert(ExitCountType->isIntegerTy());
  auto *EvalTy = Type::getIntNTy(ExitCountType->getContext(),
                                 1 + ExitCountType->getScalarSizeInBits());
  return getTripCountFromExitCount(ExitCount, EvalTy, nullptr);
}
 
const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
                                                       Type *EvalTy,
                                                       const Loop *L) {
  if (isa<SCEVCouldNotCompute>(ExitCount))
    return getCouldNotCompute();
 
  unsigned ExitCountSize = getTypeSizeInBits(ExitCount->getType());
  unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
 
  auto CanAddOneWithoutOverflow = [&]() {
    ConstantRange ExitCountRange =
      getRangeRef(ExitCount, RangeSignHint::HINT_RANGE_UNSIGNED);
    if (!ExitCountRange.contains(APInt::getMaxValue(ExitCountSize)))
      return true;
 
    return L && isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, ExitCount,
                                         getMinusOne(ExitCount->getType()));
  };
 
  // If we need to zero extend the backedge count, check if we can add one to
  // it prior to zero extending without overflow. Provided this is safe, it
  // allows better simplification of the +1.
  if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
    return getZeroExtendExpr(
        getAddExpr(ExitCount, getOne(ExitCount->getType())), EvalTy);
 
  // Get the total trip count from the count by adding 1.  This may wrap.
  return getAddExpr(getTruncateOrZeroExtend(ExitCount, EvalTy), getOne(EvalTy));
}
 
static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
  if (!ExitCount)
    return 0;
 
  ConstantInt *ExitConst = ExitCount->getValue();
 
  // Guard against huge trip counts.
  if (ExitConst->getValue().getActiveBits() > 32)
    return 0;
 
  // In case of integer overflow, this returns 0, which is correct.
  return ((unsigned)ExitConst->getZExtValue()) + 1;
}
 
unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
  auto *ExitCount = dyn_cast<SCEVConstant>(getBackedgeTakenCount(L, Exact));
  return getConstantTripCount(ExitCount);
}
 
unsigned
ScalarEvolution::getSmallConstantTripCount(const Loop *L,
                                           const BasicBlock *ExitingBlock) {
  assert(ExitingBlock && "Must pass a non-null exiting block!");
  assert(L->isLoopExiting(ExitingBlock) &&
         "Exiting block must actually branch out of the loop!");
  const SCEVConstant *ExitCount =
      dyn_cast<SCEVConstant>(getExitCount(L, ExitingBlock));
  return getConstantTripCount(ExitCount);
}
 
unsigned ScalarEvolution::getSmallConstantMaxTripCount(
    const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
 
  const auto *MaxExitCount =
      Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates)
                 : getConstantMaxBackedgeTakenCount(L);
  return getConstantTripCount(dyn_cast<SCEVConstant>(MaxExitCount));
}
 
unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
  SmallVector<BasicBlock *, 8> ExitingBlocks;
  L->getExitingBlocks(ExitingBlocks);
 
  // An exit with an uncomputable exit count makes the result 1.
  if (ExitingBlocks.empty() ||
      any_of(ExitingBlocks, [this, L](BasicBlock *ExitingBB) {
        return isa<SCEVCouldNotCompute>(getExitCount(L, ExitingBB));
      }))
    return 1;
 
  LoopGuards Guards = LoopGuards::collect(L, *this);
  unsigned Res = 0;
  for (BasicBlock *ExitingBB : ExitingBlocks)
    Res = std::gcd(
        Res, getSmallConstantTripMultiple(getExitCount(L, ExitingBB), Guards));
  return Res;
}
 
unsigned
ScalarEvolution::getSmallConstantTripMultiple(const SCEV *ExitCount,
                                              const LoopGuards &Guards) {
  assert(!isa<SCEVCouldNotCompute>(ExitCount) && "Must be computable!");
 
  // Get the trip count
  const SCEV *TCExpr =
      getTripCountFromExitCount(applyLoopGuards(ExitCount, Guards));
 
  APInt Multiple = getNonZeroConstantMultiple(TCExpr);
  // If a trip multiple is huge (>=2^32), the trip count is still divisible by
  // the greatest power of 2 divisor less than 2^32.
  return Multiple.getActiveBits() > 32
             ? 1U << std::min(31U, Multiple.countTrailingZeros())
             : (unsigned)Multiple.getZExtValue();
}
 
unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
                                                       const SCEV *ExitCount) {
  if (isa<SCEVCouldNotCompute>(ExitCount))
    return 1;
 
  return getSmallConstantTripMultiple(ExitCount, LoopGuards::collect(L, *this));
}
 
/// Returns the largest constant divisor of the trip count of this loop as a
/// normal unsigned value, if possible. This means that the actual trip count is
/// always a multiple of the returned value (don't forget the trip count could
/// very well be zero as well!).
///
/// Returns 1 if the trip count is unknown or not guaranteed to be the
/// multiple of a constant (which is also the case if the trip count is simply
/// constant, use getSmallConstantTripCount for that case), Will also return 1
/// if the trip count is very large (>= 2^32).
///
/// As explained in the comments for getSmallConstantTripCount, this assumes
/// that control exits the loop via ExitingBlock.
unsigned
ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
                                              const BasicBlock *ExitingBlock) {
  assert(ExitingBlock && "Must pass a non-null exiting block!");
  assert(L->isLoopExiting(ExitingBlock) &&
         "Exiting block must actually branch out of the loop!");
  const SCEV *ExitCount = getExitCount(L, ExitingBlock);
  return getSmallConstantTripMultiple(L, ExitCount);
}
 
const SCEV *ScalarEvolution::getExitCount(const Loop *L,
                                          const BasicBlock *ExitingBlock,
                                          ExitCountKind Kind) {
  switch (Kind) {
  case Exact:
    return getBackedgeTakenInfo(L).getExact(ExitingBlock, this);
  case SymbolicMaximum:
    return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, this);
  case ConstantMaximum:
    return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, this);
  };
  llvm_unreachable("Invalid ExitCountKind!");
}
 
const SCEV *ScalarEvolution::getPredicatedExitCount(
    const Loop *L, const BasicBlock *ExitingBlock,
    SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
  switch (Kind) {
  case Exact:
    return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, this,
                                                      Predicates);
  case SymbolicMaximum:
    return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, this,
                                                            Predicates);
  case ConstantMaximum:
    return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, this,
                                                            Predicates);
  };
  llvm_unreachable("Invalid ExitCountKind!");
}
 
const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
    const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
  return getPredicatedBackedgeTakenInfo(L).getExact(L, this, &Preds);
}
 
const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
                                                   ExitCountKind Kind) {
  switch (Kind) {
  case Exact:
    return getBackedgeTakenInfo(L).getExact(L, this);
  case ConstantMaximum:
    return getBackedgeTakenInfo(L).getConstantMax(this);
  case SymbolicMaximum:
    return getBackedgeTakenInfo(L).getSymbolicMax(L, this);
  };
  llvm_unreachable("Invalid ExitCountKind!");
}
 
const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
    const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
  return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, this, &Preds);
}
 
const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
    const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
  return getPredicatedBackedgeTakenInfo(L).getConstantMax(this, &Preds);
}
 
bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
  return getBackedgeTakenInfo(L).isConstantMaxOrZero(this);
}
 
/// Push PHI nodes in the header of the given loop onto the given Worklist.
static void PushLoopPHIs(const Loop *L,
                         SmallVectorImpl<Instruction *> &Worklist,
                         SmallPtrSetImpl<Instruction *> &Visited) {
  BasicBlock *Header = L->getHeader();
 
  // Push all Loop-header PHIs onto the Worklist stack.
  for (PHINode &PN : Header->phis())
    if (Visited.insert(&PN).second)
      Worklist.push_back(&PN);
}
 
ScalarEvolution::BackedgeTakenInfo &
ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
  auto &BTI = getBackedgeTakenInfo(L);
  if (BTI.hasFullInfo())
    return BTI;
 
  auto Pair = PredicatedBackedgeTakenCounts.try_emplace(L);
 
  if (!Pair.second)
    return Pair.first->second;
 
  BackedgeTakenInfo Result =
      computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
 
  return PredicatedBackedgeTakenCounts.find(L)->second = std::move(Result);
}
 
ScalarEvolution::BackedgeTakenInfo &
ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
  // Initially insert an invalid entry for this loop. If the insertion
  // succeeds, proceed to actually compute a backedge-taken count and
  // update the value. The temporary CouldNotCompute value tells SCEV
  // code elsewhere that it shouldn't attempt to request a new
  // backedge-taken count, which could result in infinite recursion.
  std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
      BackedgeTakenCounts.try_emplace(L);
  if (!Pair.second)
    return Pair.first->second;
 
  // computeBackedgeTakenCount may allocate memory for its result. Inserting it
  // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
  // must be cleared in this scope.
  BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
 
  // Now that we know more about the trip count for this loop, forget any
  // existing SCEV values for PHI nodes in this loop since they are only
  // conservative estimates made without the benefit of trip count
  // information. This invalidation is not necessary for correctness, and is
  // only done to produce more precise results.
  if (Result.hasAnyInfo()) {
    // Invalidate any expression using an addrec in this loop.
    SmallVector<SCEVUse, 8> ToForget;
    auto LoopUsersIt = LoopUsers.find(L);
    if (LoopUsersIt != LoopUsers.end())
      append_range(ToForget, LoopUsersIt->second);
    forgetMemoizedResults(ToForget);
 
    // Invalidate constant-evolved loop header phis.
    for (PHINode &PN : L->getHeader()->phis())
      ConstantEvolutionLoopExitValue.erase(&PN);
  }
 
  // Re-lookup the insert position, since the call to
  // computeBackedgeTakenCount above could result in a
  // recusive call to getBackedgeTakenInfo (on a different
  // loop), which would invalidate the iterator computed
  // earlier.
  return BackedgeTakenCounts.find(L)->second = std::move(Result);
}
 
void ScalarEvolution::forgetAllLoops() {
  // This method is intended to forget all info about loops. It should
  // invalidate caches as if the following happened:
  // - The trip counts of all loops have changed arbitrarily
  // - Every llvm::Value has been updated in place to produce a different
  // result.
  BackedgeTakenCounts.clear();
  PredicatedBackedgeTakenCounts.clear();
  BECountUsers.clear();
  LoopPropertiesCache.clear();
  ConstantEvolutionLoopExitValue.clear();
  ValueExprMap.clear();
  ValuesAtScopes.clear();
  ValuesAtScopesUsers.clear();
  LoopDispositions.clear();
  BlockDispositions.clear();
  UnsignedRanges.clear();
  SignedRanges.clear();
  ExprValueMap.clear();
  HasRecMap.clear();
  ConstantMultipleCache.clear();
  PredicatedSCEVRewrites.clear();
  FoldCache.clear();
  FoldCacheUser.clear();
}
void ScalarEvolution::visitAndClearUsers(
    SmallVectorImpl<Instruction *> &Worklist,
    SmallPtrSetImpl<Instruction *> &Visited,
    SmallVectorImpl<SCEVUse> &ToForget) {
  while (!Worklist.empty()) {
    Instruction *I = Worklist.pop_back_val();
    if (!isSCEVable(I->getType()) && !isa<WithOverflowInst>(I))
      continue;
 
    ValueExprMapType::iterator It =
        ValueExprMap.find_as(static_cast<Value *>(I));
    if (It != ValueExprMap.end()) {
      ToForget.push_back(It->second);
      eraseValueFromMap(It->first);
      if (PHINode *PN = dyn_cast<PHINode>(I))
        ConstantEvolutionLoopExitValue.erase(PN);
    }
 
    PushDefUseChildren(I, Worklist, Visited);
  }
}
 
void ScalarEvolution::forgetLoop(const Loop *L) {
  SmallVector<const Loop *, 16> LoopWorklist(1, L);
  SmallVector<Instruction *, 32> Worklist;
  SmallPtrSet<Instruction *, 16> Visited;
  SmallVector<SCEVUse, 16> ToForget;
 
  // Iterate over all the loops and sub-loops to drop SCEV information.
  while (!LoopWorklist.empty()) {
    auto *CurrL = LoopWorklist.pop_back_val();
 
    // Drop any stored trip count value.
    forgetBackedgeTakenCounts(CurrL, /* Predicated */ false);
    forgetBackedgeTakenCounts(CurrL, /* Predicated */ true);
 
    // Drop information about predicated SCEV rewrites for this loop.
    PredicatedSCEVRewrites.remove_if(
        [&](const auto &Entry) { return Entry.first.second == CurrL; });
 
    auto LoopUsersItr = LoopUsers.find(CurrL);
    if (LoopUsersItr != LoopUsers.end())
      llvm::append_range(ToForget, LoopUsersItr->second);
 
    // Drop information about expressions based on loop-header PHIs.
    PushLoopPHIs(CurrL, Worklist, Visited);
    visitAndClearUsers(Worklist, Visited, ToForget);
 
    LoopPropertiesCache.erase(CurrL);
    // Forget all contained loops too, to avoid dangling entries in the
    // ValuesAtScopes map.
    LoopWorklist.append(CurrL->begin(), CurrL->end());
  }
  forgetMemoizedResults(ToForget);
}
 
void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
  forgetLoop(L->getOutermostLoop());
}
 
void ScalarEvolution::forgetValue(Value *V) {
  Instruction *I = dyn_cast<Instruction>(V);
  if (!I) return;
 
  // Drop information about expressions based on loop-header PHIs.
  SmallVector<Instruction *, 16> Worklist;
  SmallPtrSet<Instruction *, 8> Visited;
  SmallVector<SCEVUse, 8> ToForget;
  Worklist.push_back(I);
  Visited.insert(I);
  visitAndClearUsers(Worklist, Visited, ToForget);
 
  forgetMemoizedResults(ToForget);
}
 
void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
  // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
  // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
  // extra predecessor is added, this is no longer valid. Find all Unknowns and
  // AddRecs defined in the loop and invalidate any SCEV's making use of them.
  auto InvalidateValue = [&](Value *Val) {
    if (!isSCEVable(Val->getType()))
      return;
    if (const SCEV *S = getExistingSCEV(Val)) {
      struct InvalidationRootCollector {
        Loop *L;
        SmallVector<SCEVUse, 8> Roots;
 
        InvalidationRootCollector(Loop *L) : L(L) {}
 
        bool follow(const SCEV *S) {
          if (auto *SU = dyn_cast<SCEVUnknown>(S)) {
            if (auto *I = dyn_cast<Instruction>(SU->getValue()))
              if (L->contains(I))
                Roots.push_back(S);
          } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(S)) {
            if (L->contains(AddRec->getLoop()))
              Roots.push_back(S);
          }
          return true;
        }
        bool isDone() const { return false; }
      };
 
      InvalidationRootCollector C(L);
      visitAll(S, C);
      forgetMemoizedResults(C.Roots);
    }
  };
 
  InvalidateValue(V);
 
  // If V has a non-SCEV-able type (e.g. {i64, i1} from a with.overflow
  // intrinsic), its users (e.g. extractvalue) may have stale SCEV
  // expressions referencing loop-internal values.
  if (!isSCEVable(V->getType()) &&
      any_of(V->incoming_values(), IsaPred<WithOverflowInst>))
    for (User *U : V->users())
      InvalidateValue(U);
  // Also perform the normal invalidation.
  forgetValue(V);
}
 
void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
 
void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
  // Unless a specific value is passed to invalidation, completely clear both
  // caches.
  if (!V) {
    BlockDispositions.clear();
    LoopDispositions.clear();
    return;
  }
 
  if (!isSCEVable(V->getType()))
    return;
 
  const SCEV *S = getExistingSCEV(V);
  if (!S)
    return;
 
  // Invalidate the block and loop dispositions cached for S. Dispositions of
  // S's users may change if S's disposition changes (i.e. a user may change to
  // loop-invariant, if S changes to loop invariant), so also invalidate
  // dispositions of S's users recursively.
  SmallVector<SCEVUse, 8> Worklist = {S};
  SmallPtrSet<const SCEV *, 8> Seen = {S};
  while (!Worklist.empty()) {
    const SCEV *Curr = Worklist.pop_back_val();
    bool LoopDispoRemoved = LoopDispositions.erase(Curr);
    bool BlockDispoRemoved = BlockDispositions.erase(Curr);
    if (!LoopDispoRemoved && !BlockDispoRemoved)
      continue;
    auto Users = SCEVUsers.find(Curr);
    if (Users != SCEVUsers.end())
      for (const auto *User : Users->second)
        if (Seen.insert(User).second)
          Worklist.push_back(User);
  }
}
 
/// Get the exact loop backedge taken count considering all loop exits. A
/// computable result can only be returned for loops with all exiting blocks
/// dominating the latch. howFarToZero assumes that the limit of each loop test
/// is never skipped. This is a valid assumption as long as the loop exits via
/// that test. For precise results, it is the caller's responsibility to specify
/// the relevant loop exiting block using getExact(ExitingBlock, SE).
const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
    const Loop *L, ScalarEvolution *SE,
    SmallVectorImpl<const SCEVPredicate *> *Preds) const {
  // If any exits were not computable, the loop is not computable.
  if (!isComplete() || ExitNotTaken.empty())
    return SE->getCouldNotCompute();
 
  const BasicBlock *Latch = L->getLoopLatch();
  // All exiting blocks we have collected must dominate the only backedge.
  if (!Latch)
    return SE->getCouldNotCompute();
 
  // All exiting blocks we have gathered dominate loop's latch, so exact trip
  // count is simply a minimum out of all these calculated exit counts.
  SmallVector<SCEVUse, 2> Ops;
  for (const auto &ENT : ExitNotTaken) {
    const SCEV *BECount = ENT.ExactNotTaken;
    assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
    assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
           "We should only have known counts for exiting blocks that dominate "
           "latch!");
 
    Ops.push_back(BECount);
 
    if (Preds)
      append_range(*Preds, ENT.Predicates);
 
    assert((Preds || ENT.hasAlwaysTruePredicate()) &&
           "Predicate should be always true!");
  }
 
  // If an earlier exit exits on the first iteration (exit count zero), then
  // a later poison exit count should not propagate into the result. This are
  // exactly the semantics provided by umin_seq.
  return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
}
 
const ScalarEvolution::ExitNotTakenInfo *
ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
    const BasicBlock *ExitingBlock,
    SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
  for (const auto &ENT : ExitNotTaken)
    if (ENT.ExitingBlock == ExitingBlock) {
      if (ENT.hasAlwaysTruePredicate())
        return &ENT;
      else if (Predicates) {
        append_range(*Predicates, ENT.Predicates);
        return &ENT;
      }
    }
 
  return nullptr;
}
 
/// getConstantMax - Get the constant max backedge taken count for the loop.
const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
    ScalarEvolution *SE,
    SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
  if (!getConstantMax())
    return SE->getCouldNotCompute();
 
  for (const auto &ENT : ExitNotTaken)
    if (!ENT.hasAlwaysTruePredicate()) {
      if (!Predicates)
        return SE->getCouldNotCompute();
      append_range(*Predicates, ENT.Predicates);
    }
 
  assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
          isa<SCEVConstant>(getConstantMax())) &&
         "No point in having a non-constant max backedge taken count!");
  return getConstantMax();
}
 
const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
    const Loop *L, ScalarEvolution *SE,
    SmallVectorImpl<const SCEVPredicate *> *Predicates) {
  if (!SymbolicMax) {
    // Form an expression for the maximum exit count possible for this loop. We
    // merge the max and exact information to approximate a version of
    // getConstantMaxBackedgeTakenCount which isn't restricted to just
    // constants.
    SmallVector<SCEVUse, 4> ExitCounts;
 
    for (const auto &ENT : ExitNotTaken) {
      const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
      if (!isa<SCEVCouldNotCompute>(ExitCount)) {
        assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
               "We should only have known counts for exiting blocks that "
               "dominate latch!");
        ExitCounts.push_back(ExitCount);
        if (Predicates)
          append_range(*Predicates, ENT.Predicates);
 
        assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
               "Predicate should be always true!");
      }
    }
    if (ExitCounts.empty())
      SymbolicMax = SE->getCouldNotCompute();
    else
      SymbolicMax =
          SE->getUMinFromMismatchedTypes(ExitCounts, /*Sequential*/ true);
  }
  return SymbolicMax;
}
 
bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
    ScalarEvolution *SE) const {
  auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
    return !ENT.hasAlwaysTruePredicate();
  };
  return MaxOrZero && !any_of(ExitNotTaken, PredicateNotAlwaysTrue);
}
 
ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
    : ExitLimit(E, E, E, false) {}
 
ScalarEvolution::ExitLimit::ExitLimit(
    const SCEV *E, const SCEV *ConstantMaxNotTaken,
    const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
    ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
    : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
      SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
  // If we prove the max count is zero, so is the symbolic bound.  This happens
  // in practice due to differences in a) how context sensitive we've chosen
  // to be and b) how we reason about bounds implied by UB.
  if (ConstantMaxNotTaken->isZero()) {
    this->ExactNotTaken = E = ConstantMaxNotTaken;
    this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
  }
 
  assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
          !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
         "Exact is not allowed to be less precise than Constant Max");
  assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
          !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
         "Exact is not allowed to be less precise than Symbolic Max");
  assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
          !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
         "Symbolic Max is not allowed to be less precise than Constant Max");
  assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
          isa<SCEVConstant>(ConstantMaxNotTaken)) &&
         "No point in having a non-constant max backedge taken count!");
  SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
  for (const auto PredList : PredLists)
    for (const auto *P : PredList) {
      if (SeenPreds.contains(P))
        continue;
      assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
      SeenPreds.insert(P);
      Predicates.push_back(P);
    }
  assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
         "Backedge count should be int");
  assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
          !ConstantMaxNotTaken->getType()->isPointerTy()) &&
         "Max backedge count should be int");
}
 
ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
                                      const SCEV *ConstantMaxNotTaken,
                                      const SCEV *SymbolicMaxNotTaken,
                                      bool MaxOrZero,
                                      ArrayRef<const SCEVPredicate *> PredList)
    : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
                ArrayRef({PredList})) {}
 
/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
/// computable exit into a persistent ExitNotTakenInfo array.
ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
    ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
    bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
    : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
  using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
 
  ExitNotTaken.reserve(ExitCounts.size());
  std::transform(ExitCounts.begin(), ExitCounts.end(),
                 std::back_inserter(ExitNotTaken),
                 [&](const EdgeExitInfo &EEI) {
        BasicBlock *ExitBB = EEI.first;
        const ExitLimit &EL = EEI.second;
        return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
                                EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
                                EL.Predicates);
  });
  assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
          isa<SCEVConstant>(ConstantMax)) &&
         "No point in having a non-constant max backedge taken count!");
}
 
/// Compute the number of times the backedge of the specified loop will execute.
ScalarEvolution::BackedgeTakenInfo
ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
                                           bool AllowPredicates) {
  SmallVector<BasicBlock *, 8> ExitingBlocks;
  L->getExitingBlocks(ExitingBlocks);
 
  using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
 
  SmallVector<EdgeExitInfo, 4> ExitCounts;
  bool CouldComputeBECount = true;
  BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
  const SCEV *MustExitMaxBECount = nullptr;
  const SCEV *MayExitMaxBECount = nullptr;
  bool MustExitMaxOrZero = false;
  bool IsOnlyExit = ExitingBlocks.size() == 1;
 
  // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
  // and compute maxBECount.
  // Do a union of all the predicates here.
  for (BasicBlock *ExitBB : ExitingBlocks) {
    // We canonicalize untaken exits to br (constant), ignore them so that
    // proving an exit untaken doesn't negatively impact our ability to reason
    // about the loop as whole.
    if (auto *BI = dyn_cast<CondBrInst>(ExitBB->getTerminator()))
      if (auto *CI = dyn_cast<ConstantInt>(BI->getCondition())) {
        bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
        if (ExitIfTrue == CI->isZero())
          continue;
      }
 
    ExitLimit EL = computeExitLimit(L, ExitBB, IsOnlyExit, AllowPredicates);
 
    assert((AllowPredicates || EL.Predicates.empty()) &&
           "Predicated exit limit when predicates are not allowed!");
 
    // 1. For each exit that can be computed, add an entry to ExitCounts.
    // CouldComputeBECount is true only if all exits can be computed.
    if (EL.ExactNotTaken != getCouldNotCompute())
      ++NumExitCountsComputed;
    else
      // We couldn't compute an exact value for this exit, so
      // we won't be able to compute an exact value for the loop.
      CouldComputeBECount = false;
    // Remember exit count if either exact or symbolic is known. Because
    // Exact always implies symbolic, only check symbolic.
    if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
      ExitCounts.emplace_back(ExitBB, EL);
    else {
      assert(EL.ExactNotTaken == getCouldNotCompute() &&
             "Exact is known but symbolic isn't?");
      ++NumExitCountsNotComputed;
    }
 
    // 2. Derive the loop's MaxBECount from each exit's max number of
    // non-exiting iterations. Partition the loop exits into two kinds:
    // LoopMustExits and LoopMayExits.
    //
    // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
    // is a LoopMayExit.  If any computable LoopMustExit is found, then
    // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
    // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
    // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
    // any
    // computable EL.ConstantMaxNotTaken.
    if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
        DT.dominates(ExitBB, Latch)) {
      if (!MustExitMaxBECount) {
        MustExitMaxBECount = EL.ConstantMaxNotTaken;
        MustExitMaxOrZero = EL.MaxOrZero;
      } else {
        MustExitMaxBECount = getUMinFromMismatchedTypes(MustExitMaxBECount,
                                                        EL.ConstantMaxNotTaken);
      }
    } else if (MayExitMaxBECount != getCouldNotCompute()) {
      if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
        MayExitMaxBECount = EL.ConstantMaxNotTaken;
      else {
        MayExitMaxBECount = getUMaxFromMismatchedTypes(MayExitMaxBECount,
                                                       EL.ConstantMaxNotTaken);
      }
    }
  }
  const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
    (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
  // The loop backedge will be taken the maximum or zero times if there's
  // a single exit that must be taken the maximum or zero times.
  bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
 
  // Remember which SCEVs are used in exit limits for invalidation purposes.
  // We only care about non-constant SCEVs here, so we can ignore
  // EL.ConstantMaxNotTaken
  // and MaxBECount, which must be SCEVConstant.
  for (const auto &Pair : ExitCounts) {
    if (!isa<SCEVConstant>(Pair.second.ExactNotTaken))
      BECountUsers[Pair.second.ExactNotTaken].insert({L, AllowPredicates});
    if (!isa<SCEVConstant>(Pair.second.SymbolicMaxNotTaken))
      BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
          {L, AllowPredicates});
  }
  return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
                           MaxBECount, MaxOrZero);
}
 
ScalarEvolution::ExitLimit
ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
                                  bool IsOnlyExit, bool AllowPredicates) {
  assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
  // If our exiting block does not dominate the latch, then its connection with
  // loop's exit limit may be far from trivial.
  const BasicBlock *Latch = L->getLoopLatch();
  if (!Latch || !DT.dominates(ExitingBlock, Latch))
    return getCouldNotCompute();
 
  Instruction *Term = ExitingBlock->getTerminator();
  if (CondBrInst *BI = dyn_cast<CondBrInst>(Term)) {
    bool ExitIfTrue = !L->contains(BI->getSuccessor(0));
    assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
           "It should have one successor in loop and one exit block!");
    // Proceed to the next level to examine the exit condition expression.
    return computeExitLimitFromCond(L, BI->getCondition(), ExitIfTrue,
                                    /*ControlsOnlyExit=*/IsOnlyExit,
                                    AllowPredicates);
  }
 
  if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) {
    // For switch, make sure that there is a single exit from the loop.
    BasicBlock *Exit = nullptr;
    for (auto *SBB : successors(ExitingBlock))
      if (!L->contains(SBB)) {
        if (Exit) // Multiple exit successors.
          return getCouldNotCompute();
        Exit = SBB;
      }
    assert(Exit && "Exiting block must have at least one exit");
    return computeExitLimitFromSingleExitSwitch(
        L, SI, Exit, /*ControlsOnlyExit=*/IsOnlyExit);
  }
 
  return getCouldNotCompute();
}
 
ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
    const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
    bool AllowPredicates) {
  ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
  return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
                                        ControlsOnlyExit, AllowPredicates);
}
 
std::optional<ScalarEvolution::ExitLimit>
ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
                                      bool ExitIfTrue, bool ControlsOnlyExit,
                                      bool AllowPredicates) {
  (void)this->L;
  (void)this->ExitIfTrue;
  (void)this->AllowPredicates;
 
  assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
         this->AllowPredicates == AllowPredicates &&
         "Variance in assumed invariant key components!");
  auto Itr = TripCountMap.find({ExitCond, ControlsOnlyExit});
  if (Itr == TripCountMap.end())