AArch64ISelLowering.cpp

Go to the documentation of this file.

//===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation  ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the AArch64TargetLowering class.
//
//===----------------------------------------------------------------------===//
 
#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64ExpandImm.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64RegisterInfo.h"
#include "AArch64SMEAttributes.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SipHash.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>
 
using namespace llvm;
 
#define DEBUG_TYPE "aarch64-lower"
 
STATISTIC(NumTailCalls, "Number of tail calls");
STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
 
// FIXME: The necessary dtprel relocations don't seem to be supported
// well in the GNU bfd and gold linkers at the moment. Therefore, by
// default, for now, fall back to GeneralDynamic code generation.
cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
    "aarch64-elf-ldtls-generation", cl::Hidden,
    cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
    cl::init(false));
 
static cl::opt<bool>
EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
                         cl::desc("Enable AArch64 logical imm instruction "
                                  "optimization"),
                         cl::init(true));
 
// Temporary option added for the purpose of testing functionality added
// to DAGCombiner.cpp in D92230. It is expected that this can be removed
// in future when both implementations will be based off MGATHER rather
// than the GLD1 nodes added for the SVE gather load intrinsics.
static cl::opt<bool>
EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden,
                                cl::desc("Combine extends of AArch64 masked "
                                         "gather intrinsics"),
                                cl::init(true));
 
static cl::opt<bool> EnableExtToTBL("aarch64-enable-ext-to-tbl", cl::Hidden,
                                    cl::desc("Combine ext and trunc to TBL"),
                                    cl::init(true));
 
// All of the XOR, OR and CMP use ALU ports, and data dependency will become the
// bottleneck after this transform on high end CPU. So this max leaf node
// limitation is guard cmp+ccmp will be profitable.
static cl::opt<unsigned> MaxXors("aarch64-max-xors", cl::init(16), cl::Hidden,
                                 cl::desc("Maximum of xors"));
 
// By turning this on, we will not fallback to DAG ISel when encountering
// scalable vector types for all instruction, even if SVE is not yet supported
// with some instructions.
// See [AArch64TargetLowering::fallbackToDAGISel] for implementation details.
cl::opt<bool> EnableSVEGISel(
    "aarch64-enable-gisel-sve", cl::Hidden,
    cl::desc("Enable / disable SVE scalable vectors in Global ISel"),
    cl::init(false));
 
static cl::opt<int> BrMergingBaseCostThresh(
    "aarch64-br-merging-base-cost", cl::init(2),
    cl::desc(
        "Cost threshold for merging multiple conditionals into one branch "
        "versus splitting into multiple branches: conditionals are merged when "
        "their instruction cost is below this limit and split above it. Set to "
        "-1 to never merge branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingCcmpBias(
    "aarch64-br-merging-ccmp-bias", cl::init(6),
    cl::desc("Increases 'aarch64-br-merging-base-cost' to account for the "
             "CCMP instruction, which is always available on AArch64 and "
             "makes merging branch conditions cheaper."),
    cl::Hidden);
 
static cl::opt<int> BrMergingCbzTbnzBias(
    "aarch64-br-merging-cbz-tbnz-bias", cl::init(6),
    cl::desc("Decreases 'aarch64-br-merging-base-cost' when a condition can "
             "lower to a single CBZ/CBNZ or TBZ/TBNZ compare-and-branch, to "
             "bias toward splitting. Set to 0 to disable."),
    cl::Hidden);
 
static cl::opt<int> BrMergingLikelyBias(
    "aarch64-br-merging-likely-bias", cl::init(0),
    cl::desc("Increases 'aarch64-br-merging-base-cost' when all conditionals "
             "are likely to be executed, biasing toward merging. Set to -1 to "
             "never merge likely branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingUnlikelyBias(
    "aarch64-br-merging-unlikely-bias", cl::init(-1),
    cl::desc(
        "Decreases 'aarch64-br-merging-base-cost' when all conditionals are "
        "unlikely to be executed, biasing toward splitting. Set to -1 to never "
        "merge unlikely branches."),
    cl::Hidden);
 
// TODO: This option should be removed once we switch to always using PTRADD in
// the SelectionDAG.
static cl::opt<bool> UseFEATCPACodegen(
    "aarch64-use-featcpa-codegen", cl::Hidden,
    cl::desc("Generate ISD::PTRADD nodes for pointer arithmetic in "
             "SelectionDAG for FEAT_CPA"),
    cl::init(false));
 
// FPMR writes might be a synchronization barrier and thus carry a significant
// cost. Give users the option to skip writes when the requested value is
// already set.
static cl::opt<bool> UseConditionalFPMRWrite(
    "aarch64-use-conditional-fpmr-write", cl::Hidden,
    cl::desc("Only write FPMR when the requested value differs from the "
             "current value"),
    cl::init(false));
 
// Development flag to allow incremental bring up. Will be removed once the
// implementation is complete.
static cl::opt<bool> EnableSVEFixedLengthBfloatSupport(
    "aarch64-sve-vls-bfloat-support", cl::Hidden,
    cl::desc("Use SVE for fixed-length vector bfloat operations"),
    cl::init(false));
 
/// Value type used for condition codes.
constexpr MVT CondCodeVT = MVT::i32;
 
/// Value type used for NZCV flags.
constexpr MVT FlagsVT = MVT::i32;
 
static const MCPhysReg GPRArgRegs[] = {AArch64::X0, AArch64::X1, AArch64::X2,
                                       AArch64::X3, AArch64::X4, AArch64::X5,
                                       AArch64::X6, AArch64::X7};
static const MCPhysReg FPRArgRegs[] = {AArch64::Q0, AArch64::Q1, AArch64::Q2,
                                       AArch64::Q3, AArch64::Q4, AArch64::Q5,
                                       AArch64::Q6, AArch64::Q7};
 
ArrayRef<MCPhysReg> llvm::AArch64::getGPRArgRegs() { return GPRArgRegs; }
 
ArrayRef<MCPhysReg> llvm::AArch64::getFPRArgRegs() { return FPRArgRegs; }
 
static inline EVT getPackedSVEVectorVT(EVT VT) {
  switch (VT.getSimpleVT().SimpleTy) {
  default:
    llvm_unreachable("unexpected element type for vector");
  case MVT::i8:
    return MVT::nxv16i8;
  case MVT::i16:
    return MVT::nxv8i16;
  case MVT::i32:
    return MVT::nxv4i32;
  case MVT::i64:
    return MVT::nxv2i64;
  case MVT::f16:
    return MVT::nxv8f16;
  case MVT::f32:
    return MVT::nxv4f32;
  case MVT::f64:
    return MVT::nxv2f64;
  case MVT::bf16:
    return MVT::nxv8bf16;
  }
}
 
// NOTE: Currently there's only a need to return integer vector types. If this
// changes then just add an extra "type" parameter.
static inline EVT getPackedSVEVectorVT(ElementCount EC) {
  switch (EC.getKnownMinValue()) {
  default:
    llvm_unreachable("unexpected element count for vector");
  case 16:
    return MVT::nxv16i8;
  case 8:
    return MVT::nxv8i16;
  case 4:
    return MVT::nxv4i32;
  case 2:
    return MVT::nxv2i64;
  }
}
 
static inline EVT getPromotedVTForPredicate(EVT VT) {
  assert(VT.isScalableVectorOf(MVT::i1) &&
         "Expected scalable predicate vector type!");
  switch (VT.getVectorMinNumElements()) {
  default:
    llvm_unreachable("unexpected element count for vector");
  case 2:
    return MVT::nxv2i64;
  case 4:
    return MVT::nxv4i32;
  case 8:
    return MVT::nxv8i16;
  case 16:
    return MVT::nxv16i8;
  }
}
 
/// Returns true if VT's elements occupy the lowest bit positions of its
/// associated register class without any intervening space.
///
/// For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the
/// same register class, but only nxv8f16 can be treated as a packed vector.
static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
  assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
         "Expected legal vector type!");
  return VT.isFixedLengthVector() ||
         VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock;
}
 
static inline bool isPackedPredicateType(EVT VT, SelectionDAG &DAG) {
  assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
         "Expected legal type!");
  return VT == MVT::nxv16i1;
}
 
/// Returns true if the conceptual representation for \p VT does not map
/// directly to its physical register representation, meaning there are gaps
/// between elements in the register. In practice, the vector elements will be
/// strided by a power of two and placed starting from lane 0. For example,
/// nxv8i1 or nxv2f32 are unpacked types.
///
///\pre VT is a legal type.
static inline bool isUnpackedType(EVT VT, SelectionDAG &DAG) {
  bool Res = !isPackedVectorType(VT, DAG) && !isPackedPredicateType(VT, DAG);
  assert((!Res || VT.isScalableVector()) &&
         "Unexpected fixed-size unpacked type.");
  return Res;
}
 
// Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
// predicate and end with a passthru value matching the result type.
static bool isMergePassthruOpcode(unsigned Opc) {
  switch (Opc) {
  default:
    return false;
  case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
  case AArch64ISD::BSWAP_MERGE_PASSTHRU:
  case AArch64ISD::REVH_MERGE_PASSTHRU:
  case AArch64ISD::REVW_MERGE_PASSTHRU:
  case AArch64ISD::REVD_MERGE_PASSTHRU:
  case AArch64ISD::CTLZ_MERGE_PASSTHRU:
  case AArch64ISD::CTPOP_MERGE_PASSTHRU:
  case AArch64ISD::DUP_MERGE_PASSTHRU:
  case AArch64ISD::ABS_MERGE_PASSTHRU:
  case AArch64ISD::NEG_MERGE_PASSTHRU:
  case AArch64ISD::FNEG_MERGE_PASSTHRU:
  case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
  case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
  case AArch64ISD::FCEIL_MERGE_PASSTHRU:
  case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
  case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
  case AArch64ISD::FRINT_MERGE_PASSTHRU:
  case AArch64ISD::FRINT32_MERGE_PASSTHRU:
  case AArch64ISD::FRINT64_MERGE_PASSTHRU:
  case AArch64ISD::FROUND_MERGE_PASSTHRU:
  case AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU:
  case AArch64ISD::FTRUNC_MERGE_PASSTHRU:
  case AArch64ISD::FTRUNC32_MERGE_PASSTHRU:
  case AArch64ISD::FTRUNC64_MERGE_PASSTHRU:
  case AArch64ISD::FP_ROUND_MERGE_PASSTHRU:
  case AArch64ISD::FP_EXTEND_MERGE_PASSTHRU:
  case AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU:
  case AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU:
  case AArch64ISD::FCVTX_MERGE_PASSTHRU:
  case AArch64ISD::FCVTZU_MERGE_PASSTHRU:
  case AArch64ISD::FCVTZS_MERGE_PASSTHRU:
  case AArch64ISD::FSQRT_MERGE_PASSTHRU:
  case AArch64ISD::FRECPX_MERGE_PASSTHRU:
  case AArch64ISD::FABS_MERGE_PASSTHRU:
  case AArch64ISD::STRICT_FCEIL_MERGE_PASSTHRU:
  case AArch64ISD::STRICT_FFLOOR_MERGE_PASSTHRU:
  case AArch64ISD::STRICT_FNEARBYINT_MERGE_PASSTHRU:
  case AArch64ISD::STRICT_FROUND_MERGE_PASSTHRU:
  case AArch64ISD::STRICT_FROUNDEVEN_MERGE_PASSTHRU:
  case AArch64ISD::STRICT_FTRUNC_MERGE_PASSTHRU:
  case AArch64ISD::STRICT_FSQRT_MERGE_PASSTHRU:
    return true;
  }
}
 
// Returns true if inactive lanes are known to be zeroed by construction.
static bool isZeroingInactiveLanes(SDValue Op) {
  switch (Op.getOpcode()) {
  default:
    return false;
  // We guarantee i1 splat_vectors to zero the other lanes
  case ISD::SPLAT_VECTOR:
  case ISD::GET_ACTIVE_LANE_MASK:
  case AArch64ISD::PTRUE:
  case AArch64ISD::SETCC_MERGE_ZERO:
    return true;
  case ISD::INTRINSIC_WO_CHAIN:
    switch (Op.getConstantOperandVal(0)) {
    default:
      return false;
    case Intrinsic::aarch64_sve_ptrue:
    case Intrinsic::aarch64_sve_pnext:
    case Intrinsic::aarch64_sve_cmpeq:
    case Intrinsic::aarch64_sve_cmpne:
    case Intrinsic::aarch64_sve_cmpge:
    case Intrinsic::aarch64_sve_cmpgt:
    case Intrinsic::aarch64_sve_cmphs:
    case Intrinsic::aarch64_sve_cmphi:
    case Intrinsic::aarch64_sve_cmpeq_wide:
    case Intrinsic::aarch64_sve_cmpne_wide:
    case Intrinsic::aarch64_sve_cmpge_wide:
    case Intrinsic::aarch64_sve_cmpgt_wide:
    case Intrinsic::aarch64_sve_cmplt_wide:
    case Intrinsic::aarch64_sve_cmple_wide:
    case Intrinsic::aarch64_sve_cmphs_wide:
    case Intrinsic::aarch64_sve_cmphi_wide:
    case Intrinsic::aarch64_sve_cmplo_wide:
    case Intrinsic::aarch64_sve_cmpls_wide:
    case Intrinsic::aarch64_sve_fcmpeq:
    case Intrinsic::aarch64_sve_fcmpne:
    case Intrinsic::aarch64_sve_fcmpge:
    case Intrinsic::aarch64_sve_fcmpgt:
    case Intrinsic::aarch64_sve_fcmpuo:
    case Intrinsic::aarch64_sve_facgt:
    case Intrinsic::aarch64_sve_facge:
    case Intrinsic::aarch64_sve_whilege:
    case Intrinsic::aarch64_sve_whilegt:
    case Intrinsic::aarch64_sve_whilehi:
    case Intrinsic::aarch64_sve_whilehs:
    case Intrinsic::aarch64_sve_whilele:
    case Intrinsic::aarch64_sve_whilelo:
    case Intrinsic::aarch64_sve_whilels:
    case Intrinsic::aarch64_sve_whilelt:
    case Intrinsic::aarch64_sve_match:
    case Intrinsic::aarch64_sve_nmatch:
    case Intrinsic::aarch64_sve_whilege_x2:
    case Intrinsic::aarch64_sve_whilegt_x2:
    case Intrinsic::aarch64_sve_whilehi_x2:
    case Intrinsic::aarch64_sve_whilehs_x2:
    case Intrinsic::aarch64_sve_whilele_x2:
    case Intrinsic::aarch64_sve_whilelo_x2:
    case Intrinsic::aarch64_sve_whilels_x2:
    case Intrinsic::aarch64_sve_whilelt_x2:
      return true;
    }
  }
}
 
static std::tuple<SDValue, SDValue>
extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
  SDLoc DL(Disc);
  SDValue AddrDisc;
  SDValue ConstDisc;
 
  // If this is a blend, remember the constant and address discriminators.
  // Otherwise, it's either a constant discriminator, or a non-blended
  // address discriminator.
  if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
      Disc->getConstantOperandVal(0) == Intrinsic::ptrauth_blend) {
    AddrDisc = Disc->getOperand(1);
    ConstDisc = Disc->getOperand(2);
  } else {
    ConstDisc = Disc;
  }
 
  // If the constant discriminator (either the blend RHS, or the entire
  // discriminator value) isn't a 16-bit constant, bail out, and let the
  // discriminator be computed separately.
  const auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
  if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
    return std::make_tuple(DAG->getTargetConstant(0, DL, MVT::i64), Disc);
 
  // If there's no address discriminator, use NoRegister, which we'll later
  // replace with XZR, or directly use a Z variant of the inst. when available.
  if (!AddrDisc)
    AddrDisc = DAG->getRegister(AArch64::NoRegister, MVT::i64);
 
  return std::make_tuple(
      DAG->getTargetConstant(ConstDiscN->getZExtValue(), DL, MVT::i64),
      AddrDisc);
}
 
AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                                             const AArch64Subtarget &STI)
    : TargetLowering(TM, STI), Subtarget(&STI) {
  // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
  // we have to make something up. Arbitrarily, choose ZeroOrOne.
  setBooleanContents(ZeroOrOneBooleanContent);
  // When comparing vectors the result sets the different elements in the
  // vector to all-one or all-zero.
  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
  // Set up the register classes.
  addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
  addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
 
  if (Subtarget->hasLS64()) {
    addRegisterClass(MVT::i64x8, &AArch64::GPR64x8ClassRegClass);
    setOperationAction(ISD::LOAD, MVT::i64x8, Custom);
    setOperationAction(ISD::STORE, MVT::i64x8, Custom);
  }
 
  if (Subtarget->hasFPARMv8()) {
    addRegisterClass(MVT::aarch64mfp8, &AArch64::FPR8RegClass);
    addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
    addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
    addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
    addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
    addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
  }
 
  if (Subtarget->hasNEON()) {
    addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
    addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
 
    addDRType(MVT::v2f32);
    addDRType(MVT::v8i8);
    addDRType(MVT::v4i16);
    addDRType(MVT::v2i32);
    addDRType(MVT::v1i64);
    addDRType(MVT::v1f64);
    addDRType(MVT::v4f16);
    addDRType(MVT::v4bf16);
 
    addQRType(MVT::v4f32);
    addQRType(MVT::v2f64);
    addQRType(MVT::v16i8);
    addQRType(MVT::v8i16);
    addQRType(MVT::v4i32);
    addQRType(MVT::v2i64);
    addQRType(MVT::v8f16);
    addQRType(MVT::v8bf16);
  }
 
  if (Subtarget->isSVEorStreamingSVEAvailable()) {
    // Add legal sve predicate types
    addRegisterClass(MVT::nxv1i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
 
    // Add sve predicate as counter type
    addRegisterClass(MVT::aarch64svcount, &AArch64::PNRRegClass);
 
    // Add legal sve data types
    addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
 
    addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
 
    addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
 
    if (Subtarget->useSVEForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
        if (useSVEForFixedLengthVectorVT(VT))
          addRegisterClass(VT, &AArch64::ZPRRegClass);
 
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
        if (useSVEForFixedLengthVectorVT(VT))
          addRegisterClass(VT, &AArch64::ZPRRegClass);
    }
  }
 
  // Compute derived properties from the register classes
  computeRegisterProperties(Subtarget->getRegisterInfo());
 
  // Provide all sorts of operation actions
  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
  setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
  setOperationAction(ISD::SETCC, MVT::i32, Custom);
  setOperationAction(ISD::SETCC, MVT::i64, Custom);
  setOperationAction(ISD::SETCC, MVT::bf16, Custom);
  setOperationAction(ISD::SETCC, MVT::f16, Custom);
  setOperationAction(ISD::SETCC, MVT::f32, Custom);
  setOperationAction(ISD::SETCC, MVT::f64, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::bf16, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
  setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
  setOperationAction(ISD::CTLS, MVT::i32, Legal);
  setOperationAction(ISD::CTLS, MVT::i64, Legal);
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
  setOperationAction(ISD::BR_CC, MVT::i32, Custom);
  setOperationAction(ISD::BR_CC, MVT::i64, Custom);
  setOperationAction(ISD::BR_CC, MVT::f16, Custom);
  setOperationAction(ISD::BR_CC, MVT::f32, Custom);
  setOperationAction(ISD::BR_CC, MVT::f64, Custom);
  setOperationAction(ISD::SELECT, MVT::i32, Custom);
  setOperationAction(ISD::SELECT, MVT::i64, Custom);
  if (Subtarget->hasFPARMv8()) {
    setOperationAction(ISD::SELECT, MVT::f16, Custom);
    setOperationAction(ISD::SELECT, MVT::bf16, Custom);
  }
  setOperationAction(ISD::SELECT, MVT::f32, Custom);
  setOperationAction(ISD::SELECT, MVT::f64, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::bf16, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
  setOperationAction(ISD::BR_JT, MVT::Other, Custom);
  setOperationAction(ISD::JumpTable, MVT::i64, Custom);
  setOperationAction(ISD::BRIND, MVT::Other, Custom);
  setOperationAction(ISD::SETCCCARRY, MVT::i64, Custom);
 
  setOperationAction(ISD::PtrAuthGlobalAddress, MVT::i64, Custom);
 
  setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
  setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
  setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
 
  setOperationAction(ISD::FREM, MVT::f32, LibCall);
  setOperationAction(ISD::FREM, MVT::f64, LibCall);
 
  setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
 
  // Custom lowering hooks are needed for XOR
  // to fold it into CSINC/CSINV.
  setOperationAction(ISD::XOR, MVT::i32, Custom);
  setOperationAction(ISD::XOR, MVT::i64, Custom);
 
  setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
  setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
 
  // Virtually no operation on f128 is legal, but LLVM can't expand them when
  // there's a valid register class, so we need custom operations in most cases.
  setOperationAction(ISD::FABS, MVT::f128, Expand);
  setOperationAction(ISD::FADD, MVT::f128, LibCall);
  setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
  setOperationAction(ISD::FCOS, MVT::f128, Expand);
  setOperationAction(ISD::FDIV, MVT::f128, LibCall);
  setOperationAction(ISD::FMA, MVT::f128, Expand);
  setOperationAction(ISD::FMUL, MVT::f128, LibCall);
  setOperationAction(ISD::FNEG, MVT::f128, Expand);
  setOperationAction(ISD::FPOW, MVT::f128, Expand);
  setOperationAction(ISD::FREM, MVT::f128, LibCall);
  setOperationAction(ISD::FRINT, MVT::f128, Expand);
  setOperationAction(ISD::FSIN, MVT::f128, Expand);
  setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
  setOperationAction(ISD::FSQRT, MVT::f128, Expand);
  setOperationAction(ISD::FSUB, MVT::f128, LibCall);
  setOperationAction(ISD::FTAN, MVT::f128, Expand);
  setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
  setOperationAction(ISD::SETCC, MVT::f128, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
  setOperationAction(ISD::BR_CC, MVT::f128, Custom);
  setOperationAction(ISD::SELECT, MVT::f128, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
  setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
  // FIXME: f128 FMINIMUM and FMAXIMUM (including STRICT versions) currently
  // aren't handled.
 
  // Lowering for many of the conversions is actually specified by the non-f128
  // type. The LowerXXX function will be trivial when f128 isn't involved.
  setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
  setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
  setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
  setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
  setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
  setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
  if (Subtarget->hasFPARMv8()) {
    setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
    setOperationAction(ISD::FP_ROUND, MVT::bf16, Custom);
  }
  setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
  setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
  if (Subtarget->hasFPARMv8()) {
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::bf16, Custom);
  }
  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
 
  setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
  setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
 
  // Variable arguments.
  setOperationAction(ISD::VASTART, MVT::Other, Custom);
  setOperationAction(ISD::VAARG, MVT::Other, Custom);
  setOperationAction(ISD::VACOPY, MVT::Other, Custom);
  setOperationAction(ISD::VAEND, MVT::Other, Expand);
 
  // Variable-sized objects.
  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
 
  // Lowering Funnel Shifts to EXTR
  setOperationAction(ISD::FSHR, MVT::i32, Custom);
  setOperationAction(ISD::FSHR, MVT::i64, Custom);
  setOperationAction(ISD::FSHL, MVT::i32, Custom);
  setOperationAction(ISD::FSHL, MVT::i64, Custom);
 
  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
 
  // Constant pool entries
  setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
 
  // BlockAddress
  setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
 
  // AArch64 lacks both left-rotate and popcount instructions.
  setOperationAction(ISD::ROTL, MVT::i32, Expand);
  setOperationAction(ISD::ROTL, MVT::i64, Expand);
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    setOperationAction(ISD::ROTL, VT, Expand);
    setOperationAction(ISD::ROTR, VT, Expand);
  }
 
  // AArch64 doesn't have i32 MULH{S|U}.
  setOperationAction(ISD::MULHU, MVT::i32, Expand);
  setOperationAction(ISD::MULHS, MVT::i32, Expand);
 
  // AArch64 doesn't have {U|S}MUL_LOHI.
  setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
  setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
  setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
  setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
 
  if (Subtarget->hasCSSC()) {
    setOperationAction(ISD::CTPOP, MVT::i32, Legal);
    setOperationAction(ISD::CTPOP, MVT::i64, Legal);
    setOperationAction(ISD::CTPOP, MVT::i128, Expand);
 
    setOperationAction(ISD::PARITY, MVT::i128, Expand);
 
    setOperationAction(ISD::CTTZ, MVT::i32, Legal);
    setOperationAction(ISD::CTTZ, MVT::i64, Legal);
    setOperationAction(ISD::CTTZ, MVT::i128, Expand);
 
    setOperationAction(ISD::ABS, MVT::i32, Legal);
    setOperationAction(ISD::ABS, MVT::i64, Legal);
 
    setOperationAction(ISD::SMAX, MVT::i32, Legal);
    setOperationAction(ISD::SMAX, MVT::i64, Legal);
    setOperationAction(ISD::UMAX, MVT::i32, Legal);
    setOperationAction(ISD::UMAX, MVT::i64, Legal);
 
    setOperationAction(ISD::SMIN, MVT::i32, Legal);
    setOperationAction(ISD::SMIN, MVT::i64, Legal);
    setOperationAction(ISD::UMIN, MVT::i32, Legal);
    setOperationAction(ISD::UMIN, MVT::i64, Legal);
  } else {
    setOperationAction(ISD::CTPOP, MVT::i32, Custom);
    setOperationAction(ISD::CTPOP, MVT::i64, Custom);
    setOperationAction(ISD::CTPOP, MVT::i128, Custom);
 
    setOperationAction(ISD::PARITY, MVT::i64, Custom);
    setOperationAction(ISD::PARITY, MVT::i128, Custom);
 
    setOperationAction(ISD::ABS, MVT::i32, Custom);
    setOperationAction(ISD::ABS, MVT::i64, Custom);
  }
 
  setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
  setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    setOperationAction(ISD::SDIVREM, VT, Expand);
    setOperationAction(ISD::UDIVREM, VT, Expand);
  }
  setOperationAction(ISD::SREM, MVT::i32, Expand);
  setOperationAction(ISD::SREM, MVT::i64, Expand);
  setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
  setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
  setOperationAction(ISD::UREM, MVT::i32, Expand);
  setOperationAction(ISD::UREM, MVT::i64, Expand);
 
  // Custom lower Add/Sub/Mul with overflow.
  setOperationAction(ISD::SADDO, MVT::i32, Custom);
  setOperationAction(ISD::SADDO, MVT::i64, Custom);
  setOperationAction(ISD::UADDO, MVT::i32, Custom);
  setOperationAction(ISD::UADDO, MVT::i64, Custom);
  setOperationAction(ISD::SSUBO, MVT::i32, Custom);
  setOperationAction(ISD::SSUBO, MVT::i64, Custom);
  setOperationAction(ISD::USUBO, MVT::i32, Custom);
  setOperationAction(ISD::USUBO, MVT::i64, Custom);
  setOperationAction(ISD::SMULO, MVT::i32, Custom);
  setOperationAction(ISD::SMULO, MVT::i64, Custom);
  setOperationAction(ISD::UMULO, MVT::i32, Custom);
  setOperationAction(ISD::UMULO, MVT::i64, Custom);
 
  setOperationAction(ISD::UADDO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::UADDO_CARRY, MVT::i64, Custom);
  setOperationAction(ISD::USUBO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::USUBO_CARRY, MVT::i64, Custom);
  setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::SADDO_CARRY, MVT::i64, Custom);
  setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::SSUBO_CARRY, MVT::i64, Custom);
 
  setOperationAction(ISD::FSIN, MVT::f64, Expand);
  setOperationAction(ISD::FSIN, MVT::f32, Expand);
  setOperationAction(ISD::FCOS, MVT::f32, Expand);
  setOperationAction(ISD::FCOS, MVT::f64, Expand);
  setOperationAction(ISD::FPOW, MVT::f32, Expand);
  setOperationAction(ISD::FPOW, MVT::f64, Expand);
  setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
  setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
  if (Subtarget->hasFullFP16()) {
    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom);
  } else {
    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote);
  }
 
  for (auto Op : {ISD::FREM,          ISD::FPOW,         ISD::FPOWI,
                  ISD::FCOS,          ISD::FSIN,         ISD::FSINCOS,
                  ISD::FSINCOSPI,     ISD::FMODF,        ISD::FACOS,
                  ISD::FASIN,         ISD::FATAN,        ISD::FATAN2,
                  ISD::FCOSH,         ISD::FSINH,        ISD::FTANH,
                  ISD::FTAN,          ISD::FEXP,         ISD::FEXP2,
                  ISD::FEXP10,        ISD::FLOG,         ISD::FLOG2,
                  ISD::FLOG10,        ISD::STRICT_FREM,  ISD::STRICT_FPOW,
                  ISD::STRICT_FPOWI,  ISD::STRICT_FCOS,  ISD::STRICT_FSIN,
                  ISD::STRICT_FACOS,  ISD::STRICT_FASIN, ISD::STRICT_FATAN,
                  ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
                  ISD::STRICT_FTANH,  ISD::STRICT_FEXP,  ISD::STRICT_FEXP2,
                  ISD::STRICT_FLOG,   ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
                  ISD::STRICT_FTAN}) {
    setOperationAction(Op, MVT::f16, Promote);
    setOperationAction(Op, MVT::v4f16, Expand);
    setOperationAction(Op, MVT::v8f16, Expand);
    setOperationAction(Op, MVT::bf16, Promote);
    setOperationAction(Op, MVT::v4bf16, Expand);
    setOperationAction(Op, MVT::v8bf16, Expand);
  }
 
  // Legalize fcanonicalize to circumvent default expansion
  setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Legal);
  if (Subtarget->hasFullFP16()) {
    setOperationAction(ISD::FCANONICALIZE, MVT::f16, Legal);
  }
 
  // fpextend from f16 or bf16 to f32 is legal
  setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
  setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Legal);
  setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f32, Legal);
  // fpextend from bf16 to f64 needs to be split into two fpextends
  setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
  setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
 
  auto LegalizeNarrowFP = [this](MVT ScalarVT) {
    for (auto Op : {
             ISD::SETCC,
             ISD::SELECT_CC,
             ISD::BR_CC,
             ISD::FADD,
             ISD::FSUB,
             ISD::FMUL,
             ISD::FDIV,
             ISD::FMA,
             ISD::FCEIL,
             ISD::FSQRT,
             ISD::FFLOOR,
             ISD::FNEARBYINT,
             ISD::FRINT,
             ISD::FROUND,
             ISD::FROUNDEVEN,
             ISD::FTRUNC,
             ISD::FMINNUM,
             ISD::FMAXNUM,
             ISD::FMINIMUM,
             ISD::FMAXIMUM,
             ISD::FMINIMUMNUM,
             ISD::FMAXIMUMNUM,
             ISD::FCANONICALIZE,
             ISD::STRICT_FADD,
             ISD::STRICT_FSUB,
             ISD::STRICT_FMUL,
             ISD::STRICT_FDIV,
             ISD::STRICT_FMA,
             ISD::STRICT_FCEIL,
             ISD::STRICT_FFLOOR,
             ISD::STRICT_FSQRT,
             ISD::STRICT_FRINT,
             ISD::STRICT_FNEARBYINT,
             ISD::STRICT_FROUND,
             ISD::STRICT_FTRUNC,
             ISD::STRICT_FROUNDEVEN,
             ISD::STRICT_FMINNUM,
             ISD::STRICT_FMAXNUM,
             ISD::STRICT_FMINIMUM,
             ISD::STRICT_FMAXIMUM,
         })
      setOperationAction(Op, ScalarVT, Promote);
 
    for (auto Op : {ISD::FNEG, ISD::FABS})
      setOperationAction(Op, ScalarVT, Legal);
 
    // Round-to-integer need custom lowering for fp16, as Promote doesn't work
    // because the result type is integer.
    for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
                    ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT,
                    ISD::STRICT_LLRINT})
      setOperationAction(Op, ScalarVT, Custom);
 
    // promote v4f16 to v4f32 when that is known to be safe.
    auto V4Narrow = MVT::getVectorVT(ScalarVT, 4);
    setOperationPromotedToType(ISD::FADD,       V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FSUB,       V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FMUL,       V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FDIV,       V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FCEIL,      V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FFLOOR,     V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FROUND,     V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FTRUNC,     V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FROUNDEVEN, V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FRINT,      V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FNEARBYINT, V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::FCANONICALIZE, V4Narrow, MVT::v4f32);
    setOperationPromotedToType(ISD::SETCC,         V4Narrow, MVT::v4f32);
 
    setOperationAction(ISD::FABS,        V4Narrow, Legal);
    setOperationAction(ISD::FNEG,        V4Narrow, Legal);
    setOperationAction(ISD::FMA,         V4Narrow, Expand);
    setOperationAction(ISD::BR_CC,       V4Narrow, Expand);
    setOperationAction(ISD::SELECT,      V4Narrow, Expand);
    setOperationAction(ISD::SELECT_CC,   V4Narrow, Expand);
    setOperationAction(ISD::FCOPYSIGN,   V4Narrow, Custom);
    setOperationAction(ISD::FSQRT,       V4Narrow, Expand);
 
    auto V8Narrow = MVT::getVectorVT(ScalarVT, 8);
    setOperationPromotedToType(ISD::FCANONICALIZE, V8Narrow, MVT::v8f32);
    setOperationPromotedToType(ISD::SETCC,         V8Narrow, MVT::v8f32);
    setOperationPromotedToType(ISD::VECREDUCE_FADD, V8Narrow, MVT::v8f32);
    setOperationPromotedToType(ISD::VECREDUCE_FMUL, V8Narrow, MVT::v8f32);
 
    setOperationAction(ISD::FABS,        V8Narrow, Legal);
    setOperationAction(ISD::FADD,        V8Narrow, Expand);
    setOperationAction(ISD::FCEIL,       V8Narrow, Expand);
    setOperationAction(ISD::FCOPYSIGN,   V8Narrow, Custom);
    setOperationAction(ISD::FDIV,        V8Narrow, Expand);
    setOperationAction(ISD::FFLOOR,      V8Narrow, Expand);
    setOperationAction(ISD::FMA,         V8Narrow, Expand);
    setOperationAction(ISD::FMUL,        V8Narrow, Expand);
    setOperationAction(ISD::FNEARBYINT,  V8Narrow, Expand);
    setOperationAction(ISD::FNEG,        V8Narrow, Legal);
    setOperationAction(ISD::FROUND,      V8Narrow, Expand);
    setOperationAction(ISD::FROUNDEVEN,  V8Narrow, Expand);
    setOperationAction(ISD::FRINT,       V8Narrow, Expand);
    setOperationAction(ISD::FSQRT,       V8Narrow, Expand);
    setOperationAction(ISD::FSUB,        V8Narrow, Expand);
    setOperationAction(ISD::FTRUNC,      V8Narrow, Expand);
    setOperationAction(ISD::BR_CC,       V8Narrow, Expand);
    setOperationAction(ISD::SELECT,      V8Narrow, Expand);
    setOperationAction(ISD::SELECT_CC,   V8Narrow, Expand);
    setOperationAction(ISD::FP_EXTEND,   V8Narrow, Expand);
  };
 
  if (!Subtarget->hasFullFP16()) {
    LegalizeNarrowFP(MVT::f16);
  }
  LegalizeNarrowFP(MVT::bf16);
  setOperationAction(ISD::FP_ROUND, MVT::v4f32, Custom);
  setOperationAction(ISD::FP_ROUND, MVT::v4bf16, Custom);
 
  // AArch64 has implementations of a lot of rounding-like FP operations.
  // clang-format off
  for (auto Op :
       {ISD::FFLOOR,          ISD::FNEARBYINT,      ISD::FCEIL,
        ISD::FRINT,           ISD::FTRUNC,          ISD::FROUND,
        ISD::FROUNDEVEN,      ISD::FMINNUM,         ISD::FMAXNUM,
        ISD::FMINIMUM,        ISD::FMAXIMUM,        ISD::LROUND,
        ISD::LLROUND,         ISD::LRINT,           ISD::LLRINT,
        ISD::FMINNUM_IEEE,    ISD::FMAXNUM_IEEE,
        ISD::STRICT_FFLOOR,   ISD::STRICT_FCEIL,    ISD::STRICT_FNEARBYINT,
        ISD::STRICT_FRINT,    ISD::STRICT_FTRUNC,   ISD::STRICT_FROUNDEVEN,
        ISD::STRICT_FROUND,   ISD::STRICT_FMINNUM,  ISD::STRICT_FMAXNUM,
        ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_LROUND,
        ISD::STRICT_LLROUND,  ISD::STRICT_LRINT,    ISD::STRICT_LLRINT}) {
    for (MVT Ty : {MVT::f32, MVT::f64})
      setOperationAction(Op, Ty, Legal);
    if (Subtarget->hasFullFP16())
      setOperationAction(Op, MVT::f16, Legal);
  }
  // clang-format on
 
  // Basic strict FP operations are legal
  for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                  ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT}) {
    for (MVT Ty : {MVT::f32, MVT::f64})
      setOperationAction(Op, Ty, Legal);
    if (Subtarget->hasFullFP16())
      setOperationAction(Op, MVT::f16, Legal);
  }
 
  setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
  setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
  setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
  setOperationAction(ISD::GET_FPMODE, MVT::i32, Custom);
  setOperationAction(ISD::SET_FPMODE, MVT::i32, Custom);
  setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
 
  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
  if (!Subtarget->hasLSE() && !Subtarget->outlineAtomics()) {
    setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, LibCall);
  } else {
    setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
    setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Expand);
  }
  setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
  setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
 
  // Generate outline atomics library calls only if LSE was not specified for
  // subtarget
  if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) {
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall);
  }
 
  if (Subtarget->hasLSE128()) {
    // Custom lowering because i128 is not legal. Must be replaced by 2x64
    // values. ATOMIC_LOAD_AND also needs op legalisation to emit LDCLRP.
    setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i128, Custom);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i128, Custom);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i128, Custom);
  }
 
  // 128-bit loads and stores can be done without expanding
  setOperationAction(ISD::LOAD, MVT::i128, Custom);
  setOperationAction(ISD::STORE, MVT::i128, Custom);
 
  // Aligned 128-bit loads and stores are single-copy atomic according to the
  // v8.4a spec. LRCPC3 introduces 128-bit STILP/LDIAPP but still requires LSE2.
  if (Subtarget->hasLSE2()) {
    setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
    setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
  }
 
  // 256 bit non-temporal stores can be lowered to STNP. Do this as part of the
  // custom lowering, as there are no un-paired non-temporal stores and
  // legalization will break up 256 bit inputs.
  setOperationAction(ISD::STORE, MVT::v32i8, Custom);
  setOperationAction(ISD::STORE, MVT::v16i16, Custom);
  setOperationAction(ISD::STORE, MVT::v16f16, Custom);
  setOperationAction(ISD::STORE, MVT::v16bf16, Custom);
  setOperationAction(ISD::STORE, MVT::v8i32, Custom);
  setOperationAction(ISD::STORE, MVT::v8f32, Custom);
  setOperationAction(ISD::STORE, MVT::v4f64, Custom);
  setOperationAction(ISD::STORE, MVT::v4i64, Custom);
 
  // 256 bit non-temporal loads can be lowered to LDNP. This is done using
  // custom lowering, as there are no un-paired non-temporal loads legalization
  // will break up 256 bit inputs.
  setOperationAction(ISD::LOAD, MVT::v32i8, Custom);
  setOperationAction(ISD::LOAD, MVT::v16i16, Custom);
  setOperationAction(ISD::LOAD, MVT::v16f16, Custom);
  setOperationAction(ISD::LOAD, MVT::v16bf16, Custom);
  setOperationAction(ISD::LOAD, MVT::v8i32, Custom);
  setOperationAction(ISD::LOAD, MVT::v8f32, Custom);
  setOperationAction(ISD::LOAD, MVT::v4f64, Custom);
  setOperationAction(ISD::LOAD, MVT::v4i64, Custom);
 
  // Lower READCYCLECOUNTER using an mrs from CNTVCT_EL0.
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
 
  // Issue __sincos_stret if available.
  setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
  setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
 
  // Make floating-point constants legal for the large code model, so they don't
  // become loads from the constant pool.
  if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
    setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
    setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
  }
 
  // AArch64 does not have floating-point extending loads, i1 sign-extending
  // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
  for (MVT VT : MVT::fp_valuetypes()) {
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
  }
  for (MVT VT : MVT::integer_valuetypes())
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
 
  for (MVT WideVT : MVT::fp_valuetypes()) {
    for (MVT NarrowVT : MVT::fp_valuetypes()) {
      if (WideVT.getScalarSizeInBits() > NarrowVT.getScalarSizeInBits()) {
        setTruncStoreAction(WideVT, NarrowVT, Expand);
      }
    }
  }
 
  if (Subtarget->hasFPARMv8()) {
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
    setOperationAction(ISD::BITCAST, MVT::f16, Custom);
    setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
  }
 
  // Indexed loads and stores are supported.
  for (unsigned im = (unsigned)ISD::PRE_INC;
       im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
    setIndexedLoadAction(im, MVT::i8, Legal);
    setIndexedLoadAction(im, MVT::i16, Legal);
    setIndexedLoadAction(im, MVT::i32, Legal);
    setIndexedLoadAction(im, MVT::i64, Legal);
    setIndexedLoadAction(im, MVT::f64, Legal);
    setIndexedLoadAction(im, MVT::f32, Legal);
    setIndexedLoadAction(im, MVT::f16, Legal);
    setIndexedLoadAction(im, MVT::bf16, Legal);
    setIndexedStoreAction(im, MVT::i8, Legal);
    setIndexedStoreAction(im, MVT::i16, Legal);
    setIndexedStoreAction(im, MVT::i32, Legal);
    setIndexedStoreAction(im, MVT::i64, Legal);
    setIndexedStoreAction(im, MVT::f64, Legal);
    setIndexedStoreAction(im, MVT::f32, Legal);
    setIndexedStoreAction(im, MVT::f16, Legal);
    setIndexedStoreAction(im, MVT::bf16, Legal);
  }
 
  // Trap.
  setOperationAction(ISD::TRAP, MVT::Other, Legal);
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
  setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
 
  // We combine OR nodes for ccmp operations.
  setTargetDAGCombine(ISD::OR);
  // Try to create BICs for vector ANDs.
  setTargetDAGCombine(ISD::AND);
 
  // llvm.init.trampoline and llvm.adjust.trampoline
  setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
  setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
 
  // Vector add and sub nodes may conceal a high-half opportunity.
  // Also, try to fold ADD into CSINC/CSINV..
  setTargetDAGCombine({ISD::ADD, ISD::ABS, ISD::SUB, ISD::XOR, ISD::SINT_TO_FP,
                       ISD::UINT_TO_FP});
 
  setTargetDAGCombine({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
                       ISD::FP_TO_UINT_SAT, ISD::FADD});
 
  // Try and combine setcc/select_cc with csel and bool-vector bitcasts.
  setTargetDAGCombine(ISD::SETCC);
  setTargetDAGCombine(ISD::SELECT_CC);
  setTargetDAGCombine(ISD::SETCCCARRY);
 
  setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
 
  setTargetDAGCombine({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND,
                       ISD::SIGN_EXTEND_INREG, ISD::CONCAT_VECTORS,
                       ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
                       ISD::STORE, ISD::BUILD_VECTOR});
  setTargetDAGCombine({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX});
  setTargetDAGCombine(ISD::TRUNCATE);
  setTargetDAGCombine(ISD::LOAD);
 
  setTargetDAGCombine(ISD::MSTORE);
 
  setTargetDAGCombine(ISD::MUL);
 
  setTargetDAGCombine({ISD::SELECT, ISD::VSELECT});
 
  setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
                       ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
                       ISD::VECREDUCE_ADD, ISD::STEP_VECTOR});
 
  setTargetDAGCombine(
      {ISD::MGATHER, ISD::MSCATTER, ISD::EXPERIMENTAL_VECTOR_HISTOGRAM});
 
  setTargetDAGCombine(ISD::FP_EXTEND);
 
  setTargetDAGCombine(ISD::GlobalAddress);
 
  setTargetDAGCombine(ISD::CTLZ);
  setTargetDAGCombine({ISD::CTTZ, ISD::CTTZ_ZERO_POISON});
 
  setTargetDAGCombine(ISD::GET_ACTIVE_LANE_MASK);
 
  setTargetDAGCombine(ISD::VECREDUCE_AND);
  setTargetDAGCombine(ISD::VECREDUCE_OR);
  setTargetDAGCombine(ISD::VECREDUCE_XOR);
  setTargetDAGCombine(ISD::VECREDUCE_FADD);
  setTargetDAGCombine(ISD::VECREDUCE_SEQ_FADD);
 
  setTargetDAGCombine(ISD::SCALAR_TO_VECTOR);
 
  setTargetDAGCombine(ISD::SHL);
  setTargetDAGCombine(ISD::VECTOR_DEINTERLEAVE);
  setTargetDAGCombine(ISD::CTPOP);
 
  if (Subtarget->isSVEorStreamingSVEAvailable())
    setTargetDAGCombine(ISD::BITCAST);
 
  // In case of strict alignment, avoid an excessive number of byte wide stores.
  MaxStoresPerMemsetOptSize = 8;
  MaxStoresPerMemset =
      Subtarget->requiresStrictAlign() ? MaxStoresPerMemsetOptSize : 32;
 
  MaxGluedStoresPerMemcpy = 4;
  MaxStoresPerMemcpyOptSize = 4;
  MaxStoresPerMemcpy =
      Subtarget->requiresStrictAlign() ? MaxStoresPerMemcpyOptSize : 16;
 
  MaxStoresPerMemmoveOptSize = 4;
  MaxStoresPerMemmove =
      Subtarget->requiresStrictAlign() ? MaxStoresPerMemmoveOptSize : 16;
 
  MaxLoadsPerMemcmpOptSize = 4;
  MaxLoadsPerMemcmp =
      Subtarget->requiresStrictAlign() ? MaxLoadsPerMemcmpOptSize : 8;
 
  setStackPointerRegisterToSaveRestore(AArch64::SP);
 
  setSchedulingPreference(Sched::Hybrid);
 
  EnableExtLdPromotion = true;
 
  // Set required alignment.
  setMinFunctionAlignment(Align(4));
  // Set preferred alignments.
 
  // Don't align loops on Windows. The SEH unwind info generation needs to
  // know the exact length of functions before the alignments have been
  // expanded.
  if (!Subtarget->isTargetWindows())
    setPrefLoopAlignment(STI.getPrefLoopAlignment());
  setMaxBytesForAlignment(STI.getMaxBytesForLoopAlignment());
  setPrefFunctionAlignment(STI.getPrefFunctionAlignment());
 
  // Only change the limit for entries in a jump table if specified by
  // the sub target, but not at the command line.
  unsigned MaxJT = STI.getMaximumJumpTableSize();
  if (MaxJT && getMaximumJumpTableSize() == UINT_MAX)
    setMaximumJumpTableSize(MaxJT);
 
  setHasExtractBitsInsn(true);
 
  setMaxDivRemBitWidthSupported(128);
 
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  if (Subtarget->hasSME())
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i1, Custom);
 
  if (Subtarget->isNeonAvailable()) {
    // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
    // silliness like this:
    // clang-format off
    for (auto Op :
         {ISD::SELECT,            ISD::SELECT_CC,      ISD::FATAN2,
          ISD::BR_CC,             ISD::FADD,           ISD::FSUB,
          ISD::FMUL,              ISD::FDIV,           ISD::FMA,
          ISD::FNEG,              ISD::FABS,           ISD::FCEIL,
          ISD::FSQRT,             ISD::FFLOOR,         ISD::FNEARBYINT,
          ISD::FSIN,              ISD::FCOS,           ISD::FTAN,
          ISD::FASIN,             ISD::FACOS,          ISD::FATAN,
          ISD::FSINH,             ISD::FCOSH,          ISD::FTANH,
          ISD::FPOW,              ISD::FLOG,           ISD::FLOG2,
          ISD::FLOG10,            ISD::FEXP,           ISD::FEXP2,
          ISD::FEXP10,            ISD::FRINT,          ISD::FROUND,
          ISD::FROUNDEVEN,        ISD::FTRUNC,         ISD::FMINNUM,
          ISD::FMAXNUM,           ISD::FMINIMUM,       ISD::FMAXIMUM,
          ISD::FMAXNUM_IEEE,      ISD::FMINNUM_IEEE,
          ISD::STRICT_FADD,       ISD::STRICT_FSUB,    ISD::STRICT_FMUL,
          ISD::STRICT_FDIV,       ISD::STRICT_FMA,     ISD::STRICT_FCEIL,
          ISD::STRICT_FFLOOR,     ISD::STRICT_FSQRT,   ISD::STRICT_FRINT,
          ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND,  ISD::STRICT_FTRUNC,
          ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
          ISD::STRICT_FMINIMUM,   ISD::STRICT_FMAXIMUM})
      setOperationAction(Op, MVT::v1f64, Expand);
    // clang-format on
 
    for (auto Op :
         {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP,
          ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, ISD::MUL,
          ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT,
          ISD::STRICT_SINT_TO_FP, ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_ROUND})
      setOperationAction(Op, MVT::v1i64, Expand);
 
    // AArch64 doesn't have a direct vector ->f32 conversion instructions for
    // elements smaller than i32, so promote the input to i32 first.
    setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
    setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
 
    // Similarly, there is no direct i32 -> f64 vector conversion instruction.
    // Or, direct i32 -> f16 vector conversion.  Set it so custom, so the
    // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
    for (auto Op : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
                    ISD::STRICT_UINT_TO_FP})
      for (auto VT : {MVT::v2i32, MVT::v2i64, MVT::v4i32})
        setOperationAction(Op, VT, Custom);
 
    if (Subtarget->hasFullFP16()) {
      setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
      setOperationAction(ISD::ConstantFP, MVT::bf16, Legal);
 
      setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v16i8, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
    } else {
      // when AArch64 doesn't have fullfp16 support, promote the input
      // to i32 first.
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v16i8, MVT::v16i32);
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v16i8, MVT::v16i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
    }
 
    setOperationAction(ISD::CTLZ,       MVT::v1i64, Expand);
    setOperationAction(ISD::CTLZ,       MVT::v2i64, Expand);
    // CTLS (Count Leading Sign bits) - Legal for BHS types (8/16/32-bit
    // elements) No hardware support for 64-bit element vectors
    for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
                    MVT::v4i32})
      setOperationAction(ISD::CTLS, VT, Legal);
    setOperationAction(ISD::BITREVERSE, MVT::v8i8, Legal);
    setOperationAction(ISD::BITREVERSE, MVT::v16i8, Legal);
    setOperationAction(ISD::BITREVERSE, MVT::v2i32, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::v4i32, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::v2i64, Custom);
    for (auto VT : {MVT::v1i64, MVT::v2i64}) {
      setOperationAction(ISD::UMAX, VT, Custom);
      setOperationAction(ISD::SMAX, VT, Custom);
      setOperationAction(ISD::UMIN, VT, Custom);
      setOperationAction(ISD::SMIN, VT, Custom);
    }
 
    // Custom handling for some quad-vector types to detect MULL.
    setOperationAction(ISD::MUL, MVT::v8i16, Custom);
    setOperationAction(ISD::MUL, MVT::v4i32, Custom);
    setOperationAction(ISD::MUL, MVT::v2i64, Custom);
    setOperationAction(ISD::MUL, MVT::v4i16, Custom);
    setOperationAction(ISD::MUL, MVT::v2i32, Custom);
    setOperationAction(ISD::MUL, MVT::v1i64, Custom);
 
    // Saturates
    for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v1i64,
                    MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
      setOperationAction(ISD::SADDSAT, VT, Legal);
      setOperationAction(ISD::UADDSAT, VT, Legal);
      setOperationAction(ISD::SSUBSAT, VT, Legal);
      setOperationAction(ISD::USUBSAT, VT, Legal);
    }
 
    for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
                   MVT::v4i32}) {
      setOperationAction(ISD::AVGFLOORS, VT, Legal);
      setOperationAction(ISD::AVGFLOORU, VT, Legal);
      setOperationAction(ISD::AVGCEILS, VT, Legal);
      setOperationAction(ISD::AVGCEILU, VT, Legal);
      setOperationAction(ISD::ABDS, VT, Legal);
      setOperationAction(ISD::ABDU, VT, Legal);
    }
 
    // Vector reductions
    for (MVT VT : { MVT::v4f16, MVT::v2f32,
                    MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
      if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
        setOperationAction(ISD::VECREDUCE_FMAX, VT, Legal);
        setOperationAction(ISD::VECREDUCE_FMIN, VT, Legal);
        setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Legal);
        setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Legal);
 
        setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
      }
    }
    if (Subtarget->hasFullFP16())
      setOperationAction(ISD::VECREDUCE_FADD, MVT::v2f16, Custom);
 
    for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
                    MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
    }
    setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom);
    setOperationAction(ISD::VECREDUCE_AND, MVT::v2i64, Custom);
    setOperationAction(ISD::VECREDUCE_OR, MVT::v2i64, Custom);
    setOperationAction(ISD::VECREDUCE_XOR, MVT::v2i64, Custom);
 
    for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v2i32, MVT::v4i32})
      setOperationAction(ISD::SMULFIXSAT, VT, Custom);
 
    setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
    setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
    // Likewise, narrowing and extending vector loads/stores aren't handled
    // directly.
    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
      setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
 
      if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
        setOperationAction(ISD::MULHS, VT, Legal);
        setOperationAction(ISD::MULHU, VT, Legal);
      } else {
        setOperationAction(ISD::MULHS, VT, Expand);
        setOperationAction(ISD::MULHU, VT, Expand);
      }
      setOperationAction(ISD::SMUL_LOHI, VT, Expand);
      setOperationAction(ISD::UMUL_LOHI, VT, Expand);
 
      if (VT == MVT::v4i16 || VT == MVT::v8i16 || VT == MVT::v2i32 ||
          VT == MVT::v4i32 || VT == MVT::v2i64)
        setOperationAction(ISD::BSWAP, VT, Legal);
      else
        setOperationAction(ISD::BSWAP, VT, Expand);
 
      if (VT == MVT::v8i8 || VT == MVT::v16i8 || VT == MVT::v8i16 ||
          VT == MVT::v4i16 || VT == MVT::v2i32 || VT == MVT::v4i32)
        setOperationAction(ISD::CTTZ, VT, Custom);
      else
        setOperationAction(ISD::CTTZ, VT, Expand);
 
      for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
        setTruncStoreAction(VT, InnerVT, Expand);
        setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
      }
    }
 
    for (auto Op :
         {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL, ISD::FRINT, ISD::FTRUNC,
          ISD::FROUND, ISD::FROUNDEVEN, ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE,
          ISD::STRICT_FFLOOR, ISD::STRICT_FNEARBYINT, ISD::STRICT_FCEIL,
          ISD::STRICT_FRINT, ISD::STRICT_FTRUNC, ISD::STRICT_FROUND,
          ISD::STRICT_FROUNDEVEN}) {
      for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64})
        setOperationAction(Op, Ty, Legal);
      if (Subtarget->hasFullFP16())
        for (MVT Ty : {MVT::v4f16, MVT::v8f16})
          setOperationAction(Op, Ty, Legal);
    }
 
    // LRINT and LLRINT.
    for (auto Op : {ISD::LRINT, ISD::LLRINT}) {
      for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64})
        setOperationAction(Op, Ty, Custom);
      if (Subtarget->hasFullFP16())
        for (MVT Ty : {MVT::v4f16, MVT::v8f16})
          setOperationAction(Op, Ty, Custom);
    }
 
    setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
 
    setOperationAction(ISD::BITCAST, MVT::i2, Custom);
    setOperationAction(ISD::BITCAST, MVT::i4, Custom);
    setOperationAction(ISD::BITCAST, MVT::i8, Custom);
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
 
    setOperationAction(ISD::BITCAST, MVT::v2i8, Custom);
    setOperationAction(ISD::BITCAST, MVT::v2i16, Custom);
    setOperationAction(ISD::BITCAST, MVT::v4i8, Custom);
 
    setLoadExtAction(ISD::EXTLOAD, MVT::v2i32, MVT::v2i8, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v2i32, MVT::v2i8, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i32, MVT::v2i8, Custom);
    setLoadExtAction(ISD::EXTLOAD, MVT::v2i64, MVT::v2i8, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Custom);
    setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
    setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
    setLoadExtAction(ISD::EXTLOAD, MVT::v2i32, MVT::v2i16, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v2i32, MVT::v2i16, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i32, MVT::v2i16, Custom);
    setLoadExtAction(ISD::EXTLOAD, MVT::v2i64, MVT::v2i16, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Custom);
 
    // ADDP custom lowering
    for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
      setOperationAction(ISD::ADD, VT, Custom);
    // FADDP custom lowering
    for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 })
      setOperationAction(ISD::FADD, VT, Custom);
 
    // Two-way integer ``pure'' add reductions lower to a [SU]ADALP.
    {
      static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
                                        ISD::PARTIAL_REDUCE_UMLA};
      setPartialReduceMLAAction(MLAOps, MVT::v4i16, MVT::v8i8, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v2i32, MVT::v4i16, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v1i64, MVT::v2i32, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v8i16, MVT::v16i8, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v4i32, MVT::v8i16, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v4i32, Custom);
    }
 
    if (Subtarget->hasDotProd()) {
      static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
                                        ISD::PARTIAL_REDUCE_UMLA};
 
      setPartialReduceMLAAction(MLAOps, MVT::v4i32, MVT::v16i8, Legal);
      setPartialReduceMLAAction(MLAOps, MVT::v2i32, MVT::v8i8, Legal);
      setPartialReduceMLAAction(MLAOps, MVT::v2i32, MVT::v16i8, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v16i8, Custom);
 
      // v2i64/v16i8 SUMLA always reduces to v4i32 SUMLA via
      // LowerPARTIAL_REDUCE_MLA, regardless of i8mm; v2i32/v16i8 SUMLA goes
      // through the same widen-to-v4i32 path as the SMLA/UMLA cases above.
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v2i64,
                                MVT::v16i8, Custom);
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v2i32,
                                MVT::v16i8, Custom);
 
      if (Subtarget->hasMatMulInt8()) {
        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v4i32,
                                  MVT::v16i8, Legal);
        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v2i32,
                                  MVT::v8i8, Legal);
      } else {
        // Native dotprod without i8mm: lower SUMLA to two UDOT products in
        // LowerPARTIAL_REDUCE_MLA.
        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v4i32,
                                  MVT::v16i8, Custom);
        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v2i32,
                                  MVT::v8i8, Custom);
      }
    }
 
    if (Subtarget->hasF16F32DOT() || Subtarget->hasFP16FML()) {
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::v2f32,
                                MVT::v4f16, Legal);
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::v4f32,
                                MVT::v8f16, Legal);
    }
 
    if (Subtarget->hasBF16())
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::v4f32,
                                MVT::v8bf16, Legal);
 
    setOperationAction(ISD::CLMUL, MVT::i8, Custom);
    setOperationAction(ISD::CLMUL, {MVT::v8i8, MVT::v16i8}, Legal);
    if (Subtarget->hasAES()) {
      setOperationAction(ISD::CLMUL, {MVT::i16, MVT::i32, MVT::i64, MVT::v4i32},
                         Custom);
      setOperationAction(ISD::CLMUL, {MVT::v1i64, MVT::v2i64}, Legal);
      setOperationAction(ISD::CLMULH, {MVT::v1i64, MVT::v2i64}, Legal);
    }
 
  } else /* !isNeonAvailable */ {
    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
        setOperationAction(Op, VT, Expand);
 
      if (VT.is128BitVector() || VT.is64BitVector()) {
        setOperationAction(ISD::LOAD, VT, Legal);
        setOperationAction(ISD::STORE, VT, Legal);
        setOperationAction(ISD::BITCAST, VT,
                           Subtarget->isLittleEndian() ? Legal : Expand);
      }
      for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
        setTruncStoreAction(VT, InnerVT, Expand);
        setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
      }
    }
  }
 
  for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
    setOperationAction(ISD::TRUNCATE_SSAT_S, VT, Legal);
    setOperationAction(ISD::TRUNCATE_SSAT_U, VT, Legal);
    setOperationAction(ISD::TRUNCATE_USAT_U, VT, Legal);
  }
 
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
 
  // FIXME: Move lowering for more nodes here if those are common between
  // SVE and SME.
  if (Subtarget->isSVEorStreamingSVEAvailable()) {
    for (auto VT :
         {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
      setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
    }
    for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) {
      setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
                         Custom);
      setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
      setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
    }
 
    if (Subtarget->hasSVE2() && Subtarget->isSVEAvailable()) {
      for (MVT VT : {MVT::nxv16i1, MVT::nxv8i1})
        setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
 
      for (MVT VT : {MVT::v16i1, MVT::v8i1, MVT::v16i8, MVT::v8i8})
        setOperationAction(ISD::VECTOR_MATCH, VT, Custom);
    }
 
    setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv1i1, Custom);
 
    if (Subtarget->isSVEorStreamingSVEAvailable() &&
        (Subtarget->hasSVE2p1() || Subtarget->hasSME2()))
      setOperationAction(ISD::GET_ACTIVE_LANE_MASK, MVT::nxv32i1, Custom);
 
    for (auto VT : {MVT::v16i8, MVT::v8i8, MVT::v4i16, MVT::v2i32})
      setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Custom);
 
    for (auto VT : {MVT::v8f16, MVT::v4f32, MVT::v2f64})
      setOperationAction(ISD::FMA, VT, Custom);
  }
 
  if (Subtarget->isSVEorStreamingSVEAvailable()) {
    for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
      setOperationAction(ISD::BITREVERSE, VT, Custom);
      setOperationAction(ISD::BSWAP, VT, Custom);
      setOperationAction(ISD::CTLZ, VT, Custom);
      setOperationAction(ISD::CTPOP, VT, Custom);
      setOperationAction(ISD::CTTZ, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::UINT_TO_FP, VT, Custom);
      setOperationAction(ISD::SINT_TO_FP, VT, Custom);
      setOperationAction(ISD::FP_TO_UINT, VT, Custom);
      setOperationAction(ISD::FP_TO_SINT, VT, Custom);
      setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
      setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::MSTORE, VT, Legal);
      setOperationAction(ISD::MUL, VT, Custom);
      setOperationAction(ISD::MULHS, VT, Custom);
      setOperationAction(ISD::MULHU, VT, Custom);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Custom);
      setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SETCC, VT, Custom);
      setOperationAction(ISD::SDIV, VT, Custom);
      setOperationAction(ISD::UDIV, VT, Custom);
      setOperationAction(ISD::SMIN, VT, Custom);
      setOperationAction(ISD::UMIN, VT, Custom);
      setOperationAction(ISD::SMAX, VT, Custom);
      setOperationAction(ISD::UMAX, VT, Custom);
      setOperationAction(ISD::SHL, VT, Custom);
      setOperationAction(ISD::SRL, VT, Custom);
      setOperationAction(ISD::SRA, VT, Custom);
      setOperationAction(ISD::ABS, VT, Custom);
      setOperationAction(ISD::ABDS, VT, Custom);
      setOperationAction(ISD::ABDU, VT, Custom);
      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
 
      setOperationAction(ISD::UMUL_LOHI, VT, Expand);
      setOperationAction(ISD::SMUL_LOHI, VT, Expand);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction(ISD::ROTL, VT, Expand);
      setOperationAction(ISD::ROTR, VT, Expand);
 
      setOperationAction(ISD::SADDSAT, VT, Legal);
      setOperationAction(ISD::UADDSAT, VT, Legal);
      setOperationAction(ISD::SSUBSAT, VT, Legal);
      setOperationAction(ISD::USUBSAT, VT, Legal);
      setOperationAction(ISD::UREM, VT, Expand);
      setOperationAction(ISD::SREM, VT, Expand);
      setOperationAction(ISD::SDIVREM, VT, Expand);
      setOperationAction(ISD::UDIVREM, VT, Expand);
 
      setOperationAction(ISD::AVGFLOORS, VT, Custom);
      setOperationAction(ISD::AVGFLOORU, VT, Custom);
      setOperationAction(ISD::AVGCEILS, VT, Custom);
      setOperationAction(ISD::AVGCEILU, VT, Custom);
 
      setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, VT, Custom);
      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
 
      if (!Subtarget->isLittleEndian())
        setOperationAction(ISD::BITCAST, VT, Custom);
 
      if (Subtarget->hasSVE2() ||
          (Subtarget->hasSME() && Subtarget->isStreaming()))
        // For SLI/SRI.
        setOperationAction(ISD::OR, VT, Custom);
    }
 
    for (auto VT : {MVT::nxv4i32, MVT::nxv2i64}) {
      setOperationAction(ISD::MASKED_UDIV, VT, Legal);
      setOperationAction(ISD::MASKED_SDIV, VT, Legal);
    }
 
    // Illegal unpacked integer vector types.
    for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
    }
 
    // Type legalize unpacked bitcasts.
    for (auto VT : {MVT::nxv2i16, MVT::nxv4i16, MVT::nxv2i32})
      setOperationAction(ISD::BITCAST, VT, Custom);
 
    for (auto VT :
         { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
           MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
      setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
 
    // Promote predicate as counter load/stores to standard predicates.
    setOperationPromotedToType(ISD::LOAD, MVT::aarch64svcount, MVT::nxv16i1);
    setOperationPromotedToType(ISD::STORE, MVT::aarch64svcount, MVT::nxv16i1);
 
    // Predicate as counter legalization actions.
    setOperationAction(ISD::SELECT, MVT::aarch64svcount, Custom);
    setOperationAction(ISD::SELECT_CC, MVT::aarch64svcount, Expand);
 
    for (auto VT :
         {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SETCC, VT, Custom);
      setOperationAction(ISD::TRUNCATE, VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
 
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
 
      // There are no legal MVT::nxv16f## based types.
      if (VT != MVT::nxv16i1) {
        setOperationAction(ISD::FP_TO_SINT, VT, Custom);
        setOperationAction(ISD::FP_TO_UINT, VT, Custom);
        setOperationAction(ISD::SINT_TO_FP, VT, Custom);
        setOperationAction(ISD::UINT_TO_FP, VT, Custom);
      }
    }
 
    // NEON doesn't support masked loads/stores, but SME and SVE do.
    for (auto VT :
         {MVT::v4f16, MVT::v8f16, MVT::v4bf16, MVT::v8bf16, MVT::v2f32,
          MVT::v4f32, MVT::v1f64, MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16,
          MVT::v8i16, MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::MSTORE, VT, Custom);
    }
 
    // Firstly, exclude all scalable vector extending loads/truncating stores,
    // include both integer and floating scalable vector.
    for (MVT VT : MVT::scalable_vector_valuetypes()) {
      for (MVT InnerVT : MVT::scalable_vector_valuetypes()) {
        setTruncStoreAction(VT, InnerVT, Expand);
        setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
      }
    }
 
    // Then, selectively enable those which we directly support.
    setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i8, Legal);
    setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i16, Legal);
    setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i32, Legal);
    setTruncStoreAction(MVT::nxv4i32, MVT::nxv4i8, Legal);
    setTruncStoreAction(MVT::nxv4i32, MVT::nxv4i16, Legal);
    setTruncStoreAction(MVT::nxv8i16, MVT::nxv8i8, Legal);
    for (auto Op : {ISD::ZEXTLOAD, ISD::SEXTLOAD, ISD::EXTLOAD}) {
      setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i8, Legal);
      setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i16, Legal);
      setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i32, Legal);
      setLoadExtAction(Op, MVT::nxv4i32, MVT::nxv4i8, Legal);
      setLoadExtAction(Op, MVT::nxv4i32, MVT::nxv4i16, Legal);
      setLoadExtAction(Op, MVT::nxv8i16, MVT::nxv8i8, Legal);
    }
 
    // SVE supports truncating stores of 64 and 128-bit vectors
    setTruncStoreAction(MVT::v2i64, MVT::v2i8, Custom);
    setTruncStoreAction(MVT::v2i64, MVT::v2i16, Custom);
    setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom);
    setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom);
 
    for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
                    MVT::nxv4f32, MVT::nxv2f64}) {
      setOperationAction(ISD::BITCAST, VT, Custom);
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SETCC, VT, Custom);
      setOperationAction({ISD::FADD, ISD::STRICT_FADD}, VT, Custom);
      setOperationAction(ISD::FCANONICALIZE, VT, Custom);
      setOperationAction(ISD::FCOPYSIGN, VT, Custom);
      setOperationAction({ISD::FDIV, ISD::STRICT_FDIV}, VT, Custom);
      setOperationAction({ISD::FMA, ISD::STRICT_FMA}, VT, Custom);
      setOperationAction({ISD::FMAXIMUM, ISD::STRICT_FMAXIMUM}, VT, Custom);
      setOperationAction({ISD::FMAXNUM, ISD::STRICT_FMAXNUM}, VT, Custom);
      setOperationAction(ISD::FMAXNUM_IEEE, VT, Custom);
      setOperationAction({ISD::FMINIMUM, ISD::STRICT_FMINIMUM}, VT, Custom);
      setOperationAction({ISD::FMINNUM, ISD::STRICT_FMINNUM}, VT, Custom);
      setOperationAction(ISD::FMINNUM_IEEE, VT, Custom);
      setOperationAction({ISD::FMUL, ISD::STRICT_FMUL}, VT, Custom);
      setOperationAction(ISD::FNEG, VT, Custom);
      setOperationAction({ISD::FSUB, ISD::STRICT_FSUB}, VT, Custom);
      setOperationAction({ISD::FCEIL, ISD::STRICT_FCEIL}, VT, Custom);
      setOperationAction({ISD::FFLOOR, ISD::STRICT_FFLOOR}, VT, Custom);
      setOperationAction({ISD::FNEARBYINT, ISD::STRICT_FNEARBYINT}, VT, Custom);
      setOperationAction(ISD::FRINT, VT, Custom);
      setOperationAction(ISD::LRINT, VT, Custom);
      setOperationAction(ISD::LLRINT, VT, Custom);
      setOperationAction({ISD::FROUND, ISD::STRICT_FROUND}, VT, Custom);
      setOperationAction({ISD::FROUNDEVEN, ISD::STRICT_FROUNDEVEN}, VT, Custom);
      setOperationAction({ISD::FTRUNC, ISD::STRICT_FTRUNC}, VT, Custom);
      setOperationAction({ISD::FSQRT, ISD::STRICT_FSQRT}, VT, Custom);
      setOperationAction(ISD::FABS, VT, Custom);
      setOperationAction(ISD::FP_EXTEND, VT, Custom);
      setOperationAction(ISD::FP_ROUND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom);
      setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Custom);
      setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
 
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction({ISD::FREM, ISD::STRICT_FREM}, VT, Expand);
      setOperationAction({ISD::FPOW, ISD::STRICT_FPOW}, VT, Expand);
      setOperationAction({ISD::FPOWI, ISD::STRICT_FPOWI}, VT, Expand);
      setOperationAction(ISD::FCBRT, VT, Expand);
      setOperationAction({ISD::FCOS, ISD::STRICT_FCOS}, VT, Expand);
      setOperationAction({ISD::FSIN, ISD::STRICT_FSIN}, VT, Expand);
      setOperationAction(ISD::FSINCOS, VT, Expand);
      setOperationAction({ISD::FTAN, ISD::STRICT_FTAN}, VT, Expand);
      setOperationAction({ISD::FACOS, ISD::STRICT_FACOS}, VT, Expand);
      setOperationAction({ISD::FASIN, ISD::STRICT_FASIN}, VT, Expand);
      setOperationAction({ISD::FATAN, ISD::STRICT_FATAN}, VT, Expand);
      setOperationAction({ISD::FATAN2, ISD::STRICT_FATAN2}, VT, Expand);
      setOperationAction({ISD::FCOSH, ISD::STRICT_FCOSH}, VT, Expand);
      setOperationAction({ISD::FSINH, ISD::STRICT_FSINH}, VT, Expand);
      setOperationAction({ISD::FTANH, ISD::STRICT_FTANH}, VT, Expand);
      setOperationAction({ISD::FEXP, ISD::STRICT_FEXP}, VT, Expand);
      setOperationAction({ISD::FEXP2, ISD::STRICT_FEXP2}, VT, Expand);
      setOperationAction(ISD::FEXP10, VT, Expand);
      setOperationAction({ISD::FLOG, ISD::STRICT_FLOG}, VT, Expand);
      setOperationAction({ISD::FLOG2, ISD::STRICT_FLOG2}, VT, Expand);
      setOperationAction({ISD::FLOG10, ISD::STRICT_FLOG10}, VT, Expand);
 
      // TODO: These require custom lowering.
      setOperationAction(ISD::STRICT_FLDEXP, VT, Expand);
      setOperationAction(ISD::STRICT_FRINT, VT, Expand);
      setOperationAction(ISD::STRICT_PSEUDO_FMIN, VT, Expand);
      setOperationAction(ISD::STRICT_PSEUDO_FMAX, VT, Expand);
      setOperationAction(ISD::STRICT_LROUND, VT, Expand);
      setOperationAction(ISD::STRICT_LLROUND, VT, Expand);
      setOperationAction(ISD::STRICT_LRINT, VT, Expand);
      setOperationAction(ISD::STRICT_LLRINT, VT, Expand);
      setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Expand);
      setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Expand);
      setOperationAction(ISD::STRICT_SINT_TO_FP, VT, Expand);
      setOperationAction(ISD::STRICT_UINT_TO_FP, VT, Expand);
      setOperationAction(ISD::STRICT_FP_ROUND, VT, Expand);
      setOperationAction(ISD::STRICT_FP_EXTEND, VT, Expand);
      setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
      setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
 
      setCondCodeAction(ISD::SETO, VT, Expand);
      setCondCodeAction(ISD::SETOLT, VT, Expand);
      setCondCodeAction(ISD::SETLT, VT, Expand);
      setCondCodeAction(ISD::SETOLE, VT, Expand);
      setCondCodeAction(ISD::SETLE, VT, Expand);
      setCondCodeAction(ISD::SETULT, VT, Expand);
      setCondCodeAction(ISD::SETULE, VT, Expand);
      setCondCodeAction(ISD::SETUGE, VT, Expand);
      setCondCodeAction(ISD::SETUGT, VT, Expand);
      setCondCodeAction(ISD::SETUEQ, VT, Expand);
      setCondCodeAction(ISD::SETONE, VT, Expand);
    }
 
    for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
      setOperationAction(ISD::BITCAST, VT, Custom);
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
      setOperationAction(ISD::FABS, VT, Custom);
      setOperationAction(ISD::FCOPYSIGN, VT, Custom);
      setOperationAction(ISD::FNEG, VT, Custom);
      setOperationAction(ISD::FP_EXTEND, VT, Custom);
      setOperationAction(ISD::FP_ROUND, VT, Custom);
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Custom);
      setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Custom);
    }
 
    if (Subtarget->hasSVEB16B16() &&
        Subtarget->isNonStreamingSVEorSME2Available()) {
      // Note: Use SVE for bfloat16 operations when +sve-b16b16 is available.
      for (auto VT : {MVT::v4bf16, MVT::v8bf16, MVT::nxv2bf16, MVT::nxv4bf16,
                      MVT::nxv8bf16}) {
        setOperationAction(ISD::FADD, VT, Custom);
        setOperationAction(ISD::FMA, VT, Custom);
        setOperationAction(ISD::FMAXIMUM, VT, Custom);
        setOperationAction(ISD::FMAXNUM, VT, Custom);
        setOperationAction(ISD::FMINIMUM, VT, Custom);
        setOperationAction(ISD::FMINNUM, VT, Custom);
        setOperationAction(ISD::FMUL, VT, Custom);
        setOperationAction(ISD::FSUB, VT, Custom);
      }
    }
 
    for (auto Opcode :
         {ISD::FCEIL, ISD::FDIV, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
          ISD::FROUND, ISD::FROUNDEVEN, ISD::FSQRT, ISD::FTRUNC, ISD::SETCC,
          ISD::VECREDUCE_FADD, ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMAXIMUM,
          ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMINIMUM}) {
      setOperationPromotedToType(Opcode, MVT::nxv2bf16, MVT::nxv2f32);
      setOperationPromotedToType(Opcode, MVT::nxv4bf16, MVT::nxv4f32);
      setOperationPromotedToType(Opcode, MVT::nxv8bf16, MVT::nxv8f32);
    }
 
    if (!Subtarget->hasSVEB16B16() ||
        !Subtarget->isNonStreamingSVEorSME2Available()) {
      for (MVT VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
        MVT PromotedVT = VT.changeVectorElementType(MVT::f32);
        setOperationPromotedToType(ISD::FADD, VT, PromotedVT);
        setOperationPromotedToType(ISD::FMA, VT, PromotedVT);
        setOperationPromotedToType(ISD::FMAXIMUM, VT, PromotedVT);
        setOperationPromotedToType(ISD::FMAXNUM, VT, PromotedVT);
        setOperationPromotedToType(ISD::FMINIMUM, VT, PromotedVT);
        setOperationPromotedToType(ISD::FMINNUM, VT, PromotedVT);
        setOperationPromotedToType(ISD::FSUB, VT, PromotedVT);
 
        if (VT != MVT::nxv2bf16 && Subtarget->hasBF16())
          setOperationAction(ISD::FMUL, VT, Custom);
        else
          setOperationPromotedToType(ISD::FMUL, VT, PromotedVT);
      }
 
      if (Subtarget->hasBF16() && Subtarget->isNeonAvailable())
        setOperationAction(ISD::FMUL, MVT::v8bf16, Custom);
    }
 
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
 
    // A number of operations like MULH and integer divides are not supported by
    // NEON but are available in SVE.
    for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
                    MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
      setOperationAction(ISD::MASKED_SDIV, VT, Custom);
      setOperationAction(ISD::MASKED_UDIV, VT, Custom);
      setOperationAction(ISD::SDIV, VT, Custom);
      setOperationAction(ISD::UDIV, VT, Custom);
      setOperationAction(ISD::MULHS, VT, Custom);
      setOperationAction(ISD::MULHU, VT, Custom);
    }
 
    for (auto VT : {MVT::v4i16, MVT::v8i16, MVT::v2i32, MVT::v4i32, MVT::v1i64,
                    MVT::v2i64})
      setOperationAction(ISD::BITREVERSE, VT, Custom);
 
    // NEON doesn't support 64-bit vector integer muls, but SVE does.
    setOperationAction(ISD::MUL, MVT::v1i64, Custom);
    setOperationAction(ISD::MUL, MVT::v2i64, Custom);
 
    // NEON doesn't support 128-bit [s|u][min|max] operations, but SVE does.
    setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom);
    setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
    setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
    setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
 
    for (auto VT : {MVT::v1i64, MVT::v2i64}) {
      setOperationAction(ISD::CTLZ, VT, Custom);
      setOperationAction(ISD::CTTZ, VT, Custom);
    }
 
    // NOTE: Currently this has to happen after computeRegisterProperties rather
    // than the preferred option of combining it with the addRegisterClass call.
    if (Subtarget->useSVEForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
        if (useSVEForFixedLengthVectorVT(
                VT, /*OverrideNEON=*/!Subtarget->isNeonAvailable()))
          addTypeForFixedLengthSVE(VT);
      }
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
        if (useSVEForFixedLengthVectorVT(
                VT, /*OverrideNEON=*/!Subtarget->isNeonAvailable()))
          addTypeForFixedLengthSVE(VT);
      }
 
      // 64bit results can mean a bigger than NEON input.
      for (auto VT : {MVT::v8i8, MVT::v4i16})
        setOperationAction(ISD::TRUNCATE, VT, Custom);
      setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
 
      // 128bit results imply a bigger than NEON input.
      for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
        setOperationAction(ISD::TRUNCATE, VT, Custom);
      for (auto VT : {MVT::v8f16, MVT::v4f32, MVT::v8bf16})
        setOperationAction(ISD::FP_ROUND, VT, Custom);
 
      // These operations are not supported on NEON but SVE can do them.
      setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
      setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
      setOperationAction(ISD::SMIN, MVT::v2i64, Custom);
      setOperationAction(ISD::UMAX, MVT::v1i64, Custom);
      setOperationAction(ISD::UMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::UMIN, MVT::v1i64, Custom);
      setOperationAction(ISD::UMIN, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
 
      // Int operations with no NEON support.
      for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
                      MVT::v4i32, MVT::v2i64}) {
        setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
        setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
        setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
      }
 
      // Use SVE for vectors with more than 2 elements.
      for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
        setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
    }
 
    setOperationPromotedToType(ISD::VECTOR_SPLICE_LEFT, MVT::nxv2i1,
                               MVT::nxv2i64);
    setOperationPromotedToType(ISD::VECTOR_SPLICE_RIGHT, MVT::nxv2i1,
                               MVT::nxv2i64);
    setOperationPromotedToType(ISD::VECTOR_SPLICE_LEFT, MVT::nxv4i1,
                               MVT::nxv4i32);
    setOperationPromotedToType(ISD::VECTOR_SPLICE_RIGHT, MVT::nxv4i1,
                               MVT::nxv4i32);
    setOperationPromotedToType(ISD::VECTOR_SPLICE_LEFT, MVT::nxv8i1,
                               MVT::nxv8i16);
    setOperationPromotedToType(ISD::VECTOR_SPLICE_RIGHT, MVT::nxv8i1,
                               MVT::nxv8i16);
    setOperationPromotedToType(ISD::VECTOR_SPLICE_LEFT, MVT::nxv16i1,
                               MVT::nxv16i8);
    setOperationPromotedToType(ISD::VECTOR_SPLICE_RIGHT, MVT::nxv16i1,
                               MVT::nxv16i8);
 
    setOperationAction(ISD::VSCALE, MVT::i32, Custom);
 
    for (auto VT : {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1})
      setOperationAction(
          {ISD::INTRINSIC_WO_CHAIN, ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON},
          VT, Custom);
 
    // Without SubReg Liveness the multi-vector instructions can introduce
    // unnecessary COPY and/or MOVPFRX instructions.
    if (Subtarget->enableSubRegLiveness() &&
        (Subtarget->hasSVE2p1() ||
         (Subtarget->hasSME2() && Subtarget->isStreaming()))) {
 
      for (unsigned Opcode : {ISD::LOAD, ISD::STORE}) {
        // 2x multi-vector load/stores
        setOperationAction(Opcode, MVT::nxv32i8, Custom);
        setOperationAction(Opcode, MVT::nxv16i16, Custom);
        setOperationAction(Opcode, MVT::nxv8i32, Custom);
        setOperationAction(Opcode, MVT::nxv4i64, Custom);
        setOperationAction(Opcode, MVT::nxv16f16, Custom);
        setOperationAction(Opcode, MVT::nxv8f32, Custom);
        setOperationAction(Opcode, MVT::nxv4f64, Custom);
        setOperationAction(Opcode, MVT::nxv16bf16, Custom);
 
        // 4x multi-vector load/stores
        setOperationAction(Opcode, MVT::nxv64i8, Custom);
        setOperationAction(Opcode, MVT::nxv32i16, Custom);
        setOperationAction(Opcode, MVT::nxv16i32, Custom);
        setOperationAction(Opcode, MVT::nxv8i64, Custom);
        setOperationAction(Opcode, MVT::nxv32f16, Custom);
        setOperationAction(Opcode, MVT::nxv16f32, Custom);
        setOperationAction(Opcode, MVT::nxv8f64, Custom);
        setOperationAction(Opcode, MVT::nxv32bf16, Custom);
      }
    }
  }
 
  // Handle partial reduction operations
  if (Subtarget->isSVEorStreamingSVEAvailable()) {
    // Mark known legal pairs as 'Legal' (these will expand to UDOT or SDOT).
    // Other pairs will default to 'Expand'.
    static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
                                      ISD::PARTIAL_REDUCE_UMLA};
    setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv8i16, Legal);
    setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv16i8, Legal);
 
    setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom);
 
    if (Subtarget->hasMatMulInt8()) {
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::nxv4i32,
                                MVT::nxv16i8, Legal);
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::nxv2i64,
                                MVT::nxv16i8, Custom);
    }
 
    if (Subtarget->hasSVE2() || Subtarget->hasSME()) {
      // Wide add types
      setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv4i32, Legal);
      setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv8i16, Legal);
      setPartialReduceMLAAction(MLAOps, MVT::nxv8i16, MVT::nxv16i8, Legal);
 
      setOperationAction(ISD::CLMUL, {MVT::nxv16i8, MVT::nxv4i32}, Legal);
      setOperationAction(ISD::CLMUL, MVT::nxv8i16, Custom);
 
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::nxv4f32,
                                MVT::nxv8f16, Legal);
 
      // We can use SVE2p1 fdot or SVE2 fmlalb/t to emulate the fixed-length
      // variant (unless NEON fdot is natively available).
      if (!Subtarget->isNeonAvailable() ||
          (!Subtarget->hasF16F32DOT() && !Subtarget->hasFP16FML())) {
        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::v4f32,
                                  MVT::v8f16, Custom);
        setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::v2f32,
                                  MVT::v4f16, Custom);
      }
    }
 
    // Map generic PEXT/PDEP to SVE2 bitperm BEXT/BDEP instructions.
    if (Subtarget->hasSVEBitPerm() &&
        (Subtarget->isSVEAvailable() ||
         (Subtarget->isSVEorStreamingSVEAvailable() &&
          Subtarget->hasSSVE_BitPerm()))) {
      for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
        setOperationAction({ISD::PEXT, ISD::PDEP}, VT, Custom);
      }
      setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i32, Custom);
      setOperationAction({ISD::PEXT, ISD::PDEP}, MVT::i64, Custom);
    }
 
    if (Subtarget->hasBF16())
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, MVT::nxv4f32,
                                MVT::nxv8bf16, Legal);
    setOperationAction(ISD::CLMUL, MVT::nxv2i64, Custom);
  }
 
  if (Subtarget->isSVEAvailable() ||
      (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) {
    // We can lower types that have <vscale x {2|4}> elements to compact.
    for (auto VT :
         {MVT::nxv4i32, MVT::nxv2i64, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv2f64})
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
 
    // If we have SVE, we can use SVE logic for legal NEON vectors in the lowest
    // bits of the SVE register.
    for (auto VT : {MVT::v2i32, MVT::v4i32, MVT::v2i64, MVT::v2f32, MVT::v4f32,
                    MVT::v2f64})
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
 
    if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) {
      // With +sve2p2/+sme2p2 the full range of vector types are supported.
      for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
        setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
 
      for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v4f16,
                      MVT::v8f16, MVT::v4bf16, MVT::v8bf16})
        setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
    } else {
      // Promote v4i16/f16 to v4i32/f32 as the SVE container for v4i16 is nxv8,
      // which is not supported with for compact (with only +sve).
      setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4bf16, MVT::v4i16);
      setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4f16, MVT::v4i16);
      setOperationPromotedToType(ISD::VECTOR_COMPRESS, MVT::v4i16, MVT::v4i32);
    }
  }
 
  // Handle non-aliasing elements mask
  if (Subtarget->hasSVE2() ||
      (Subtarget->hasSME() && Subtarget->isStreaming())) {
    // FIXME: Support wider fixed-length types when msve-vector-bits is used.
    for (auto VT : {MVT::v2i32, MVT::v4i16, MVT::v8i8, MVT::v16i8}) {
      setOperationAction(ISD::LOOP_DEPENDENCE_RAW_MASK, VT, Custom);
      setOperationAction(ISD::LOOP_DEPENDENCE_WAR_MASK, VT, Custom);
    }
    for (auto VT : {MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1, MVT::nxv16i1}) {
      setOperationAction(ISD::LOOP_DEPENDENCE_RAW_MASK, VT, Custom);
      setOperationAction(ISD::LOOP_DEPENDENCE_WAR_MASK, VT, Custom);
    }
  }
 
  // Handle operations that are only available in non-streaming SVE mode.
  if (Subtarget->isSVEAvailable()) {
    for (auto VT : {MVT::nxv16i8,  MVT::nxv8i16, MVT::nxv4i32,  MVT::nxv2i64,
                    MVT::nxv2f16,  MVT::nxv4f16, MVT::nxv8f16,  MVT::nxv2f32,
                    MVT::nxv4f32,  MVT::nxv2f64, MVT::nxv2bf16, MVT::nxv4bf16,
                    MVT::nxv8bf16, MVT::v4f16,   MVT::v8f16,    MVT::v4bf16,
                    MVT::v8bf16,   MVT::v2f32,   MVT::v4f32,    MVT::v1f64,
                    MVT::v2f64,    MVT::v8i8,    MVT::v16i8,    MVT::v4i16,
                    MVT::v8i16,    MVT::v2i32,   MVT::v4i32,    MVT::v1i64,
                    MVT::v2i64}) {
      setOperationAction(ISD::MGATHER, VT, Custom);
      setOperationAction(ISD::MSCATTER, VT, Custom);
    }
 
    for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
                    MVT::nxv4f32, MVT::nxv2f64, MVT::v4f16, MVT::v8f16,
                    MVT::v2f32, MVT::v4f32, MVT::v2f64})
      setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
 
    for (auto VT : {MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64,
                    MVT::nxv2f32, MVT::nxv2f64, MVT::nxv4i8, MVT::nxv4i16,
                    MVT::nxv4i32, MVT::nxv4f32}) {
      // Use a custom lowering for masked stores that could be a supported
      // compressing store. Note: These types still use the normal (Legal)
      // lowering for non-compressing masked stores.
      setOperationAction(ISD::MSTORE, VT, Custom);
    }
 
    // Histcnt is SVE2 only
    if (Subtarget->hasSVE2()) {
      setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::nxv4i32,
                         Custom);
      setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::nxv2i64,
                         Custom);
 
      static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
                                        ISD::PARTIAL_REDUCE_UMLA};
      // Must be lowered to SVE instructions.
      setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v4i32, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v8i16, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v2i64, MVT::v16i8, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v4i32, MVT::v8i16, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v4i32, MVT::v16i8, Custom);
      setPartialReduceMLAAction(MLAOps, MVT::v8i16, MVT::v16i8, Custom);
    }
  }
 
  if (Subtarget->hasMOPS() && Subtarget->hasMTE()) {
    // Only required for llvm.aarch64.mops.memset.tag
    setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
  }
 
  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 
  if (Subtarget->hasSVE()) {
    setOperationAction(ISD::FLDEXP, MVT::f64, Custom);
    setOperationAction(ISD::FLDEXP, MVT::f32, Custom);
    setOperationAction(ISD::FLDEXP, MVT::f16, Custom);
    setOperationAction(ISD::FLDEXP, MVT::bf16, Custom);
  }
 
  PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
 
  IsStrictFPEnabled = true;
  setMaxAtomicSizeInBitsSupported(128);
 
  // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined.  MinGW has
  // it, but it's just a wrapper around ldexp.
  if (Subtarget->isTargetWindows()) {
    for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
      if (isOperationExpand(Op, MVT::f32))
        setOperationAction(Op, MVT::f32, Promote);
  }
 
  // LegalizeDAG currently can't expand fp16/bf16 LDEXP/FREXP on targets where
  // i16 isn't legal.
  for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP}) {
    if (isOperationExpand(Op, MVT::f16))
      setOperationAction(Op, MVT::f16, Promote);
    if (isOperationExpand(Op, MVT::bf16))
      setOperationAction(Op, MVT::bf16, Promote);
  }
}
 
const AArch64TargetMachine &AArch64TargetLowering::getTM() const {
  return static_cast<const AArch64TargetMachine &>(getTargetMachine());
}
 
void AArch64TargetLowering::addTypeForNEON(MVT VT) {
  assert(VT.isVector() && "VT should be a vector type");
 
  if (VT.isFloatingPoint()) {
    MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
    setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
    setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
  }
 
  // Mark vector float intrinsics as expand.
  if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
    setOperationAction(ISD::FSIN, VT, Expand);
    setOperationAction(ISD::FCOS, VT, Expand);
    setOperationAction(ISD::FTAN, VT, Expand);
    setOperationAction(ISD::FASIN, VT, Expand);
    setOperationAction(ISD::FACOS, VT, Expand);
    setOperationAction(ISD::FATAN, VT, Expand);
    setOperationAction(ISD::FATAN2, VT, Expand);
    setOperationAction(ISD::FSINH, VT, Expand);
    setOperationAction(ISD::FCOSH, VT, Expand);
    setOperationAction(ISD::FTANH, VT, Expand);
    setOperationAction(ISD::FPOW, VT, Expand);
    setOperationAction(ISD::FCBRT, VT, Expand);
    setOperationAction(ISD::FLOG, VT, Expand);
    setOperationAction(ISD::FLOG2, VT, Expand);
    setOperationAction(ISD::FLOG10, VT, Expand);
    setOperationAction(ISD::FEXP, VT, Expand);
    setOperationAction(ISD::FEXP2, VT, Expand);
    setOperationAction(ISD::FEXP10, VT, Expand);
  }
 
  // But we do support custom-lowering for FCOPYSIGN.
  if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
      ((VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v4f16 ||
        VT == MVT::v8f16) &&
       Subtarget->hasFullFP16()))
    setOperationAction(ISD::FCOPYSIGN, VT, Custom);
 
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
  setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
  setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
  setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
  setOperationAction(ISD::SRA, VT, Custom);
  setOperationAction(ISD::SRL, VT, Custom);
  setOperationAction(ISD::SHL, VT, Custom);
  setOperationAction(ISD::OR, VT, Custom);
  setOperationAction(ISD::SETCC, VT, Custom);
  setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
 
  setOperationAction(ISD::SELECT, VT, Expand);
  setOperationAction(ISD::SELECT_CC, VT, Expand);
  setOperationAction(ISD::VSELECT, VT, Expand);
  for (MVT InnerVT : MVT::all_valuetypes())
    setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
 
  // CNT supports only B element sizes, then use UADDLP to widen.
  if (VT != MVT::v8i8 && VT != MVT::v16i8)
    setOperationAction(ISD::CTPOP, VT, Custom);
 
  setOperationAction(ISD::UDIV, VT, Expand);
  setOperationAction(ISD::SDIV, VT, Expand);
  setOperationAction(ISD::UREM, VT, Expand);
  setOperationAction(ISD::SREM, VT, Expand);
  setOperationAction(ISD::FREM, VT, Expand);
 
  for (unsigned Opcode :
       {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
        ISD::FP_TO_UINT_SAT, ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
    setOperationAction(Opcode, VT, Custom);
 
  if (!VT.isFloatingPoint())
    setOperationAction(ISD::ABS, VT, Legal);
 
  // [SU][MIN|MAX] are available for all NEON types apart from i64.
  if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
    for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
      setOperationAction(Opcode, VT, Legal);
 
  // F[MIN|MAX][NUM|NAN] and simple strict operations are available for all FP
  // NEON types.
  if (VT.isFloatingPoint() &&
      VT.getVectorElementType() != MVT::bf16 &&
      (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
    for (unsigned Opcode :
         {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM,
          ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::STRICT_FMINIMUM,
          ISD::STRICT_FMAXIMUM, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
          ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
          ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
      setOperationAction(Opcode, VT, Legal);
 
  // Strict fp extend and trunc are legal
  if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 16)
    setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
  if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 64)
    setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal);
 
  // FIXME: We could potentially make use of the vector comparison instructions
  // for STRICT_FSETCC and STRICT_FSETCSS, but there's a number of
  // complications:
  //  * FCMPEQ/NE are quiet comparisons, the rest are signalling comparisons,
  //    so we would need to expand when the condition code doesn't match the
  //    kind of comparison.
  //  * Some kinds of comparison require more than one FCMXY instruction so
  //    would need to be expanded instead.
  //  * The lowering of the non-strict versions involves target-specific ISD
  //    nodes so we would likely need to add strict versions of all of them and
  //    handle them appropriately.
  setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
  setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
 
  // When little-endian we can use ordinary d and q register loads/stores for
  // vector types, but when big-endian we need to use structure load/store which
  // only allow post-index addressing.
  // With +strict-align, we also need to use LD1/ST1 when the alignment is
  // less than the vector size, but we can still use LDR/STR for more-aligned
  // accesses, so these are marked as legal here, and the invalid cases are
  // rejected in getIndexedAddressParts.
  if (Subtarget->isLittleEndian()) {
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im, VT, Legal);
      setIndexedStoreAction(im, VT, Legal);
    }
  } else {
    setIndexedLoadAction(ISD::POST_INC, VT, Legal);
    setIndexedStoreAction(ISD::POST_INC, VT, Legal);
  }
 
  if (Subtarget->hasD128()) {
    setOperationAction(ISD::READ_REGISTER, MVT::i128, Custom);
    setOperationAction(ISD::WRITE_REGISTER, MVT::i128, Custom);
  }
 
  if (VT.isInteger()) {
    // Let common code emit inverted variants of compares we do support.
    setCondCodeAction(ISD::SETNE, VT, Expand);
    setCondCodeAction(ISD::SETLE, VT, Expand);
    setCondCodeAction(ISD::SETLT, VT, Expand);
    setCondCodeAction(ISD::SETULE, VT, Expand);
    setCondCodeAction(ISD::SETULT, VT, Expand);
  }
}
 
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
                                                          EVT OpVT) const {
  // Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo).
  if (!Subtarget->isSVEorStreamingSVEAvailable() ||
      ResVT.getVectorElementType() != MVT::i1)
    return true;
 
  // Expand 1 length fixed length vector.
  if (ResVT.isFixedLengthVector() && ResVT.getVectorNumElements() == 1)
    return true;
 
  // 32 & 64 bit operands are supported. We can promote anything < 64 bits,
  // but anything larger should be expanded.
  if (OpVT.getFixedSizeInBits() > 64)
    return true;
 
  return false;
}
 
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
 
  // By default everything must be expanded.
  for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
    setOperationAction(Op, VT, Expand);
 
  if (VT.isFloatingPoint()) {
    setCondCodeAction(ISD::SETO, VT, Expand);
    setCondCodeAction(ISD::SETOLT, VT, Expand);
    setCondCodeAction(ISD::SETOLE, VT, Expand);
    setCondCodeAction(ISD::SETULT, VT, Expand);
    setCondCodeAction(ISD::SETULE, VT, Expand);
    setCondCodeAction(ISD::SETUGE, VT, Expand);
    setCondCodeAction(ISD::SETUGT, VT, Expand);
    setCondCodeAction(ISD::SETUEQ, VT, Expand);
    setCondCodeAction(ISD::SETONE, VT, Expand);
  }
 
  TargetLoweringBase::LegalizeAction Default =
      VT == MVT::v1f64 ? Expand : Custom;
 
  // Mark integer truncating stores/extending loads as having custom lowering
  if (VT.isInteger()) {
    MVT InnerVT = VT.changeVectorElementType(MVT::i8);
    while (InnerVT != VT) {
      setTruncStoreAction(VT, InnerVT, Default);
      setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Default);
      setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Default);
      setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Default);
      InnerVT = InnerVT.changeVectorElementType(
          MVT::getIntegerVT(2 * InnerVT.getScalarSizeInBits()));
    }
  }
 
  // Mark floating-point truncating stores/extending loads as having custom
  // lowering
  if (VT.getScalarType() == MVT::f32 || VT.getScalarType() == MVT::f64) {
    MVT InnerVT = VT.changeVectorElementType(MVT::f16);
    while (InnerVT != VT) {
      setTruncStoreAction(VT, InnerVT, Custom);
      setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Default);
      InnerVT = InnerVT.changeVectorElementType(
          MVT::getFloatingPointVT(2 * InnerVT.getScalarSizeInBits()));
    }
  }
 
  bool PreferNEON = VT.is64BitVector() || VT.is128BitVector();
  bool PreferSVE = !PreferNEON && Subtarget->isSVEAvailable();
 
  static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
                                    ISD::PARTIAL_REDUCE_UMLA};
  unsigned NumElts = VT.getVectorNumElements();
  if (VT.getVectorElementType() == MVT::i64) {
    setPartialReduceMLAAction(MLAOps, VT,
                              MVT::getVectorVT(MVT::i8, NumElts * 8), Custom);
    setPartialReduceMLAAction(MLAOps, VT,
                              MVT::getVectorVT(MVT::i16, NumElts * 4), Custom);
    setPartialReduceMLAAction(MLAOps, VT,
                              MVT::getVectorVT(MVT::i32, NumElts * 2), Custom);
  } else if (VT.getVectorElementType() == MVT::i32) {
    setPartialReduceMLAAction(MLAOps, VT,
                              MVT::getVectorVT(MVT::i8, NumElts * 4), Custom);
    setPartialReduceMLAAction(MLAOps, VT,
                              MVT::getVectorVT(MVT::i16, NumElts * 2), Custom);
  } else if (VT.getVectorElementType() == MVT::i16) {
    setPartialReduceMLAAction(MLAOps, VT,
                              MVT::getVectorVT(MVT::i8, NumElts * 2), Custom);
  }
  if (Subtarget->hasMatMulInt8()) {
    if (VT.getVectorElementType() == MVT::i32)
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, VT,
                                MVT::getVectorVT(MVT::i8, NumElts * 4), Custom);
    else if (VT.getVectorElementType() == MVT::i64)
      setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, VT,
                                MVT::getVectorVT(MVT::i8, NumElts * 8), Custom);
  }
 
  if (Subtarget->hasSVE2p1() && VT.getVectorElementType() == MVT::f32) {
    setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_FMLA, VT,
                              MVT::getVectorVT(MVT::f16, NumElts * 2), Custom);
  }
 
  // Lower fixed length vector operations to scalable equivalents.
  setOperationAction(ISD::ABDS, VT, Default);
  setOperationAction(ISD::ABDU, VT, Default);
  setOperationAction(ISD::ABS, VT, Default);
  setOperationAction(ISD::ADD, VT, Default);
  setOperationAction(ISD::AND, VT, Default);
  setOperationAction(ISD::ANY_EXTEND, VT, Default);
  setOperationAction(ISD::BITCAST, VT, PreferNEON ? Legal : Default);
  setOperationAction(ISD::BITREVERSE, VT, Default);
  setOperationAction(ISD::BSWAP, VT, Default);
  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
  setOperationAction(ISD::CONCAT_VECTORS, VT, Default);
  setOperationAction(ISD::CTLZ, VT, Default);
  setOperationAction(ISD::CTPOP, VT, Default);
  setOperationAction(ISD::CTTZ, VT, Default);
  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Default);
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Default);
  setOperationAction(ISD::FABS, VT, Default);
  setOperationAction(ISD::FADD, VT, Default);
  setOperationAction(ISD::FCANONICALIZE, VT, Default);
  setOperationAction(ISD::FCEIL, VT, Default);
  setOperationAction(ISD::FCOPYSIGN, VT, Default);
  setOperationAction(ISD::FDIV, VT, Default);
  setOperationAction(ISD::FFLOOR, VT, Default);
  setOperationAction(ISD::FMA, VT, Default);
  setOperationAction(ISD::FMAXIMUM, VT, Default);
  setOperationAction(ISD::FMAXNUM, VT, Default);
  setOperationAction(ISD::FMAXNUM_IEEE, VT, Default);
  setOperationAction(ISD::FMINIMUM, VT, Default);
  setOperationAction(ISD::FMINNUM, VT, Default);
  setOperationAction(ISD::FMINNUM_IEEE, VT, Default);
  setOperationAction(ISD::FMUL, VT, Default);
  setOperationAction(ISD::FNEARBYINT, VT, Default);
  setOperationAction(ISD::FNEG, VT, Default);
  setOperationAction(ISD::FP_EXTEND, VT, Default);
  setOperationAction(ISD::FP_ROUND, VT, Default);
  setOperationAction(ISD::FP_TO_SINT, VT, Default);
  setOperationAction(ISD::FP_TO_UINT, VT, Default);
  setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
  setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
  setOperationAction(ISD::FRINT, VT, Default);
  setOperationAction(ISD::LRINT, VT, Default);
  setOperationAction(ISD::LLRINT, VT, Default);
  setOperationAction(ISD::FROUND, VT, Default);
  setOperationAction(ISD::FROUNDEVEN, VT, Default);
  setOperationAction(ISD::FSQRT, VT, Default);
  setOperationAction(ISD::FSUB, VT, Default);
  setOperationAction(ISD::FTRUNC, VT, Default);
  setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Default);
  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Default);
  setOperationAction(ISD::LOAD, VT, PreferNEON ? Legal : Default);
  setOperationAction(ISD::MASKED_UDIV, VT, Default);
  setOperationAction(ISD::MASKED_SDIV, VT, Default);
  setOperationAction(ISD::MGATHER, VT, PreferSVE ? Default : Expand);
  setOperationAction(ISD::MLOAD, VT, Default);
  setOperationAction(ISD::MSCATTER, VT, PreferSVE ? Default : Expand);
  setOperationAction(ISD::MSTORE, VT, Default);
  setOperationAction(ISD::MUL, VT, Default);
  setOperationAction(ISD::MULHS, VT, Default);
  setOperationAction(ISD::MULHU, VT, Default);
  setOperationAction(ISD::OR, VT, Default);
  setOperationAction(ISD::SCALAR_TO_VECTOR, VT, PreferNEON ? Legal : Expand);
  setOperationAction(ISD::SDIV, VT, Default);
  setOperationAction(ISD::SELECT, VT, Default);
  setOperationAction(ISD::SETCC, VT, Default);
  setOperationAction(ISD::SHL, VT, Default);
  setOperationAction(ISD::SIGN_EXTEND, VT, Default);
  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Default);
  setOperationAction(ISD::SINT_TO_FP, VT, Default);
  setOperationAction(ISD::SMAX, VT, Default);
  setOperationAction(ISD::SMIN, VT, Default);
  setOperationAction(ISD::SPLAT_VECTOR, VT, Default);
  setOperationAction(ISD::SRA, VT, Default);
  setOperationAction(ISD::SRL, VT, Default);
  setOperationAction(ISD::STORE, VT, PreferNEON ? Legal : Default);
  setOperationAction(ISD::SUB, VT, Default);
  setOperationAction(ISD::TRUNCATE, VT, Default);
  setOperationAction(ISD::UDIV, VT, Default);
  setOperationAction(ISD::UINT_TO_FP, VT, Default);
  setOperationAction(ISD::UMAX, VT, Default);
  setOperationAction(ISD::UMIN, VT, Default);
  setOperationAction(ISD::VECREDUCE_ADD, VT, Default);
  setOperationAction(ISD::VECREDUCE_AND, VT, Default);
  setOperationAction(ISD::VECREDUCE_FADD, VT, Default);
  setOperationAction(ISD::VECREDUCE_FMAX, VT, Default);
  setOperationAction(ISD::VECREDUCE_FMIN, VT, Default);
  setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Default);
  setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Default);
  setOperationAction(ISD::VECREDUCE_OR, VT, Default);
  setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, PreferSVE ? Default : Expand);
  setOperationAction(ISD::VECREDUCE_SMAX, VT, Default);
  setOperationAction(ISD::VECREDUCE_SMIN, VT, Default);
  setOperationAction(ISD::VECREDUCE_UMAX, VT, Default);
  setOperationAction(ISD::VECREDUCE_UMIN, VT, Default);
  setOperationAction(ISD::VECREDUCE_XOR, VT, Default);
  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Default);
  setOperationAction(ISD::VECTOR_SPLICE_LEFT, VT, Default);
  setOperationAction(ISD::VECTOR_SPLICE_RIGHT, VT, Default);
  setOperationAction(ISD::VSELECT, VT, Default);
  setOperationAction(ISD::XOR, VT, Default);
  setOperationAction(ISD::ZERO_EXTEND, VT, Default);
}
 
void AArch64TargetLowering::addDRType(MVT VT) {
  addRegisterClass(VT, &AArch64::FPR64RegClass);
  if (Subtarget->isNeonAvailable())
    addTypeForNEON(VT);
}
 
void AArch64TargetLowering::addQRType(MVT VT) {
  addRegisterClass(VT, &AArch64::FPR128RegClass);
  if (Subtarget->isNeonAvailable())
    addTypeForNEON(VT);
}
 
EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &,
                                              LLVMContext &C, EVT VT) const {
  if (!VT.isVector())
    return MVT::i32;
  if (VT.isScalableVector())
    return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
  return VT.changeVectorElementTypeToInteger();
}
 
// isIntImmediate - This method tests to see if the node is a constant
// operand. If so Imm will receive the value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
  if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(N)) {
    Imm = C->getZExtValue();
    return true;
  }
  return false;
}
 
static bool isVectorizedBinOp(unsigned Opcode) {
  switch (Opcode) {
  case AArch64ISD::SQDMULH:
    return true;
  default:
    return false;
  }
}
 
// isOpcWithIntImmediate - This method tests to see if the node is a specific
// opcode and that it has a immediate integer right operand.
// If so Imm will receive the value.
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
                                  uint64_t &Imm) {
  return N->getOpcode() == Opc &&
         isIntImmediate(N->getOperand(1).getNode(), Imm);
}
 
static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
                               const APInt &Demanded,
                               TargetLowering::TargetLoweringOpt &TLO,
                               unsigned NewOpc) {
  uint64_t OldImm = Imm, NewImm, Enc;
  uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
 
  // Return if the immediate is already all zeros, all ones, a bimm32 or a
  // bimm64.
  if (Imm == 0 || Imm == Mask ||
      AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
    return false;
 
  unsigned EltSize = Size;
  uint64_t DemandedBits = Demanded.getZExtValue();
 
  // Clear bits that are not demanded.
  Imm &= DemandedBits;
 
  while (true) {
    // The goal here is to set the non-demanded bits in a way that minimizes
    // the number of switching between 0 and 1. In order to achieve this goal,
    // we set the non-demanded bits to the value of the preceding demanded bits.
    // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
    // non-demanded bit), we copy bit0 (1) to the least significant 'x',
    // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
    // The final result is 0b11000011.
    uint64_t NonDemandedBits = ~DemandedBits;
    uint64_t InvertedImm = ~Imm & DemandedBits;
    uint64_t RotatedImm =
        ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
        NonDemandedBits;
    uint64_t Sum = RotatedImm + NonDemandedBits;
    bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
    uint64_t Ones = (Sum + Carry) & NonDemandedBits;
    NewImm = (Imm | Ones) & Mask;
 
    // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
    // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
    // we halve the element size and continue the search.
    if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
      break;
 
    // We cannot shrink the element size any further if it is 2-bits.
    if (EltSize == 2)
      return false;
 
    EltSize /= 2;
    Mask >>= EltSize;
    uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
 
    // Return if there is mismatch in any of the demanded bits of Imm and Hi.
    if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
      return false;
 
    // Merge the upper and lower halves of Imm and DemandedBits.
    Imm |= Hi;
    DemandedBits |= DemandedBitsHi;
  }
 
  ++NumOptimizedImms;
 
  // Replicate the element across the register width.
  while (EltSize < Size) {
    NewImm |= NewImm << EltSize;
    EltSize *= 2;
  }
 
  (void)OldImm;
  assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
         "demanded bits should never be altered");
  assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm");
 
  // Create the new constant immediate node.
  EVT VT = Op.getValueType();
  SDLoc DL(Op);
  SDValue New;
 
  // If the new constant immediate is all-zeros or all-ones, let the target
  // independent DAG combine optimize this node.
  if (NewImm == 0 || NewImm == OrigMask) {
    New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
                          TLO.DAG.getConstant(NewImm, DL, VT));
  // Otherwise, create a machine node so that target independent DAG combine
  // doesn't undo this optimization.
  } else {
    Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
    SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
    New = SDValue(
        TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
  }
 
  return TLO.CombineTo(Op, New);
}
 
bool AArch64TargetLowering::targetShrinkDemandedConstant(
    SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
    TargetLoweringOpt &TLO) const {
  // Delay this optimization to as late as possible.
  if (!TLO.LegalOps)
    return false;
 
  if (!EnableOptimizeLogicalImm)
    return false;
 
  EVT VT = Op.getValueType();
  if (VT.isVector())
    return false;
 
  unsigned Size = VT.getSizeInBits();
 
  if (Size != 32 && Size != 64)
    return false;
 
  // Exit early if we demand all bits.
  if (DemandedBits.isAllOnes())
    return false;
 
  unsigned NewOpc;
  switch (Op.getOpcode()) {
  default:
    return false;
  case ISD::AND:
    NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
    break;
  case ISD::OR:
    NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
    break;
  case ISD::XOR:
    NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
    break;
  }
  ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
  if (!C)
    return false;
  uint64_t Imm = C->getZExtValue();
  return optimizeLogicalImm(Op, Size, Imm, DemandedBits, TLO, NewOpc);
}
 
/// computeKnownBitsForTargetNode - Determine which of the bits specified in
/// Mask are known to be either zero or one and return them Known.
void AArch64TargetLowering::computeKnownBitsForTargetNode(
    const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
    const SelectionDAG &DAG, unsigned Depth) const {
  switch (Op.getOpcode()) {
  default:
    break;
  case AArch64ISD::DUP: {
    SDValue SrcOp = Op.getOperand(0);
    Known = DAG.computeKnownBits(SrcOp, Depth + 1);
    if (SrcOp.getValueSizeInBits() != Op.getScalarValueSizeInBits()) {
      assert(SrcOp.getValueSizeInBits() > Op.getScalarValueSizeInBits() &&
             "Expected DUP implicit truncation");
      Known = Known.trunc(Op.getScalarValueSizeInBits());
    }
    break;
  }
  case AArch64ISD::CSEL: {
    KnownBits Known2;
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
    Known = Known.intersectWith(Known2);
    break;
  }
  case AArch64ISD::CSNEG:
  case AArch64ISD::CSINC:
  case AArch64ISD::CSINV: {
    KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
 
    // The result is either:
    // CSINC: KnownOp0 or KnownOp1 + 1
    // CSINV: KnownOp0 or ~KnownOp1
    // CSNEG: KnownOp0 or KnownOp1 * -1
    if (Op.getOpcode() == AArch64ISD::CSINC)
      KnownOp1 = KnownBits::add(
          KnownOp1,
          KnownBits::makeConstant(APInt(Op.getScalarValueSizeInBits(), 1)));
    else if (Op.getOpcode() == AArch64ISD::CSINV)
      std::swap(KnownOp1.Zero, KnownOp1.One);
    else if (Op.getOpcode() == AArch64ISD::CSNEG)
      KnownOp1 =
          KnownBits::mul(KnownOp1, KnownBits::makeConstant(APInt::getAllOnes(
                                       Op.getScalarValueSizeInBits())));
 
    Known = KnownOp0.intersectWith(KnownOp1);
    break;
  }
  case AArch64ISD::BICi: {
    // Compute the bit cleared value.
    APInt Mask =
        ~(Op->getConstantOperandAPInt(1) << Op->getConstantOperandAPInt(2))
             .trunc(Known.getBitWidth());
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known &= KnownBits::makeConstant(Mask);
    break;
  }
  case AArch64ISD::VLSHR: {
    KnownBits Known2;
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
    Known = KnownBits::lshr(Known, Known2);
    break;
  }
  case AArch64ISD::VASHR: {
    KnownBits Known2;
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
    Known = KnownBits::ashr(Known, Known2);
    break;
  }
  case AArch64ISD::VSHL: {
    KnownBits Known2;
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
    Known = KnownBits::shl(Known, Known2);
    break;
  }
  case AArch64ISD::MOVI: {
    Known = KnownBits::makeConstant(
        APInt(Known.getBitWidth(), Op->getConstantOperandVal(0)));
    break;
  }
  case AArch64ISD::MOVIshift: {
    Known = KnownBits::makeConstant(
        APInt(Known.getBitWidth(), Op->getConstantOperandVal(0)
                                       << Op->getConstantOperandVal(1)));
    break;
  }
  case AArch64ISD::MOVImsl: {
    unsigned ShiftAmt = AArch64_AM::getShiftValue(Op->getConstantOperandVal(1));
    Known = KnownBits::makeConstant(APInt(
        Known.getBitWidth(), ~(~Op->getConstantOperandVal(0) << ShiftAmt)));
    break;
  }
  case AArch64ISD::MOVIedit: {
    Known = KnownBits::makeConstant(APInt(
        Known.getBitWidth(),
        AArch64_AM::decodeAdvSIMDModImmType10(Op->getConstantOperandVal(0))));
    break;
  }
  case AArch64ISD::MVNIshift: {
    Known = KnownBits::makeConstant(
        APInt(Known.getBitWidth(),
              ~(Op->getConstantOperandVal(0) << Op->getConstantOperandVal(1)),
              /*isSigned*/ false, /*implicitTrunc*/ true));
    break;
  }
  case AArch64ISD::MVNImsl: {
    unsigned ShiftAmt = AArch64_AM::getShiftValue(Op->getConstantOperandVal(1));
    Known = KnownBits::makeConstant(
        APInt(Known.getBitWidth(), (~Op->getConstantOperandVal(0) << ShiftAmt),
              /*isSigned*/ false, /*implicitTrunc*/ true));
    break;
  }
  case AArch64ISD::LOADgot:
  case AArch64ISD::ADDlow: {
    if (!Subtarget->isTargetILP32())
      break;
    // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
    Known.Zero = APInt::getHighBitsSet(64, 32);
    break;
  }
  case AArch64ISD::ASSERT_ZEXT_BOOL: {
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known.Zero |= APInt(Known.getBitWidth(), 0xFE);
    break;
  }
  case ISD::INTRINSIC_W_CHAIN: {
    Intrinsic::ID IntID =
        static_cast<Intrinsic::ID>(Op->getConstantOperandVal(1));
    switch (IntID) {
    default: return;
    case Intrinsic::aarch64_ldaxr:
    case Intrinsic::aarch64_ldxr: {
      unsigned BitWidth = Known.getBitWidth();
      EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
      unsigned MemBits = VT.getScalarSizeInBits();
      Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
      return;
    }
    }
    break;
  }
  case AArch64ISD::SHL_PRED:
  case AArch64ISD::SRL_PRED:
  case AArch64ISD::SRA_PRED: {
    SDValue Pg = Op->getOperand(0);
    if (!isAllActivePredicate(DAG, Pg))
      break;
 
    KnownBits KnownVal =
        DAG.computeKnownBits(Op->getOperand(1), DemandedElts, Depth + 1);
    KnownBits KnownAmt =
        DAG.computeKnownBits(Op->getOperand(2), DemandedElts, Depth + 1);
 
    if (Op.getOpcode() == AArch64ISD::SHL_PRED)
      Known = KnownBits::shl(KnownVal, KnownAmt);
    else if (Op.getOpcode() == AArch64ISD::SRL_PRED)
      Known = KnownBits::lshr(KnownVal, KnownAmt);
    else
      Known = KnownBits::ashr(KnownVal, KnownAmt);
    break;
  }
  case ISD::INTRINSIC_WO_CHAIN:
  case ISD::INTRINSIC_VOID: {
    unsigned IntNo = Op.getConstantOperandVal(0);
    switch (IntNo) {
    default:
      break;
    case Intrinsic::aarch64_neon_uaddlv: {
      MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
      unsigned BitWidth = Known.getBitWidth();
      if (VT == MVT::v8i8 || VT == MVT::v16i8) {
        unsigned Bound = (VT == MVT::v8i8) ?  11 : 12;
        assert(BitWidth >= Bound && "Unexpected width!");
        APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - Bound);
        Known.Zero |= Mask;
      }
      break;
    }
    case Intrinsic::aarch64_neon_umaxv:
    case Intrinsic::aarch64_neon_uminv: {
      // Figure out the datatype of the vector operand. The UMINV instruction
      // will zero extend the result, so we can mark as known zero all the
      // bits larger than the element datatype. 32-bit or larget doesn't need
      // this as those are legal types and will be handled by isel directly.
      MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
      unsigned BitWidth = Known.getBitWidth();
      if (VT == MVT::v8i8 || VT == MVT::v16i8) {
        assert(BitWidth >= 8 && "Unexpected width!");
        APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
        Known.Zero |= Mask;
      } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
        assert(BitWidth >= 16 && "Unexpected width!");
        APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
        Known.Zero |= Mask;
      }
      break;
    } break;
    }
  }
  }
}
 
unsigned AArch64TargetLowering::ComputeNumSignBitsForTargetNode(
    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
    unsigned Depth) const {
  EVT VT = Op.getValueType();
  unsigned VTBits = VT.getScalarSizeInBits();
  unsigned Opcode = Op.getOpcode();
  switch (Opcode) {
  case AArch64ISD::FCMEQ:
  case AArch64ISD::FCMGE:
  case AArch64ISD::FCMGT:
    // Compares return either 0 or all-ones
    return VTBits;
  case AArch64ISD::VASHR: {
    unsigned Tmp =
        DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
    return std::min<uint64_t>(Tmp + Op.getConstantOperandVal(1), VTBits);
  }
  }
 
  return 1;
}
 
unsigned AArch64TargetLowering::computeNumSignBitsForTargetInstr(
    GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
    const MachineRegisterInfo &MRI, unsigned Depth) const {
  const MachineInstr *MI = MRI.getVRegDef(R);
  if (!MI)
    return 1;
 
  switch (MI->getOpcode()) {
  case AArch64::G_FCMEQ:
  case AArch64::G_FCMGE:
  case AArch64::G_FCMGT: {
    LLT VT = MRI.getType(R);
    return VT.getScalarSizeInBits();
  }
  case AArch64::G_VASHR: {
    unsigned Tmp = Analysis.computeNumSignBits(MI->getOperand(1).getReg(),
                                               DemandedElts, Depth + 1);
    LLT VT = MRI.getType(R);
    return std::min<uint64_t>(Tmp + MI->getOperand(2).getImm(),
                              VT.getScalarSizeInBits());
  }
  default:
    return 1;
  }
}
 
MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
                                                  EVT) const {
  return MVT::i64;
}
 
bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
    EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
    unsigned *Fast) const {
 
  // Allow SVE loads/stores where the alignment >= the size of the element type,
  // even with +strict-align. Predicated SVE loads/stores (e.g. ld1/st1), used
  // for stores that come from IR, only require element-size alignment (even if
  // unaligned accesses are disabled). Without this, these will be forced to
  // have 16-byte alignment with +strict-align (and fail to lower as we don't
  // yet support TLI.expandUnalignedLoad() and TLI.expandUnalignedStore()).
  //
  // For NEON in strict-align mode, we need to use LD1/ST1 when the alignment
  // is less than the size of the vector, but greater than or equal to the size
  // of the elements.
  bool UseNEONLd1 = Subtarget->requiresStrictAlign() &&
                    VT.isFixedLengthVector() &&
                    (VT.getSizeInBits() == 64 || VT.getSizeInBits() == 128);
  if (VT.isScalableVector() || UseNEONLd1) {
    unsigned ElementSizeBits = VT.getScalarSizeInBits();
    if (ElementSizeBits % 8 == 0 && Alignment >= Align(ElementSizeBits / 8))
      return true;
  }
 
  if (Subtarget->requiresStrictAlign())
    return false;
 
  if (Fast) {
    // Some CPUs are fine with unaligned stores except for 128-bit ones.
    *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
            // See comments in performSTORECombine() for more details about
            // these conditions.
 
            // Code that uses clang vector extensions can mark that it
            // wants unaligned accesses to be treated as fast by
            // underspecifying alignment to be 1 or 2.
            Alignment <= 2 ||
 
            // Disregard v2i64. Memcpy lowering produces those and splitting
            // them regresses performance on micro-benchmarks and olden/bh.
            VT == MVT::v2i64;
  }
  return true;
}
 
// Same as above but handling LLTs instead.
bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
    LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
    unsigned *Fast) const {
  if (Subtarget->requiresStrictAlign())
    return false;
 
  if (Fast) {
    // Some CPUs are fine with unaligned stores except for 128-bit ones.
    *Fast = !Subtarget->isMisaligned128StoreSlow() ||
            Ty.getSizeInBytes() != 16 ||
            // See comments in performSTORECombine() for more details about
            // these conditions.
 
            // Code that uses clang vector extensions can mark that it
            // wants unaligned accesses to be treated as fast by
            // underspecifying alignment to be 1 or 2.
            Alignment <= 2 ||
 
            // Disregard v2i64. Memcpy lowering produces those and splitting
            // them regresses performance on micro-benchmarks and olden/bh.
            Ty == LLT::fixed_vector(2, 64);
  }
  return true;
}
 
FastISel *AArch64TargetLowering::createFastISel(
    FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo,
    const LibcallLoweringInfo *libcallLowering) const {
  return AArch64::createFastISel(funcInfo, libInfo, libcallLowering);
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
                                    MachineBasicBlock *MBB) const {
  // We materialise the F128CSEL pseudo-instruction as some control flow and a
  // phi node:
 
  // OrigBB:
  //     [... previous instrs leading to comparison ...]
  //     b.ne TrueBB
  //     b EndBB
  // TrueBB:
  //     ; Fallthrough
  // EndBB:
  //     Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
 
  MachineFunction *MF = MBB->getParent();
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
  DebugLoc DL = MI.getDebugLoc();
  MachineFunction::iterator It = ++MBB->getIterator();
 
  Register DestReg = MI.getOperand(0).getReg();
  Register IfTrueReg = MI.getOperand(1).getReg();
  Register IfFalseReg = MI.getOperand(2).getReg();
  unsigned CondCode = MI.getOperand(3).getImm();
  bool NZCVKilled = MI.getOperand(4).isKill();
 
  MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
  MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
  MF->insert(It, TrueBB);
  MF->insert(It, EndBB);
 
  // Transfer rest of current basic-block to EndBB
  EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
                MBB->end());
  EndBB->transferSuccessorsAndUpdatePHIs(MBB);
 
  BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
  BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
  MBB->addSuccessor(TrueBB);
  MBB->addSuccessor(EndBB);
 
  // TrueBB falls through to the end.
  TrueBB->addSuccessor(EndBB);
 
  if (!NZCVKilled) {
    TrueBB->addLiveIn(AArch64::NZCV);
    EndBB->addLiveIn(AArch64::NZCV);
  }
 
  BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
      .addReg(IfTrueReg)
      .addMBB(TrueBB)
      .addReg(IfFalseReg)
      .addMBB(MBB);
 
  MI.eraseFromParent();
  return EndBB;
}
 
MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
       MachineInstr &MI, MachineBasicBlock *BB) const {
  assert(!isAsynchronousEHPersonality(classifyEHPersonality(
             BB->getParent()->getFunction().getPersonalityFn())) &&
         "SEH does not use catchret!");
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitLoweredSetFpmr(MachineInstr &MI,
                                          MachineBasicBlock *MBB) const {
  MachineFunction *MF = MBB->getParent();
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  DebugLoc DL = MI.getDebugLoc();
 
  if (!UseConditionalFPMRWrite) {
    BuildMI(*MBB, MI, DL, TII->get(AArch64::MSR))
        .addImm(0xda22)
        .add(MI.getOperand(0))
        .addDef(AArch64::FPMR, RegState::Implicit);
    MI.eraseFromParent();
    return MBB;
  }
 
  Register NewFpmrVal = MI.getOperand(0).getReg();
  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
  MachineBasicBlock *MsrBB = MF->CreateMachineBasicBlock(LLVM_BB);
  MachineBasicBlock *EndBB = MBB->splitAt(MI);
  MF->insert(++MBB->getIterator(), MsrBB);
 
  Register CurrentFpmrVal =
      MF->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
  BuildMI(*MBB, MI, DL, TII->get(AArch64::MRS), CurrentFpmrVal)
      .addImm(0xda22)
      .addUse(AArch64::FPMR, RegState::Implicit);
  BuildMI(*MBB, MI, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
      .addReg(CurrentFpmrVal, RegState::Kill)
      .addReg(NewFpmrVal)
      .addImm(0);
  BuildMI(*MBB, MI, DL, TII->get(AArch64::Bcc))
      .addImm(AArch64CC::EQ)
      .addMBB(EndBB);
  BuildMI(*MsrBB, MsrBB->begin(), DL, TII->get(AArch64::MSR))
      .addImm(0xda22)
      .addReg(NewFpmrVal, getKillRegState(MI.getOperand(0).isDead()))
      .addDef(AArch64::FPMR, RegState::Implicit);
 
  MBB->addSuccessor(MsrBB);
  MsrBB->addSuccessor(EndBB);
 
  MI.eraseFromParent();
  return EndBB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitDynamicProbedAlloc(MachineInstr &MI,
                                              MachineBasicBlock *MBB) const {
  MachineFunction &MF = *MBB->getParent();
  MachineBasicBlock::iterator MBBI = MI.getIterator();
  const AArch64InstrInfo &TII =
      *MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
  Register TargetReg = MI.getOperand(0).getReg();
  MachineBasicBlock::iterator NextInst =
      TII.probedStackAlloc(MBBI, TargetReg, false);
 
  MI.eraseFromParent();
  return NextInst->getParent();
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitCheckMatchingVL(MachineInstr &MI,
                                           MachineBasicBlock *MBB) const {
  MachineFunction *MF = MBB->getParent();
  MachineRegisterInfo &MRI = MF->getRegInfo();
 
  const TargetRegisterClass *RC_GPR = &AArch64::GPR64RegClass;
  const TargetRegisterClass *RC_GPRsp = &AArch64::GPR64spRegClass;
 
  Register RegVL_GPR = MRI.createVirtualRegister(RC_GPR);
  Register RegVL_GPRsp = MRI.createVirtualRegister(RC_GPRsp); // for ADDSVL src
  Register RegSVL_GPR = MRI.createVirtualRegister(RC_GPR);
  Register RegSVL_GPRsp = MRI.createVirtualRegister(RC_GPRsp); // for ADDSVL dst
 
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  DebugLoc DL = MI.getDebugLoc();
 
  // RDVL requires GPR64, ADDSVL requires GPR64sp
  // We need to insert COPY instructions, these will later be removed by the
  // RegisterCoalescer
  BuildMI(*MBB, MI, DL, TII->get(AArch64::RDVLI_XI), RegVL_GPR).addImm(1);
  BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::COPY), RegVL_GPRsp)
      .addReg(RegVL_GPR);
 
  BuildMI(*MBB, MI, DL, TII->get(AArch64::ADDSVL_XXI), RegSVL_GPRsp)
      .addReg(RegVL_GPRsp)
      .addImm(-1);
  BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::COPY), RegSVL_GPR)
      .addReg(RegSVL_GPRsp);
 
  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
  MachineFunction::iterator It = ++MBB->getIterator();
  MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock(LLVM_BB);
  MachineBasicBlock *PassBB = MF->CreateMachineBasicBlock(LLVM_BB);
  MF->insert(It, TrapBB);
  MF->insert(It, PassBB);
 
  // Continue if vector lengths match
  BuildMI(*MBB, MI, DL, TII->get(AArch64::CBZX))
      .addReg(RegSVL_GPR)
      .addMBB(PassBB);
 
  // Transfer rest of current BB to PassBB
  PassBB->splice(PassBB->begin(), MBB,
                 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
  PassBB->transferSuccessorsAndUpdatePHIs(MBB);
 
  // Trap if vector lengths mismatch
  BuildMI(TrapBB, DL, TII->get(AArch64::BRK)).addImm(1);
 
  MBB->addSuccessor(TrapBB);
  MBB->addSuccessor(PassBB);
 
  MI.eraseFromParent();
  return PassBB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitTileLoad(unsigned Opc, unsigned BaseReg,
                                    MachineInstr &MI,
                                    MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
 
  MIB.addReg(BaseReg + MI.getOperand(0).getImm(), RegState::Define);
  MIB.add(MI.getOperand(1)); // slice index register
  MIB.add(MI.getOperand(2)); // slice index offset
  MIB.add(MI.getOperand(3)); // pg
  MIB.add(MI.getOperand(4)); // base
  MIB.add(MI.getOperand(5)); // offset
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitFill(MachineInstr &MI, MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB =
      BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::LDR_ZA));
 
  MIB.addReg(AArch64::ZA, RegState::Define);
  MIB.add(MI.getOperand(0)); // Vector select register
  MIB.add(MI.getOperand(1)); // Vector select offset
  MIB.add(MI.getOperand(2)); // Base
  MIB.add(MI.getOperand(1)); // Offset, same as vector select offset
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *AArch64TargetLowering::EmitZTInstr(MachineInstr &MI,
                                                      MachineBasicBlock *BB,
                                                      unsigned Opcode,
                                                      bool Op0IsDef) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB;
 
  MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opcode))
            .addReg(MI.getOperand(0).getReg(), getDefRegState(Op0IsDef));
  for (unsigned I = 1; I < MI.getNumOperands(); ++I)
    MIB.add(MI.getOperand(I));
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitZAInstr(unsigned Opc, unsigned BaseReg,
                                   MachineInstr &MI,
                                   MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
  unsigned StartIdx = 0;
 
  bool HasTile = BaseReg != AArch64::ZA;
  bool HasZPROut = HasTile && MI.getOperand(0).isReg();
  if (HasZPROut) {
    MIB.add(MI.getOperand(StartIdx)); // Output ZPR
    ++StartIdx;
  }
  if (HasTile) {
    MIB.addReg(BaseReg + MI.getOperand(StartIdx).getImm(),
               RegState::Define);                           // Output ZA Tile
    MIB.addReg(BaseReg + MI.getOperand(StartIdx).getImm()); // Input Za Tile
    StartIdx++;
  } else {
    // Avoids all instructions with mnemonic za.<sz>[Reg, Imm,
    if (MI.getOperand(0).isReg() && !MI.getOperand(1).isImm()) {
      MIB.add(MI.getOperand(StartIdx)); // Output ZPR
      ++StartIdx;
    }
    MIB.addReg(BaseReg, RegState::Define).addReg(BaseReg);
  }
  for (unsigned I = StartIdx; I < MI.getNumOperands(); ++I)
    MIB.add(MI.getOperand(I));
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB =
      BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::ZERO_M));
  MIB.add(MI.getOperand(0)); // Mask
 
  unsigned Mask = MI.getOperand(0).getImm();
  for (unsigned I = 0; I < 8; I++) {
    if (Mask & (1 << I))
      MIB.addDef(AArch64::ZAD0 + I, RegState::ImplicitDefine);
  }
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitEntryPStateSM(MachineInstr &MI,
                                         MachineBasicBlock *BB) const {
  MachineFunction *MF = BB->getParent();
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  const DebugLoc &DL = MI.getDebugLoc();
  Register ResultReg = MI.getOperand(0).getReg();
  if (MF->getRegInfo().use_empty(ResultReg)) {
    // Nothing to do. Pseudo erased below.
  } else if (Subtarget->hasSME()) {
    BuildMI(*BB, MI, DL, TII->get(AArch64::MRS), ResultReg)
        .addImm(AArch64SysReg::SVCR)
        .addReg(AArch64::VG, RegState::Implicit);
  } else {
    RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE;
    const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
    BuildMI(*BB, MI, DL, TII->get(AArch64::BL))
        .addExternalSymbol(getLibcallName(LC))
        .addReg(AArch64::X0, RegState::ImplicitDefine)
        .addRegMask(TRI->getCallPreservedMask(*MF, getLibcallCallingConv(LC)));
    BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), ResultReg)
        .addReg(AArch64::X0);
  }
  MI.eraseFromParent();
  return BB;
}
 
// Helper function to find the instruction that defined a virtual register.
// If unable to find such instruction, returns nullptr.
static const MachineInstr *stripVRegCopies(const MachineRegisterInfo &MRI,
                                           Register Reg) {
  while (Reg.isVirtual()) {
    MachineInstr *DefMI = MRI.getVRegDef(Reg);
    assert(DefMI && "Virtual register definition not found");
    unsigned Opcode = DefMI->getOpcode();
 
    if (Opcode == AArch64::COPY) {
      Reg = DefMI->getOperand(1).getReg();
      // Vreg is defined by copying from physreg.
      if (Reg.isPhysical())
        return DefMI;
      continue;
    }
    if (Opcode == AArch64::SUBREG_TO_REG) {
      Reg = DefMI->getOperand(1).getReg();
      continue;
    }
 
    return DefMI;
  }
  return nullptr;
}
 
void AArch64TargetLowering::fixupPtrauthDiscriminator(
    MachineInstr &MI, MachineBasicBlock *BB, MachineOperand &IntDiscOp,
    MachineOperand &AddrDiscOp, const TargetRegisterClass *AddrDiscRC) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
  const DebugLoc &DL = MI.getDebugLoc();
 
  Register AddrDisc = AddrDiscOp.getReg();
  int64_t IntDisc = IntDiscOp.getImm();
  assert(IntDisc == 0 && "Blend components are already expanded");
 
  const MachineInstr *DiscMI = stripVRegCopies(MRI, AddrDisc);
  if (DiscMI) {
    switch (DiscMI->getOpcode()) {
    case AArch64::MOVKXi:
      // blend(addr, imm) which is lowered as "MOVK addr, #imm, #48".
      // #imm should be an immediate and not a global symbol, for example.
      if (DiscMI->getOperand(2).isImm() &&
          DiscMI->getOperand(3).getImm() == 48) {
        AddrDisc = DiscMI->getOperand(1).getReg();
        IntDisc = DiscMI->getOperand(2).getImm();
      }
      break;
    case AArch64::MOVi32imm:
    case AArch64::MOVi64imm:
      // Small immediate integer constant passed via VReg.
      if (DiscMI->getOperand(1).isImm() &&
          isUInt<16>(DiscMI->getOperand(1).getImm())) {
        AddrDisc = AArch64::NoRegister;
        IntDisc = DiscMI->getOperand(1).getImm();
      }
      break;
    }
  }
 
  // For uniformity, always use NoRegister, as XZR is not necessarily contained
  // in the requested register class.
  if (AddrDisc == AArch64::XZR)
    AddrDisc = AArch64::NoRegister;
 
  // Make sure AddrDisc operand respects the register class imposed by MI.
  if (AddrDisc && MRI.getRegClass(AddrDisc) != AddrDiscRC) {
    Register TmpReg = MRI.createVirtualRegister(AddrDiscRC);
    BuildMI(*BB, MI, DL, TII->get(AArch64::COPY), TmpReg).addReg(AddrDisc);
    AddrDisc = TmpReg;
  }
 
  if (AddrDiscOp.getReg() != AddrDisc)
    AddrDiscOp.setIsKill(false);
 
  AddrDiscOp.setReg(AddrDisc);
  IntDiscOp.setImm(IntDisc);
}
 
MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
    MachineInstr &MI, MachineBasicBlock *BB) const {
 
  int SMEOrigInstr = AArch64::getSMEPseudoMap(MI.getOpcode());
  if (SMEOrigInstr != -1) {
    const TargetInstrInfo *TII = Subtarget->getInstrInfo();
    uint64_t SMEMatrixType =
        TII->get(MI.getOpcode()).TSFlags & AArch64::SMEMatrixTypeMask;
    switch (SMEMatrixType) {
    case (AArch64::SMEMatrixArray):
      return EmitZAInstr(SMEOrigInstr, AArch64::ZA, MI, BB);
    case (AArch64::SMEMatrixTileB):
      return EmitZAInstr(SMEOrigInstr, AArch64::ZAB0, MI, BB);
    case (AArch64::SMEMatrixTileH):
      return EmitZAInstr(SMEOrigInstr, AArch64::ZAH0, MI, BB);
    case (AArch64::SMEMatrixTileS):
      return EmitZAInstr(SMEOrigInstr, AArch64::ZAS0, MI, BB);
    case (AArch64::SMEMatrixTileD):
      return EmitZAInstr(SMEOrigInstr, AArch64::ZAD0, MI, BB);
    case (AArch64::SMEMatrixTileQ):
      return EmitZAInstr(SMEOrigInstr, AArch64::ZAQ0, MI, BB);
    }
  }
 
  switch (MI.getOpcode()) {
  default:
#ifndef NDEBUG
    MI.dump();
#endif
    llvm_unreachable("Unexpected instruction for custom inserter!");
  case AArch64::EntryPStateSM:
    return EmitEntryPStateSM(MI, BB);
  case AArch64::F128CSEL:
    return EmitF128CSEL(MI, BB);
  case TargetOpcode::STATEPOINT:
    // STATEPOINT is a pseudo instruction which has no implicit defs/uses
    // while bl call instruction (where statepoint will be lowered at the end)
    // has implicit def. This def is early-clobber as it will be set at
    // the moment of the call and earlier than any use is read.
    // Add this implicit dead def here as a workaround.
    MI.addOperand(*MI.getMF(),
                  MachineOperand::CreateReg(
                      AArch64::LR, /*isDef*/ true,
                      /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
                      /*isUndef*/ false, /*isEarlyClobber*/ true));
    [[fallthrough]];
  case TargetOpcode::STACKMAP:
  case TargetOpcode::PATCHPOINT:
    return emitPatchPoint(MI, BB);
 
  case TargetOpcode::PATCHABLE_EVENT_CALL:
  case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
    return BB;
 
  case AArch64::CATCHRET:
    return EmitLoweredCatchRet(MI, BB);
 
  case AArch64::PROBED_STACKALLOC_DYN:
    return EmitDynamicProbedAlloc(MI, BB);
 
  case AArch64::CHECK_MATCHING_VL_PSEUDO:
    return EmitCheckMatchingVL(MI, BB);
 
  case AArch64::LD1_MXIPXX_H_PSEUDO_B:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_B, AArch64::ZAB0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_H:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_H, AArch64::ZAH0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_S:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_S, AArch64::ZAS0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_D:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_D, AArch64::ZAD0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_Q:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_Q, AArch64::ZAQ0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_B:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_B, AArch64::ZAB0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_H:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_H, AArch64::ZAH0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_S:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_S, AArch64::ZAS0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_D:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_D, AArch64::ZAD0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_Q:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_Q, AArch64::ZAQ0, MI, BB);
  case AArch64::LDR_ZA_PSEUDO:
    return EmitFill(MI, BB);
  case AArch64::LDR_TX_PSEUDO:
    return EmitZTInstr(MI, BB, AArch64::LDR_TX, /*Op0IsDef=*/true);
  case AArch64::STR_TX_PSEUDO:
    return EmitZTInstr(MI, BB, AArch64::STR_TX, /*Op0IsDef=*/false);
  case AArch64::ZERO_M_PSEUDO:
    return EmitZero(MI, BB);
  case AArch64::ZERO_T_PSEUDO:
    return EmitZTInstr(MI, BB, AArch64::ZERO_T, /*Op0IsDef=*/true);
  case AArch64::MOVT_TIZ_PSEUDO:
    return EmitZTInstr(MI, BB, AArch64::MOVT_TIZ, /*Op0IsDef=*/true);
  case AArch64::MSR_FPMR:
    return EmitLoweredSetFpmr(MI, BB);
 
  case AArch64::PAC:
    fixupPtrauthDiscriminator(MI, BB, MI.getOperand(3), MI.getOperand(4),
                              &AArch64::GPR64noipRegClass);
    return BB;
  }
}
 
//===----------------------------------------------------------------------===//
// AArch64 Lowering private implementation.
//===----------------------------------------------------------------------===//
 
//===----------------------------------------------------------------------===//
// Lowering Code
//===----------------------------------------------------------------------===//
 
// Forward declarations of SVE fixed length lowering helpers
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT);
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
static SDValue convertFixedMaskToScalableVector(SDValue Mask,
                                                SelectionDAG &DAG);
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
                                                EVT VT);
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
                                             EVT VT);
static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG);
 
/// isZerosVector - Check whether SDNode N is a zero-filled vector.
static bool isZerosVector(const SDNode *N) {
  // Look through a bit convert.
  while (N->getOpcode() == ISD::BITCAST)
    N = N->getOperand(0).getNode();
 
  if (ISD::isConstantSplatVectorAllZeros(N))
    return true;
 
  if (N->getOpcode() != AArch64ISD::DUP)
    return false;
 
  auto Opnd0 = N->getOperand(0);
  return isNullConstant(Opnd0) || isNullFPConstant(Opnd0);
}
 
static bool isOneVector(SDValue V) {
  return isOneOrOneSplat(V) ||
         (V.getOpcode() == AArch64ISD::DUP && isOneConstant(V.getOperand(0)));
}
 
/// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
/// CC
static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC,
                                                  SDValue RHS = {}) {
  switch (CC) {
  default:
    llvm_unreachable("Unknown condition code!");
  case ISD::SETNE:
    return AArch64CC::NE;
  case ISD::SETEQ:
    return AArch64CC::EQ;
  case ISD::SETGT:
    return AArch64CC::GT;
  case ISD::SETGE:
    return (RHS && isNullConstant(RHS)) ? AArch64CC::PL : AArch64CC::GE;
  case ISD::SETLT:
    return (RHS && isNullConstant(RHS)) ? AArch64CC::MI : AArch64CC::LT;
  case ISD::SETLE:
    return AArch64CC::LE;
  case ISD::SETUGT:
    return AArch64CC::HI;
  case ISD::SETUGE:
    return AArch64CC::HS;
  case ISD::SETULT:
    return AArch64CC::LO;
  case ISD::SETULE:
    return AArch64CC::LS;
  }
}
 
/// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
static void changeFPCCToAArch64CC(ISD::CondCode CC,
                                  AArch64CC::CondCode &CondCode,
                                  AArch64CC::CondCode &CondCode2) {
  CondCode2 = AArch64CC::AL;
  switch (CC) {
  default:
    llvm_unreachable("Unknown FP condition!");
  case ISD::SETEQ:
  case ISD::SETOEQ:
    CondCode = AArch64CC::EQ;
    break;
  case ISD::SETGT:
  case ISD::SETOGT:
    CondCode = AArch64CC::GT;
    break;
  case ISD::SETGE:
  case ISD::SETOGE:
    CondCode = AArch64CC::GE;
    break;
  case ISD::SETOLT:
    CondCode = AArch64CC::MI;
    break;
  case ISD::SETOLE:
    CondCode = AArch64CC::LS;
    break;
  case ISD::SETONE:
    CondCode = AArch64CC::MI;
    CondCode2 = AArch64CC::GT;
    break;
  case ISD::SETO:
    CondCode = AArch64CC::VC;
    break;
  case ISD::SETUO:
    CondCode = AArch64CC::VS;
    break;
  case ISD::SETUEQ:
    CondCode = AArch64CC::EQ;
    CondCode2 = AArch64CC::VS;
    break;
  case ISD::SETUGT:
    CondCode = AArch64CC::HI;
    break;
  case ISD::SETUGE:
    CondCode = AArch64CC::PL;
    break;
  case ISD::SETLT:
  case ISD::SETULT:
    CondCode = AArch64CC::LT;
    break;
  case ISD::SETLE:
  case ISD::SETULE:
    CondCode = AArch64CC::LE;
    break;
  case ISD::SETNE:
  case ISD::SETUNE:
    CondCode = AArch64CC::NE;
    break;
  }
}
 
/// Convert a DAG fp condition code to an AArch64 CC.
/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
/// should be AND'ed instead of OR'ed.
static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
                                     AArch64CC::CondCode &CondCode,
                                     AArch64CC::CondCode &CondCode2) {
  CondCode2 = AArch64CC::AL;
  switch (CC) {
  default:
    changeFPCCToAArch64CC(CC, CondCode, CondCode2);
    assert(CondCode2 == AArch64CC::AL);
    break;
  case ISD::SETONE:
    // (a one b)
    // == ((a olt b) || (a ogt b))
    // == ((a ord b) && (a une b))
    CondCode = AArch64CC::VC;
    CondCode2 = AArch64CC::NE;
    break;
  case ISD::SETUEQ:
    // (a ueq b)
    // == ((a uno b) || (a oeq b))
    // == ((a ule b) && (a uge b))
    CondCode = AArch64CC::PL;
    CondCode2 = AArch64CC::LE;
    break;
  }
}
 
/// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
/// CC usable with the vector instructions. Fewer operations are available
/// without a real NZCV register, so we have to use less efficient combinations
/// to get the same effect.
static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
                                        AArch64CC::CondCode &CondCode,
                                        AArch64CC::CondCode &CondCode2,
                                        bool &Invert) {
  Invert = false;
  switch (CC) {
  default:
    // Mostly the scalar mappings work fine.
    changeFPCCToAArch64CC(CC, CondCode, CondCode2);
    break;
  case ISD::SETUO:
    Invert = true;
    [[fallthrough]];
  case ISD::SETO:
    CondCode = AArch64CC::MI;
    CondCode2 = AArch64CC::GE;
    break;
  case ISD::SETLE:
    CondCode = AArch64CC::LS;
    CondCode2 = AArch64CC::AL;
    break;
  case ISD::SETLT:
    CondCode = AArch64CC::MI;
    CondCode2 = AArch64CC::AL;
    break;
  case ISD::SETUEQ:
  case ISD::SETULT:
  case ISD::SETULE:
  case ISD::SETUGT:
  case ISD::SETUGE:
    // All of the compare-mask comparisons are ordered, but we can switch
    // between the two by a double inversion. E.g. ULE == !OGT.
    Invert = true;
    changeFPCCToAArch64CC(getSetCCInverse(CC, /* FP inverse */ MVT::f32),
                          CondCode, CondCode2);
    break;
  }
}
 
/// Like SelectionDAG::getCondCode(), but for AArch64 condition codes.
static SDValue getCondCode(SelectionDAG &DAG, AArch64CC::CondCode CC) {
  // TODO: Should be TargetConstant (need to s/imm/timm in patterns).
  return DAG.getConstant(CC, SDLoc(), CondCodeVT);
}
 
static unsigned numberOfInstrToLoadImm(const APInt &C) {
  uint64_t Imm = C.getZExtValue();
  SmallVector<AArch64_IMM::ImmInsnModel> Insn;
  AArch64_IMM::expandMOVImm(Imm, 32, Insn);
  return Insn.size();
}
 
static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG) {
  // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
  if (Op->getFlags().hasNoSignedWrap())
    return true;
 
  // We can still figure out if the second operand is safe to use
  // in a CMN instruction by checking if it is known to be not the minimum
  // signed value. If it is not, then we can safely use CMN.
  // Note: We can eventually remove this check and simply rely on
  // Op->getFlags().hasNoSignedWrap() once SelectionDAG/ISelLowering
  // consistently sets them appropriately when making said nodes.
 
  KnownBits KnownSrc = DAG.computeKnownBits(Op.getOperand(1));
  return !KnownSrc.getSignedMinValue().isMinSignedValue();
}
 
// Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
// the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
// can be set differently by this operation. It comes down to whether
// "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
// everything is fine. If not then the optimization is wrong. Thus general
// comparisons are only valid if op2 != 0 and op2 != INT_MIN.
//
// So, finally, the only LLVM-native comparisons that don't mention C or V
// are the ones that aren't unsigned comparisons. They're the only ones we can
// safely use CMN for in the absence of information about op2.
static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
  return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
         (isIntEqualitySetCC(CC) ||
          (isUnsignedIntSetCC(CC) && DAG.isKnownNeverZero(Op.getOperand(1))) ||
          (isSignedIntSetCC(CC) && isSafeSignedCMN(Op, DAG)));
}
 
static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &DL,
                                      SelectionDAG &DAG, SDValue Chain,
                                      bool IsSignaling) {
  EVT VT = LHS.getValueType();
  assert(VT != MVT::f128);
 
  const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
  if ((VT == MVT::f16 && !FullFP16) || VT == MVT::bf16) {
    LHS = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                      {Chain, LHS});
    RHS = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                      {LHS.getValue(1), RHS});
    Chain = RHS.getValue(1);
  }
  unsigned Opcode =
      IsSignaling ? AArch64ISD::STRICT_FCMPE : AArch64ISD::STRICT_FCMP;
  return DAG.getNode(Opcode, DL, {FlagsVT, MVT::Other}, {Chain, LHS, RHS});
}
 
static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                              const SDLoc &DL, SelectionDAG &DAG) {
  EVT VT = LHS.getValueType();
  const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
  if (VT.isFloatingPoint()) {
    assert(VT != MVT::f128);
    if ((VT == MVT::f16 && !FullFP16) || VT == MVT::bf16) {
      LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
      RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
    }
    return DAG.getNode(AArch64ISD::FCMP, DL, FlagsVT, LHS, RHS);
  }
 
  // The CMP instruction is just an alias for SUBS, and representing it as
  // SUBS means that it's possible to get CSE with subtract operations.
  // A later phase can perform the optimization of setting the destination
  // register to WZR/XZR if it ends up being unused.
  unsigned Opcode = AArch64ISD::SUBS;
 
  if (isCMN(RHS, CC, DAG)) {
    // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
    Opcode = AArch64ISD::ADDS;
    RHS = RHS.getOperand(1);
  } else if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) &&
             isIntEqualitySetCC(CC)) {
    // As we are looking for EQ/NE compares, the operands can be commuted ; can
    // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
    Opcode = AArch64ISD::ADDS;
    LHS = LHS.getOperand(1);
  } else if (isNullConstant(RHS) && !isUnsignedIntSetCC(CC)) {
    if (LHS.getOpcode() == ISD::AND) {
      // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
      // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
      // of the signed comparisons.
      const SDValue ANDSNode =
          DAG.getNode(AArch64ISD::ANDS, DL, DAG.getVTList(VT, FlagsVT),
                      LHS.getOperand(0), LHS.getOperand(1));
      // Replace all users of (and X, Y) with newly generated (ands X, Y)
      DAG.ReplaceAllUsesWith(LHS, ANDSNode);
      return ANDSNode.getValue(1);
    } else if (LHS.getOpcode() == AArch64ISD::ANDS) {
      // Use result of ANDS
      return LHS.getValue(1);
    }
  }
 
  return DAG.getNode(Opcode, DL, DAG.getVTList(VT, FlagsVT), LHS, RHS)
      .getValue(1);
}
 
/// \defgroup AArch64CCMP CMP;CCMP matching
///
/// These functions deal with the formation of CMP;CCMP;... sequences.
/// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
/// a comparison. They set the NZCV flags to a predefined value if their
/// predicate is false. This allows to express arbitrary conjunctions, for
/// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B)))"
/// expressed as:
///   cmp A
///   ccmp B, inv(CB), CA
///   check for CB flags
///
/// This naturally lets us implement chains of AND operations with SETCC
/// operands. And we can even implement some other situations by transforming
/// them:
///   - We can implement (NEG SETCC) i.e. negating a single comparison by
///     negating the flags used in a CCMP/FCCMP operations.
///   - We can negate the result of a whole chain of CMP/CCMP/FCCMP operations
///     by negating the flags we test for afterwards. i.e.
///     NEG (CMP CCMP CCCMP ...) can be implemented.
///   - Note that we can only ever negate all previously processed results.
///     What we can not implement by flipping the flags to test is a negation
///     of two sub-trees (because the negation affects all sub-trees emitted so
///     far, so the 2nd sub-tree we emit would also affect the first).
/// With those tools we can implement some OR operations:
///   - (OR (SETCC A) (SETCC B)) can be implemented via:
///     NEG (AND (NEG (SETCC A)) (NEG (SETCC B)))
///   - After transforming OR to NEG/AND combinations we may be able to use NEG
///     elimination rules from earlier to implement the whole thing as a
///     CCMP/FCCMP chain.
///
/// As complete example:
///     or (or (setCA (cmp A)) (setCB (cmp B)))
///        (and (setCC (cmp C)) (setCD (cmp D)))"
/// can be reassociated to:
///     or (and (setCC (cmp C)) setCD (cmp D))
//         (or (setCA (cmp A)) (setCB (cmp B)))
/// can be transformed to:
///     not (and (not (and (setCC (cmp C)) (setCD (cmp D))))
///              (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
/// which can be implemented as:
///   cmp C
///   ccmp D, inv(CD), CC
///   ccmp A, CA, inv(CD)
///   ccmp B, CB, inv(CA)
///   check for CB flags
///
/// A counterexample is "or (and A B) (and C D)" which translates to
/// not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we
/// can only implement 1 of the inner (not) operations, but not both!
/// @{
 
/// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
                                         ISD::CondCode CC, SDValue CCOp,
                                         AArch64CC::CondCode Predicate,
                                         AArch64CC::CondCode OutCC,
                                         const SDLoc &DL, SelectionDAG &DAG) {
  unsigned Opcode = 0;
  const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
  if (LHS.getValueType().isFloatingPoint()) {
    assert(LHS.getValueType() != MVT::f128);
    if ((LHS.getValueType() == MVT::f16 && !FullFP16) ||
        LHS.getValueType() == MVT::bf16) {
      LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
      RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
    }
    Opcode = AArch64ISD::FCCMP;
  } else if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(RHS)) {
    APInt Imm = Const->getAPIntValue();
    if (Imm.isNegative() && Imm.sgt(-32)) {
      Opcode = AArch64ISD::CCMN;
      RHS = DAG.getConstant(Imm.abs(), DL, Const->getValueType(0));
    }
  } else if (isCMN(RHS, CC, DAG)) {
    Opcode = AArch64ISD::CCMN;
    RHS = RHS.getOperand(1);
  } else if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) &&
             isIntEqualitySetCC(CC)) {
    // As we are looking for EQ/NE compares, the operands can be commuted ; can
    // we combine a (CCMP (sub 0, op1), op2) into a CCMN instruction ?
    Opcode = AArch64ISD::CCMN;
    LHS = LHS.getOperand(1);
  }
  if (Opcode == 0)
    Opcode = AArch64ISD::CCMP;
 
  SDValue Condition = getCondCode(DAG, Predicate);
  AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
  unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
  SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
  return DAG.getNode(Opcode, DL, FlagsVT, LHS, RHS, NZCVOp, Condition, CCOp);
}
 
/// Returns true if @p Val is a tree of AND/OR/SETCC operations that can be
/// expressed as a conjunction. See \ref AArch64CCMP.
/// \param CanNegate    Set to true if we can negate the whole sub-tree just by
///                     changing the conditions on the SETCC tests.
///                     (this means we can call emitConjunctionRec() with
///                      Negate==true on this sub-tree)
/// \param MustBeFirst  Set to true if this subtree needs to be negated and we
///                     cannot do the negation naturally. We are required to
///                     emit the subtree first in this case.
/// \param PreferFirst  Set to true if processing this subtree first may
///                     result in more efficient code.
/// \param WillNegate   Is true if are called when the result of this
///                     subexpression must be negated. This happens when the
///                     outer expression is an OR. We can use this fact to know
///                     that we have a double negation (or (or ...) ...) that
///                     can be implemented for free.
static bool canEmitConjunction(SelectionDAG &DAG, const SDValue Val,
                               bool &CanNegate, bool &MustBeFirst,
                               bool &PreferFirst, bool WillNegate,
                               unsigned Depth = 0) {
  if (!Val.hasOneUse())
    return false;
  unsigned Opcode = Val->getOpcode();
  if (Opcode == ISD::SETCC) {
    EVT VT = Val->getOperand(0).getValueType();
    if (VT == MVT::f128)
      return false;
    CanNegate = true;
    MustBeFirst = false;
    // Designate this operation as a preferred first operation if the result
    // of a SUB operation can be reused.
    PreferFirst = DAG.doesNodeExist(ISD::SUB, DAG.getVTList(VT),
                                    {Val->getOperand(0), Val->getOperand(1)});
    return true;
  }
  // Protect against exponential runtime and stack overflow.
  if (Depth > 6)
    return false;
  if (Opcode == ISD::AND || Opcode == ISD::OR) {
    bool IsOR = Opcode == ISD::OR;
    SDValue O0 = Val->getOperand(0);
    SDValue O1 = Val->getOperand(1);
    bool CanNegateL;
    bool MustBeFirstL;
    bool PreferFirstL;
    if (!canEmitConjunction(DAG, O0, CanNegateL, MustBeFirstL, PreferFirstL,
                            IsOR, Depth + 1))
      return false;
    bool CanNegateR;
    bool MustBeFirstR;
    bool PreferFirstR;
    if (!canEmitConjunction(DAG, O1, CanNegateR, MustBeFirstR, PreferFirstR,
                            IsOR, Depth + 1))
      return false;
 
    if (MustBeFirstL && MustBeFirstR)
      return false;
 
    if (IsOR) {
      // For an OR expression we need to be able to naturally negate at least
      // one side or we cannot do the transformation at all.
      if (!CanNegateL && !CanNegateR)
        return false;
      // If we the result of the OR will be negated and we can naturally negate
      // the leaves, then this sub-tree as a whole negates naturally.
      CanNegate = WillNegate && CanNegateL && CanNegateR;
      // If we cannot naturally negate the whole sub-tree, then this must be
      // emitted first.
      MustBeFirst = !CanNegate;
    } else {
      assert(Opcode == ISD::AND && "Must be OR or AND");
      // We cannot naturally negate an AND operation.
      CanNegate = false;
      MustBeFirst = MustBeFirstL || MustBeFirstR;
    }
    PreferFirst = PreferFirstL || PreferFirstR;
    return true;
  }
  return false;
}
 
/// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
/// of CCMP/CFCMP ops. See @ref AArch64CCMP.
/// Tries to transform the given i1 producing node @p Val to a series compare
/// and conditional compare operations. @returns an NZCV flags producing node
/// and sets @p OutCC to the flags that should be tested or returns SDValue() if
/// transformation was not possible.
/// \p Negate is true if we want this sub-tree being negated just by changing
/// SETCC conditions.
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
    AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
    AArch64CC::CondCode Predicate) {
  // We're at a tree leaf, produce a conditional comparison operation.
  unsigned Opcode = Val->getOpcode();
  if (Opcode == ISD::SETCC) {
    SDValue LHS = Val->getOperand(0);
    SDValue RHS = Val->getOperand(1);
    ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
    bool isInteger = LHS.getValueType().isInteger();
    if (Negate)
      CC = getSetCCInverse(CC, LHS.getValueType());
    SDLoc DL(Val);
    // Determine OutCC and handle FP special case.
    if (isInteger) {
      OutCC = changeIntCCToAArch64CC(CC, RHS);
    } else {
      assert(LHS.getValueType().isFloatingPoint());
      AArch64CC::CondCode ExtraCC;
      changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
      // Some floating point conditions can't be tested with a single condition
      // code. Construct an additional comparison in this case.
      if (ExtraCC != AArch64CC::AL) {
        SDValue ExtraCmp;
        if (!CCOp.getNode())
          ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
        else
          ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
                                               ExtraCC, DL, DAG);
        CCOp = ExtraCmp;
        Predicate = ExtraCC;
      }
    }
 
    // Produce a normal comparison if we are first in the chain
    if (!CCOp)
      return emitComparison(LHS, RHS, CC, DL, DAG);
    // Otherwise produce a ccmp.
    return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
                                     DAG);
  }
  assert(Val->hasOneUse() && "Valid conjunction/disjunction tree");
 
  bool IsOR = Opcode == ISD::OR;
 
  SDValue LHS = Val->getOperand(0);
  bool CanNegateL;
  bool MustBeFirstL;
  bool PreferFirstL;
  bool ValidL = canEmitConjunction(DAG, LHS, CanNegateL, MustBeFirstL,
                                   PreferFirstL, IsOR);
  assert(ValidL && "Valid conjunction/disjunction tree");
  (void)ValidL;
 
  SDValue RHS = Val->getOperand(1);
  bool CanNegateR;
  bool MustBeFirstR;
  bool PreferFirstR;
  bool ValidR = canEmitConjunction(DAG, RHS, CanNegateR, MustBeFirstR,
                                   PreferFirstR, IsOR);
  assert(ValidR && "Valid conjunction/disjunction tree");
  (void)ValidR;
 
  bool ShouldFirstL = PreferFirstL && !PreferFirstR && !MustBeFirstR;
 
  // Swap sub-tree that must or should come first to the right side.
  if (MustBeFirstL || ShouldFirstL) {
    assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
    std::swap(LHS, RHS);
    std::swap(CanNegateL, CanNegateR);
    std::swap(MustBeFirstL, MustBeFirstR);
  }
 
  bool NegateR;
  bool NegateAfterR;
  bool NegateL;
  bool NegateAfterAll;
  if (Opcode == ISD::OR) {
    // Swap the sub-tree that we can negate naturally to the left.
    if (!CanNegateL) {
      assert(CanNegateR && "at least one side must be negatable");
      assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
      assert(!Negate);
      std::swap(LHS, RHS);
      NegateR = false;
      NegateAfterR = true;
    } else {
      // Negate the left sub-tree if possible, otherwise negate the result.
      NegateR = CanNegateR;
      NegateAfterR = !CanNegateR;
    }
    NegateL = true;
    NegateAfterAll = !Negate;
  } else {
    assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree");
    assert(!Negate && "Valid conjunction/disjunction tree");
 
    NegateL = false;
    NegateR = false;
    NegateAfterR = false;
    NegateAfterAll = false;
  }
 
  // Emit sub-trees.
  AArch64CC::CondCode RHSCC;
  SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
  if (NegateAfterR)
    RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
  SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
  if (NegateAfterAll)
    OutCC = AArch64CC::getInvertedCondCode(OutCC);
  return CmpL;
}
 
/// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
/// In some cases this is even possible with OR operations in the expression.
/// See \ref AArch64CCMP.
/// \see emitConjunctionRec().
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
                               AArch64CC::CondCode &OutCC) {
  bool DummyCanNegate;
  bool DummyMustBeFirst;
  bool DummyPreferFirst;
  if (!canEmitConjunction(DAG, Val, DummyCanNegate, DummyMustBeFirst,
                          DummyPreferFirst, false))
    return SDValue();
 
  return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
}
 
/// @}
 
static bool isSupportedExtend(SDValue V) {
  if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
    return true;
 
  if (V.getOpcode() == ISD::AND)
    if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
      uint64_t Mask = MaskCst->getZExtValue();
      return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
    }
  return false;
}
 
/// Returns how profitable it is to fold a comparison's operand's shift and/or
/// extension operations.
static unsigned getCmpOperandFoldingProfit(SDValue Op, bool AllowExtend) {
  if (!Op.hasOneUse())
    return 0;
 
  if (AllowExtend && isSupportedExtend(Op))
    return 1;
 
  unsigned Opc = Op.getOpcode();
  if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
    if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
      uint64_t Shift = ShiftCst->getZExtValue();
      if (AllowExtend && isSupportedExtend(Op.getOperand(0)))
        return (Shift <= 4) ? 2 : 1;
      EVT VT = Op.getValueType();
      if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
        return 1;
    }
 
  return 0;
}
 
static unsigned getCmpOrCmnOperandFoldingProfit(SDValue Op, ISD::CondCode CC,
                                                SelectionDAG &DAG) {
  if (isCMN(Op, CC, DAG))
    return getCmpOperandFoldingProfit(Op.getOperand(1), true) + 1;
  return getCmpOperandFoldingProfit(Op, true);
}
 
// emitComparison() converts comparison with one or negative one to comparison
// with 0. Note that this only works for signed comparisons because of how ANDS
// works.
static bool shouldBeAdjustedToZero(SDValue LHS, const APInt &C,
                                   ISD::CondCode &CC) {
  // Only works for ANDS and AND.
  if (LHS.getOpcode() != ISD::AND && LHS.getOpcode() != AArch64ISD::ANDS)
    return false;
 
  if (C.isOne() && (CC == ISD::SETLT || CC == ISD::SETGE)) {
    CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
    return true;
  }
 
  if (C.isAllOnes() && (CC == ISD::SETLE || CC == ISD::SETGT)) {
    CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
    return true;
  }
 
  return false;
}
 
static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                             SDValue &AArch64cc, SelectionDAG &DAG,
                             const SDLoc &DL) {
  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
    EVT VT = RHS.getValueType();
    APInt C = RHSC->getAPIntValue();
    // shouldBeAdjustedToZero is a special case to better fold with
    // emitComparison().
    if (shouldBeAdjustedToZero(LHS, C, CC)) {
      // Adjust the constant to zero.
      // CC has already been adjusted.
      RHS = DAG.getConstant(0, DL, VT);
    } else if (!AArch64_AM::isLegalCmpImmed(C)) {
      unsigned NumImmForC = numberOfInstrToLoadImm(C);
      // Constant does not fit, try adjusting it by one?
      switch (CC) {
      default:
        break;
      case ISD::SETLT:
      case ISD::SETGE:
        if (!C.isMinSignedValue()) {
          APInt CMinusOne = C - 1;
          if (AArch64_AM::isLegalCmpImmed(CMinusOne) ||
              (NumImmForC > numberOfInstrToLoadImm(CMinusOne))) {
            CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
            RHS = DAG.getConstant(CMinusOne, DL, VT);
          }
        }
        break;
      case ISD::SETULT:
      case ISD::SETUGE: {
        // C is not 0 because it is a legal immediate.
        assert(!C.isZero() && "C should not be zero here");
        APInt CMinusOne = C - 1;
        if (AArch64_AM::isLegalCmpImmed(CMinusOne) ||
            (NumImmForC > numberOfInstrToLoadImm(CMinusOne))) {
          CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
          RHS = DAG.getConstant(CMinusOne, DL, VT);
        }
        break;
      }
      case ISD::SETLE:
      case ISD::SETGT:
        if (!C.isMaxSignedValue()) {
          APInt CPlusOne = C + 1;
          if (AArch64_AM::isLegalCmpImmed(CPlusOne) ||
              (NumImmForC > numberOfInstrToLoadImm(CPlusOne))) {
            CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
            RHS = DAG.getConstant(CPlusOne, DL, VT);
          }
        }
        break;
      case ISD::SETULE:
      case ISD::SETUGT:
        if (!C.isAllOnes()) {
          APInt CPlusOne = C + 1;
          if (AArch64_AM::isLegalCmpImmed(CPlusOne) ||
              (NumImmForC > numberOfInstrToLoadImm(CPlusOne))) {
            CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
            RHS = DAG.getConstant(CPlusOne, DL, VT);
          }
        }
        break;
      }
    } else if (C.isZero() && getCmpOperandFoldingProfit(LHS, false) != 0) {
      // Fold into CMP WZR, reg, shift #amount
      std::swap(LHS, RHS);
      CC = ISD::getSetCCSwappedOperands(CC);
    }
  }
 
  // Comparisons are canonicalized so that the RHS operand is simpler than the
  // LHS one, the extreme case being when RHS is an immediate. However, AArch64
  // can fold some shift+extend operations on the RHS operand, so swap the
  // operands if that can be done.
  //
  // For example:
  //    lsl     w13, w11, #1
  //    cmp     w13, w12
  // can be turned into:
  //    cmp     w12, w11, lsl #1
  if (!isa<ConstantSDNode>(RHS) ||
      !AArch64_AM::isLegalCmpImmed(RHS->getAsAPIntVal())) {
    if (getCmpOrCmnOperandFoldingProfit(LHS, CC, DAG) >
        getCmpOrCmnOperandFoldingProfit(RHS, CC, DAG)) {
      std::swap(LHS, RHS);
      CC = ISD::getSetCCSwappedOperands(CC);
    }
  }
 
  SDValue Cmp;
  AArch64CC::CondCode AArch64CC;
  if (isIntEqualitySetCC(CC) && isa<ConstantSDNode>(RHS)) {
    const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
 
    // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
    // For the i8 operand, the largest immediate is 255, so this can be easily
    // encoded in the compare instruction. For the i16 operand, however, the
    // largest immediate cannot be encoded in the compare.
    // Therefore, use a sign extending load and cmn to avoid materializing the
    // -1 constant. For example,
    // movz w1, #65535
    // ldrh w0, [x0, #0]
    // cmp w0, w1
    // >
    // ldrsh w0, [x0, #0]
    // cmn w0, #1
    // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
    // if and only if (sext LHS) == (sext RHS). The checks are in place to
    // ensure both the LHS and RHS are truly zero extended and to make sure the
    // transformation is profitable.
    if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
        cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
        cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
        LHS->hasNUsesOfValue(1, 0)) {
      int16_t ValueofRHS = RHS->getAsZExtVal();
      if (ValueofRHS < 0 && AArch64_AM::isLegalArithImmed(-ValueofRHS)) {
        SDValue SExt =
            DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, LHS.getValueType(), LHS,
                        DAG.getValueType(MVT::i16));
        Cmp = emitComparison(
            SExt, DAG.getSignedConstant(ValueofRHS, DL, RHS.getValueType()), CC,
            DL, DAG);
        AArch64CC = changeIntCCToAArch64CC(CC);
      }
    }
 
    if (!Cmp && (RHSC->isZero() || RHSC->isOne())) {
      if ((Cmp = emitConjunction(DAG, LHS, AArch64CC))) {
        if ((CC == ISD::SETNE) ^ RHSC->isZero())
          AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
      }
    }
  }
 
  if (!Cmp) {
    Cmp = emitComparison(LHS, RHS, CC, DL, DAG);
    AArch64CC = changeIntCCToAArch64CC(CC, RHS);
  }
  AArch64cc = getCondCode(DAG, AArch64CC);
  return Cmp;
}
 
static std::pair<SDValue, SDValue>
getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
  assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&
         "Unsupported value type");
  SDValue Value, Overflow;
  SDLoc DL(Op);
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
  unsigned Opc = 0;
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Unknown overflow instruction!");
  case ISD::SADDO:
    Opc = AArch64ISD::ADDS;
    CC = AArch64CC::VS;
    break;
  case ISD::UADDO:
    Opc = AArch64ISD::ADDS;
    CC = AArch64CC::HS;
    break;
  case ISD::SSUBO:
    Opc = AArch64ISD::SUBS;
    CC = AArch64CC::VS;
    break;
  case ISD::USUBO:
    Opc = AArch64ISD::SUBS;
    CC = AArch64CC::LO;
    break;
  // Multiply needs a little bit extra work.
  case ISD::SMULO:
  case ISD::UMULO: {
    CC = AArch64CC::NE;
    bool IsSigned = Op.getOpcode() == ISD::SMULO;
    if (Op.getValueType() == MVT::i32) {
      // Extend to 64-bits, then perform a 64-bit multiply.
      unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
      LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
      RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
      SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
      Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
 
      // Check that the result fits into a 32-bit integer.
      SDVTList VTs = DAG.getVTList(MVT::i64, FlagsVT);
      if (IsSigned) {
        // cmp xreg, wreg, sxtw
        SDValue SExtMul = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Value);
        Overflow =
            DAG.getNode(AArch64ISD::SUBS, DL, VTs, Mul, SExtMul).getValue(1);
      } else {
        // tst xreg, #0xffffffff00000000
        SDValue UpperBits = DAG.getConstant(0xFFFFFFFF00000000, DL, MVT::i64);
        Overflow =
            DAG.getNode(AArch64ISD::ANDS, DL, VTs, Mul, UpperBits).getValue(1);
      }
      break;
    }
    assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type");
    // For the 64 bit multiply
    Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
    if (IsSigned) {
      SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
      SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
                                      DAG.getConstant(63, DL, MVT::i64));
      // It is important that LowerBits is last, otherwise the arithmetic
      // shift will not be folded into the compare (SUBS).
      SDVTList VTs = DAG.getVTList(MVT::i64, FlagsVT);
      Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
                     .getValue(1);
    } else {
      SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
      SDVTList VTs = DAG.getVTList(MVT::i64, FlagsVT);
      Overflow =
          DAG.getNode(AArch64ISD::SUBS, DL, VTs,
                      DAG.getConstant(0, DL, MVT::i64),
                      UpperBits).getValue(1);
    }
    break;
  }
  } // switch (...)
 
  if (Opc) {
    SDVTList VTs = DAG.getVTList(Op->getValueType(0), FlagsVT);
 
    // Emit the AArch64 operation with overflow check.
    Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
    Overflow = Value.getValue(1);
  }
  return std::make_pair(Value, Overflow);
}
 
SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
  if (useSVEForFixedLengthVectorVT(Op.getValueType(),
                                   !Subtarget->isNeonAvailable()))
    return LowerToScalableOp(Op, DAG);
 
  SDValue Sel = Op.getOperand(0);
  SDValue Other = Op.getOperand(1);
  SDLoc DL(Sel);
 
  // If the operand is an overflow checking operation, invert the condition
  // code and kill the Not operation. I.e., transform:
  // (xor (overflow_op_bool, 1))
  //   -->
  // (csel 1, 0, invert(cc), overflow_op_bool)
  // ... which later gets transformed to just a cset instruction with an
  // inverted condition code, rather than a cset + eor sequence.
  if (isOneConstant(Other) && ISD::isOverflowIntrOpRes(Sel)) {
    // Only lower legal XALUO ops.
    if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
      return SDValue();
 
    SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
    SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
    AArch64CC::CondCode CC;
    SDValue Value, Overflow;
    std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
    SDValue CCVal = getCondCode(DAG, getInvertedCondCode(CC));
    return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
                       CCVal, Overflow);
  }
  // If neither operand is a SELECT_CC, give up.
  if (Sel.getOpcode() != ISD::SELECT_CC)
    std::swap(Sel, Other);
  if (Sel.getOpcode() != ISD::SELECT_CC)
    return Op;
 
  // The folding we want to perform is:
  // (xor x, (select_cc a, b, cc, 0, -1) )
  //   -->
  // (csel x, (xor x, -1), cc ...)
  //
  // The latter will get matched to a CSINV instruction.
 
  ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
  SDValue LHS = Sel.getOperand(0);
  SDValue RHS = Sel.getOperand(1);
  SDValue TVal = Sel.getOperand(2);
  SDValue FVal = Sel.getOperand(3);
 
  // FIXME: This could be generalized to non-integer comparisons.
  if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
    return Op;
 
  ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
  ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
 
  // The values aren't constants, this isn't the pattern we're looking for.
  if (!CFVal || !CTVal)
    return Op;
 
  // We can commute the SELECT_CC by inverting the condition.  This
  // might be needed to make this fit into a CSINV pattern.
  if (CTVal->isAllOnes() && CFVal->isZero()) {
    std::swap(TVal, FVal);
    std::swap(CTVal, CFVal);
    CC = ISD::getSetCCInverse(CC, LHS.getValueType());
  }
 
  // If the constants line up, perform the transform!
  if (CTVal->isZero() && CFVal->isAllOnes()) {
    SDValue CCVal;
    SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, DL);
 
    FVal = Other;
    TVal = DAG.getNode(ISD::XOR, DL, Other.getValueType(), Other,
                       DAG.getAllOnesConstant(DL, Other.getValueType()));
 
    return DAG.getNode(AArch64ISD::CSEL, DL, Sel.getValueType(), FVal, TVal,
                       CCVal, Cmp);
  }
 
  return Op;
}
 
// If Invert is false, sets 'C' bit of NZCV to 0 if value is 0, else sets 'C'
// bit to 1. If Invert is true, sets 'C' bit of NZCV to 1 if value is 0, else
// sets 'C' bit to 0.
static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
  SDLoc DL(Value);
  EVT VT = Value.getValueType();
  SDValue Op0 = Invert ? DAG.getConstant(0, DL, VT) : Value;
  SDValue Op1 = Invert ? Value : DAG.getConstant(1, DL, VT);
  SDValue Cmp =
      DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, FlagsVT), Op0, Op1);
  return Cmp.getValue(1);
}
 
// If Invert is false, value is 1 if 'C' bit of NZCV is 1, else 0.
// If Invert is true, value is 0 if 'C' bit of NZCV is 1, else 1.
static SDValue carryFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG,
                                bool Invert) {
  assert(Glue.getResNo() == 1);
  SDLoc DL(Glue);
  SDValue Zero = DAG.getConstant(0, DL, VT);
  SDValue One = DAG.getConstant(1, DL, VT);
  AArch64CC::CondCode Cond = Invert ? AArch64CC::LO : AArch64CC::HS;
  SDValue CC = getCondCode(DAG, Cond);
  return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Glue);
}
 
// Value is 1 if 'V' bit of NZCV is 1, else 0
static SDValue overflowFlagToValue(SDValue Glue, EVT VT, SelectionDAG &DAG) {
  assert(Glue.getResNo() == 1);
  SDLoc DL(Glue);
  SDValue Zero = DAG.getConstant(0, DL, VT);
  SDValue One = DAG.getConstant(1, DL, VT);
  SDValue CC = getCondCode(DAG, AArch64CC::VS);
  return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Glue);
}
 
// This lowering is inefficient, but it will get cleaned up by
// `foldOverflowCheck`
static SDValue lowerADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG,
                                  unsigned Opcode, bool IsSigned) {
  EVT VT0 = Op.getValue(0).getValueType();
  EVT VT1 = Op.getValue(1).getValueType();
 
  if (VT0 != MVT::i32 && VT0 != MVT::i64)
    return SDValue();
 
  bool InvertCarry = Opcode == AArch64ISD::SBCS;
  SDValue OpLHS = Op.getOperand(0);
  SDValue OpRHS = Op.getOperand(1);
  SDValue OpCarryIn = valueToCarryFlag(Op.getOperand(2), DAG, InvertCarry);
 
  SDLoc DL(Op);
 
  SDValue Sum = DAG.getNode(Opcode, DL, DAG.getVTList(VT0, FlagsVT), OpLHS,
                            OpRHS, OpCarryIn);
 
  SDValue OutFlag =
      IsSigned ? overflowFlagToValue(Sum.getValue(1), VT1, DAG)
               : carryFlagToValue(Sum.getValue(1), VT1, DAG, InvertCarry);
 
  return DAG.getMergeValues({Sum, OutFlag}, DL);
}
 
static SDValue lowerIntNeonIntrinsic(SDValue Op, unsigned Opcode,
                                     SelectionDAG &DAG,
                                     bool LastOperandIsImm = false) {
  if (Op.getValueType().isVector())
    return SDValue();
 
  SDLoc DL(Op);
  SmallVector<SDValue, 2> NewOps;
  const unsigned NumOperands = Op.getNumOperands();
  auto getFloatVT = [](EVT VT) {
    assert((VT == MVT::i32 || VT == MVT::i64) && "Unexpected VT");
    return VT == MVT::i32 ? MVT::f32 : MVT::f64;
  };
  auto bitcastToFloat = [&](SDValue Val) {
    return DAG.getBitcast(getFloatVT(Val.getValueType()), Val);
  };
 
  // Skip first operand as it is intrinsic ID.
  for (unsigned I = 1; I < NumOperands; ++I) {
    SDValue Val = Op.getOperand(I);
    const bool KeepInt = LastOperandIsImm && (I == NumOperands - 1);
    NewOps.push_back(KeepInt ? Val : bitcastToFloat(Val));
  }
  EVT OrigVT = Op.getValueType();
  SDValue OpNode = DAG.getNode(Opcode, DL, getFloatVT(OrigVT), NewOps);
  return DAG.getBitcast(OrigVT, OpNode);
}
 
static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
  // Let legalize expand this if it isn't a legal type yet.
  if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
    return SDValue();
 
  SDLoc DL(Op);
  AArch64CC::CondCode CC;
  // The actual operation that sets the overflow or carry flag.
  SDValue Value, Overflow;
  std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
 
  // We use 0 and 1 as false and true values.
  SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
  SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
 
  // We use an inverted condition, because the conditional select is inverted
  // too. This will allow it to be selected to a single instruction:
  // CSINC Wd, WZR, WZR, invert(cond).
  SDValue CCVal = getCondCode(DAG, getInvertedCondCode(CC));
  Overflow =
      DAG.getNode(AArch64ISD::CSEL, DL, MVT::i32, FVal, TVal, CCVal, Overflow);
 
  return DAG.getMergeValues({Value, Overflow}, DL);
}
 
// Prefetch operands are:
// 1: Address to prefetch
// 2: bool isWrite
// 3: int locality (0 = no locality ... 3 = extreme locality)
// 4: bool isDataCache
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  unsigned IsWrite = Op.getConstantOperandVal(2);
  unsigned Locality = Op.getConstantOperandVal(3);
  unsigned IsData = Op.getConstantOperandVal(4);
 
  bool IsStream = !Locality;
  // When the locality number is set
  if (Locality) {
    // The front-end should have filtered out the out-of-range values
    assert(Locality <= 3 && "Prefetch locality out-of-range");
    // The locality degree is the opposite of the cache speed.
    // Put the number the other way around.
    // The encoding starts at 0 for level 1
    Locality = 3 - Locality;
  }
 
  // built the mask value encoding the expected behavior.
  unsigned PrfOp = (IsWrite << 4) |     // Load/Store bit
                   (!IsData << 3) |     // IsDataCache bit
                   (Locality << 1) |    // Cache level bits
                   (unsigned)IsStream;  // Stream bit
  return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
                     DAG.getTargetConstant(PrfOp, DL, MVT::i32),
                     Op.getOperand(1));
}
 
// Converts SETCC (AND X Y) Z ULT -> SETCC (AND X (Y & ~(Z - 1)) 0 EQ when Y is
// a power of 2. This is then lowered to ANDS X (Y & ~(Z - 1)) instead of SUBS
// (AND X Y) Z which produces a better opt with EmitComparison
static void simplifySetCCIntoEq(ISD::CondCode &CC, SDValue &LHS, SDValue &RHS,
                                SelectionDAG &DAG, const SDLoc DL) {
  if (CC == ISD::SETULT && LHS.getOpcode() == ISD::AND && LHS->hasOneUse()) {
    ConstantSDNode *LHSConstOp = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
    ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
    if (LHSConstOp && RHSConst) {
      uint64_t LHSConstValue = LHSConstOp->getZExtValue();
      uint64_t RHSConstant = RHSConst->getZExtValue();
      if (isPowerOf2_64(RHSConstant)) {
        uint64_t NewMaskValue = LHSConstValue & ~(RHSConstant - 1);
        LHS =
            DAG.getNode(ISD::AND, DL, LHS.getValueType(), LHS.getOperand(0),
                        DAG.getConstant(NewMaskValue, DL, LHS.getValueType()));
        RHS = DAG.getConstant(0, DL, RHS.getValueType());
        CC = ISD::SETEQ;
      }
    }
  }
}
 
SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
                                              SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  if (VT.isScalableVector()) {
    SDValue SrcVal = Op.getOperand(0);
 
    if (VT == MVT::nxv2f64 && SrcVal.getValueType() == MVT::nxv2bf16) {
      // Break conversion in two with the first part converting to f32 and the
      // second using native f32->VT instructions.
      SDLoc DL(Op);
      return DAG.getNode(ISD::FP_EXTEND, DL, VT,
                         DAG.getNode(ISD::FP_EXTEND, DL, MVT::nxv2f32, SrcVal));
    }
 
    return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_EXTEND_MERGE_PASSTHRU);
  }
 
  if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()))
    return LowerFixedLengthFPExtendToSVE(Op, DAG);
 
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
  EVT Op0VT = Op0.getValueType();
  if (VT == MVT::f64) {
    // FP16->FP32 extends are legal for v32 and v4f32.
    if (Op0VT == MVT::f32 || Op0VT == MVT::f16)
      return Op;
    // Split bf16->f64 extends into two fpextends.
    if (Op0VT == MVT::bf16 && IsStrict) {
      SDValue Ext1 =
          DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(Op), {MVT::f32, MVT::Other},
                      {Op0, Op.getOperand(0)});
      return DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(Op), {VT, MVT::Other},
                         {Ext1, Ext1.getValue(1)});
    }
    if (Op0VT == MVT::bf16)
      return DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), VT,
                         DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Op0));
    return SDValue();
  }
 
  assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
  return SDValue();
}
 
SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
                                             SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
  EVT SrcVT = SrcVal.getValueType();
  bool Trunc = Op.getConstantOperandVal(IsStrict ? 2 : 1) == 1;
  SDNodeFlags Flags = Op->getFlags();
 
  if (VT.isScalableVector()) {
    // Let common code split the operation.
    if (SrcVT == MVT::nxv8f32)
      return Op;
 
    if (VT.getScalarType() != MVT::bf16)
      return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_ROUND_MERGE_PASSTHRU);
 
    SDLoc DL(Op);
    constexpr EVT I32 = MVT::nxv4i32;
    auto ImmV = [&](int I) -> SDValue { return DAG.getConstant(I, DL, I32); };
 
    SDValue NaN;
    SDValue Narrow;
 
    if (SrcVT == MVT::nxv2f32 || SrcVT == MVT::nxv4f32) {
      if (Subtarget->hasBF16())
        return LowerToPredicatedOp(Op, DAG,
                                   AArch64ISD::FP_ROUND_MERGE_PASSTHRU);
 
      Narrow = getSVESafeBitCast(I32, SrcVal, DAG);
 
      // Set the quiet bit.
      if (!DAG.isKnownNeverSNaN(SrcVal) && !Flags.hasNoNaNs())
        NaN = DAG.getNode(ISD::OR, DL, I32, Narrow, ImmV(0x400000));
    } else if (SrcVT == MVT::nxv2f64 &&
               (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable())) {
      // Round to float without introducing rounding errors and try again.
      SDValue Pg = getPredicateForVector(DAG, DL, MVT::nxv2f32);
      Narrow = DAG.getNode(AArch64ISD::FCVTX_MERGE_PASSTHRU, DL, MVT::nxv2f32,
                           Pg, SrcVal, DAG.getPOISON(MVT::nxv2f32));
 
      SmallVector<SDValue, 3> NewOps;
      if (IsStrict)
        NewOps.push_back(Op.getOperand(0));
      NewOps.push_back(Narrow);
      NewOps.push_back(Op.getOperand(IsStrict ? 2 : 1));
      return DAG.getNode(Op.getOpcode(), DL, VT, NewOps, Op->getFlags());
    } else
      return SDValue();
 
    if (!Trunc) {
      SDValue Lsb = DAG.getNode(ISD::SRL, DL, I32, Narrow, ImmV(16));
      Lsb = DAG.getNode(ISD::AND, DL, I32, Lsb, ImmV(1));
      SDValue RoundingBias = DAG.getNode(ISD::ADD, DL, I32, Lsb, ImmV(0x7fff));
      Narrow = DAG.getNode(ISD::ADD, DL, I32, Narrow, RoundingBias);
    }
 
    // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
    // 0x80000000.
    if (NaN) {
      EVT I1 = I32.changeElementType(*DAG.getContext(), MVT::i1);
      EVT CondVT = VT.changeElementType(*DAG.getContext(), MVT::i1);
      SDValue IsNaN = DAG.getSetCC(DL, CondVT, SrcVal, SrcVal, ISD::SETUO);
      IsNaN = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, I1, IsNaN);
      Narrow = DAG.getSelect(DL, I32, IsNaN, NaN, Narrow);
    }
 
    // Now that we have rounded, shift the bits into position.
    Narrow = DAG.getNode(ISD::SRL, DL, I32, Narrow, ImmV(16));
    return getSVESafeBitCast(VT, Narrow, DAG);
  }
 
  if (useSVEForFixedLengthVectorVT(SrcVT, !Subtarget->isNeonAvailable()))
    return LowerFixedLengthFPRoundToSVE(Op, DAG);
 
  // Expand cases where the result type is BF16 but we don't have hardware
  // instructions to lower it.
  if (VT.getScalarType() == MVT::bf16 &&
      !((Subtarget->hasNEON() || Subtarget->hasSME()) &&
        Subtarget->hasBF16())) {
    SDLoc DL(Op);
    SDValue Narrow = SrcVal;
    SDValue NaN;
    EVT I32 = SrcVT.changeElementType(*DAG.getContext(), MVT::i32);
    EVT F32 = SrcVT.changeElementType(*DAG.getContext(), MVT::f32);
    if (SrcVT.getScalarType() == MVT::f32) {
      bool NeverSNaN = DAG.isKnownNeverSNaN(Narrow);
      Narrow = DAG.getNode(ISD::BITCAST, DL, I32, Narrow);
      if (!NeverSNaN) {
        // Set the quiet bit.
        NaN = DAG.getNode(ISD::OR, DL, I32, Narrow,
                          DAG.getConstant(0x400000, DL, I32));
      }
    } else if (SrcVT.getScalarType() == MVT::f64) {
      Narrow = DAG.getNode(AArch64ISD::FCVTXN, DL, F32, Narrow);
      Narrow = DAG.getNode(ISD::BITCAST, DL, I32, Narrow);
    } else {
      return SDValue();
    }
    if (!Trunc) {
      SDValue One = DAG.getConstant(1, DL, I32);
      SDValue Lsb = DAG.getNode(ISD::SRL, DL, I32, Narrow,
                                DAG.getShiftAmountConstant(16, I32, DL));
      Lsb = DAG.getNode(ISD::AND, DL, I32, Lsb, One);
      SDValue RoundingBias =
          DAG.getNode(ISD::ADD, DL, I32, DAG.getConstant(0x7fff, DL, I32), Lsb);
      Narrow = DAG.getNode(ISD::ADD, DL, I32, Narrow, RoundingBias);
    }
 
    // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
    // 0x80000000.
    if (NaN) {
      SDValue IsNaN = DAG.getSetCC(
          DL, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT),
          SrcVal, SrcVal, ISD::SETUO);
      Narrow = DAG.getSelect(DL, I32, IsNaN, NaN, Narrow);
    }
 
    // Now that we have rounded, shift the bits into position.
    Narrow = DAG.getNode(ISD::SRL, DL, I32, Narrow,
                         DAG.getShiftAmountConstant(16, I32, DL));
    if (VT.isVector()) {
      EVT I16 = I32.changeVectorElementType(*DAG.getContext(), MVT::i16);
      Narrow = DAG.getNode(ISD::TRUNCATE, DL, I16, Narrow);
      return DAG.getNode(ISD::BITCAST, DL, VT, Narrow);
    }
    Narrow = DAG.getNode(ISD::BITCAST, DL, F32, Narrow);
    SDValue Result = DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Narrow);
    return IsStrict ? DAG.getMergeValues({Result, Op.getOperand(0)}, DL)
                    : Result;
  }
 
  if (SrcVT != MVT::f128) {
    // Expand cases where the input is a vector bigger than NEON.
    if (useSVEForFixedLengthVectorVT(SrcVT))
      return SDValue();
 
    // It's legal except when f128 is involved
    return Op;
  }
 
  return SDValue();
}
 
SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
                                                    SelectionDAG &DAG) const {
  // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
  // Any additional optimization in this function should be recorded
  // in the cost tables.
  bool IsStrict = Op->isStrictFPOpcode();
  EVT InVT = Op.getOperand(IsStrict ? 1 : 0).getValueType();
  EVT VT = Op.getValueType();
 
  assert(!(IsStrict && VT.isScalableVector()) &&
         "Unimplemented SVE support for STRICT_FP_to_INT!");
 
  // f16 conversions are promoted to f32 when full fp16 is not supported.
  if ((InVT.getVectorElementType() == MVT::f16 && !Subtarget->hasFullFP16()) ||
      InVT.getVectorElementType() == MVT::bf16) {
    EVT NewVT = VT.changeElementType(*DAG.getContext(), MVT::f32);
    SDLoc DL(Op);
    if (IsStrict) {
      SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {NewVT, MVT::Other},
                                {Op.getOperand(0), Op.getOperand(1)});
      return DAG.getNode(Op.getOpcode(), DL, {VT, MVT::Other},
                         {Ext.getValue(1), Ext.getValue(0)});
    }
    return DAG.getNode(
        Op.getOpcode(), DL, Op.getValueType(),
        DAG.getNode(ISD::FP_EXTEND, DL, NewVT, Op.getOperand(0)));
  }
 
  if (SDValue Res = LowerFPToIntToSVE(Op, DAG))
    return Res;
 
  uint64_t VTSize = VT.getFixedSizeInBits();
  uint64_t InVTSize = InVT.getFixedSizeInBits();
  if (VTSize < InVTSize) {
    SDLoc DL(Op);
    if (IsStrict) {
      InVT = InVT.changeVectorElementTypeToInteger();
      SDValue Cv = DAG.getNode(Op.getOpcode(), DL, {InVT, MVT::Other},
                               {Op.getOperand(0), Op.getOperand(1)});
      SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, Cv);
      return DAG.getMergeValues({Trunc, Cv.getValue(1)}, DL);
    }
    SDValue Cv =
        DAG.getNode(Op.getOpcode(), DL, InVT.changeVectorElementTypeToInteger(),
                    Op.getOperand(0));
    return DAG.getNode(ISD::TRUNCATE, DL, VT, Cv);
  }
 
  if (VTSize > InVTSize) {
    SDLoc DL(Op);
    MVT ExtVT =
        MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
                         VT.getVectorNumElements());
    if (IsStrict) {
      SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {ExtVT, MVT::Other},
                                {Op.getOperand(0), Op.getOperand(1)});
      return DAG.getNode(Op.getOpcode(), DL, {VT, MVT::Other},
                         {Ext.getValue(1), Ext.getValue(0)});
    }
    SDValue Ext = DAG.getNode(ISD::FP_EXTEND, DL, ExtVT, Op.getOperand(0));
    return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
  }
 
  // Use a scalar operation for conversions between single-element vectors of
  // the same size.
  if (InVT.getVectorNumElements() == 1) {
    SDLoc DL(Op);
    SDValue Extract = DAG.getNode(
        ISD::EXTRACT_VECTOR_ELT, DL, InVT.getScalarType(),
        Op.getOperand(IsStrict ? 1 : 0), DAG.getConstant(0, DL, MVT::i64));
    EVT ScalarVT = VT.getScalarType();
    if (IsStrict)
      return DAG.getNode(Op.getOpcode(), DL, {ScalarVT, MVT::Other},
                         {Op.getOperand(0), Extract});
    return DAG.getNode(Op.getOpcode(), DL, ScalarVT, Extract);
  }
 
  // Type changing conversions are illegal.
  return Op;
}
 
SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
                                              SelectionDAG &DAG) const {
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
 
  if (SrcVal.getValueType().isVector())
    return LowerVectorFP_TO_INT(Op, DAG);
 
  // f16 conversions are promoted to f32 when full fp16 is not supported.
  if ((SrcVal.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) ||
      SrcVal.getValueType() == MVT::bf16) {
    SDLoc DL(Op);
    if (IsStrict) {
      SDValue Ext =
          DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                      {Op.getOperand(0), SrcVal});
      return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
                         {Ext.getValue(1), Ext.getValue(0)});
    }
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                       DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, SrcVal));
  }
 
  if (SrcVal.getValueType() != MVT::f128) {
    // It's legal except when f128 is involved
    return Op;
  }
 
  return SDValue();
}
 
SDValue
AArch64TargetLowering::LowerVectorFP_TO_INT_SAT(SDValue Op,
                                                SelectionDAG &DAG) const {
  // AArch64 FP-to-int conversions saturate to the destination element size, so
  // we can lower common saturating conversions to simple instructions.
  SDValue SrcVal = Op.getOperand(0);
  const EVT SrcVT = SrcVal.getValueType();
  const EVT DstVT = Op.getValueType();
  const EVT DstElementVT = DstVT.getVectorElementType();
  const EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
 
  [[maybe_unused]] const uint64_t DstElementWidth = DstVT.getScalarSizeInBits();
  const uint64_t SrcElementWidth = SrcVT.getScalarSizeInBits();
  const uint64_t SatWidth = SatVT.getScalarSizeInBits();
  assert(SatWidth <= DstElementWidth &&
         "Saturation width cannot exceed result width");
 
  const EVT SrcElementVT = SrcVT.getVectorElementType();
  if (SrcElementVT != MVT::f64 && SrcElementVT != MVT::f32 &&
      SrcElementVT != MVT::f16 && SrcElementVT != MVT::bf16)
    return SDValue();
 
  if (SDValue Res = LowerFPToIntToSVE(Op, DAG))
    return Res;
 
  // Returns true if the operation can be matched by an isel pattern directly.
  auto CanHandleNatively = [&DstVT, &SatWidth](EVT SrcVT) -> bool {
    return SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits() &&
           SrcVT.getScalarSizeInBits() == SatWidth;
  };
 
  // Returns true if the operation is best expanded.
  auto Expand = [&SatWidth, &CanHandleNatively](EVT SrcVT) -> bool {
    return !CanHandleNatively(SrcVT) &&
           (SrcVT.getScalarSizeInBits() < SatWidth ||
            // NEON has no vector MIN/MAX for i64, so it's simpler to scalarize
            // (at least until sqxtn is selected).
            (SrcVT == MVT::v1f64 || SrcVT == MVT::v2f64));
  };
 
  // Try to promote the operation to a wider type if SrcVT < DstVT,
  // or if type is bf16 or if the target has no +fullfp16.
  std::optional<EVT> PromVT;
  switch (SrcElementVT.getSimpleVT().SimpleTy) {
  case MVT::f16:
  case MVT::bf16:
    if (DstElementVT == MVT::i32 || SrcElementVT == MVT::bf16 ||
        !Subtarget->hasFullFP16()) {
      PromVT = MVT::getVectorVT(MVT::f32, SrcVT.getVectorElementCount());
      break;
    }
    [[fallthrough]];
  case MVT::f32:
    // Promote to f64
    if (DstElementVT == MVT::i64) {
      PromVT = MVT::getVectorVT(MVT::f64, SrcVT.getVectorElementCount());
      break;
    }
    [[fallthrough]];
  default:
    break;
  }
 
  SDLoc DL(Op);
  unsigned Opc = Op.getOpcode();
  if (PromVT && !Expand(*PromVT)) {
    // When promoting the input type, SatWidth stays unchanged.
    SrcVal = DAG.getNode(ISD::FP_EXTEND, DL, *PromVT, SrcVal);
    if (*PromVT != MVT::v8f32 || isTypeLegal(MVT::v8f32))
      return DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, Op.getOperand(1));
 
    // If we are extending to a wider type (e.g. v8f16 -> v8f32) due to lack
    // of fp16 support, then it's more efficient to split the operation
    // into two v4f32 to produce legal types.
    auto [SrcValLo, SrcValHi] = DAG.SplitVector(SrcVal, DL);
    SDValue Lo = DAG.getNode(Opc, DL, MVT::v4i32, SrcValLo, Op.getOperand(1));
    SDValue Hi = DAG.getNode(Opc, DL, MVT::v4i32, SrcValHi, Op.getOperand(1));
    Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Lo);
    Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i16, Hi);
    return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Lo, Hi);
  }
 
  // Cases that we can emit directly.
  if (CanHandleNatively(SrcVT)) {
    assert(isTypeLegal(SrcVT) && "Expected SrcVT to be a legal type");
    return DAG.getNode(Opc, DL, DstVT, SrcVal,
                       DAG.getValueType(DstVT.getScalarType()));
  } else if (Expand(SrcVT)) {
    return SDValue();
  }
 
  assert((SrcElementWidth > DstElementWidth) ||
         (SrcElementWidth == DstElementWidth && SatWidth < DstElementWidth));
 
  EVT IntVT = SrcVT.changeVectorElementTypeToInteger();
  SDValue NativeCvt = DAG.getNode(Opc, DL, IntVT, SrcVal,
                                  DAG.getValueType(IntVT.getScalarType()));
  SDValue Sat;
  if (Opc == ISD::FP_TO_SINT_SAT) {
    SDValue MinC = DAG.getConstant(
        APInt::getSignedMaxValue(SatWidth).sext(SrcElementWidth), DL, IntVT);
    SDValue Min = DAG.getNode(ISD::SMIN, DL, IntVT, NativeCvt, MinC);
    SDValue MaxC = DAG.getConstant(
        APInt::getSignedMinValue(SatWidth).sext(SrcElementWidth), DL, IntVT);
    Sat = DAG.getNode(ISD::SMAX, DL, IntVT, Min, MaxC);
  } else {
    SDValue MinC = DAG.getConstant(
        APInt::getAllOnes(SatWidth).zext(SrcElementWidth), DL, IntVT);
    Sat = DAG.getNode(ISD::UMIN, DL, IntVT, NativeCvt, MinC);
  }
 
  return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Sat);
}
 
SDValue AArch64TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
                                                  SelectionDAG &DAG) const {
  // AArch64 FP-to-int conversions saturate to the destination register size, so
  // we can lower common saturating conversions to simple instructions.
  SDValue SrcVal = Op.getOperand(0);
  EVT SrcVT = SrcVal.getValueType();
 
  if (SrcVT.isVector())
    return LowerVectorFP_TO_INT_SAT(Op, DAG);
 
  EVT DstVT = Op.getValueType();
  EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  uint64_t SatWidth = SatVT.getScalarSizeInBits();
  uint64_t DstWidth = DstVT.getScalarSizeInBits();
  assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
 
  // In the absence of FP16 support, promote f16 to f32 and saturate the result.
  if ((SrcVT == MVT::f16 && !Subtarget->hasFullFP16()) || SrcVT == MVT::bf16) {
    SrcVal = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, SrcVal);
    SrcVT = MVT::f32;
  } else if (SrcVT != MVT::f64 && SrcVT != MVT::f32 && SrcVT != MVT::f16 &&
             SrcVT != MVT::bf16)
    return SDValue();
 
  SDLoc DL(Op);
  // Cases that we can emit directly.
  if ((SrcVT == MVT::f64 || SrcVT == MVT::f32 ||
       (SrcVT == MVT::f16 && Subtarget->hasFullFP16())) &&
      DstVT == SatVT && (DstVT == MVT::i64 || DstVT == MVT::i32))
    return DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal,
                       DAG.getValueType(DstVT));
 
  // Otherwise we emit a cvt that saturates to a higher BW, and saturate the
  // result. This is only valid if the legal cvt is larger than the saturate
  // width.
  if (DstWidth < SatWidth)
    return SDValue();
 
  if (SrcVT == MVT::f16 && SatVT == MVT::i16 && DstVT == MVT::i32) {
    if (Op.getOpcode() == ISD::FP_TO_SINT_SAT) {
      SDValue CVTf32 =
          DAG.getNode(AArch64ISD::FCVTZS_HALF, DL, MVT::f32, SrcVal);
      SDValue Bitcast = DAG.getBitcast(DstVT, CVTf32);
      return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, Bitcast,
                         DAG.getValueType(SatVT));
    }
    SDValue CVTf32 = DAG.getNode(AArch64ISD::FCVTZU_HALF, DL, MVT::f32, SrcVal);
    return DAG.getBitcast(DstVT, CVTf32);
  }
 
  SDValue NativeCvt =
      DAG.getNode(Op.getOpcode(), DL, DstVT, SrcVal, DAG.getValueType(DstVT));
  SDValue Sat;
  if (Op.getOpcode() == ISD::FP_TO_SINT_SAT) {
    SDValue MinC = DAG.getConstant(
        APInt::getSignedMaxValue(SatWidth).sext(DstWidth), DL, DstVT);
    SDValue Min = DAG.getNode(ISD::SMIN, DL, DstVT, NativeCvt, MinC);
    SDValue MaxC = DAG.getConstant(
        APInt::getSignedMinValue(SatWidth).sext(DstWidth), DL, DstVT);
    Sat = DAG.getNode(ISD::SMAX, DL, DstVT, Min, MaxC);
  } else {
    SDValue MinC = DAG.getConstant(
        APInt::getAllOnes(SatWidth).zext(DstWidth), DL, DstVT);
    Sat = DAG.getNode(ISD::UMIN, DL, DstVT, NativeCvt, MinC);
  }
 
  return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Sat);
}
 
SDValue AArch64TargetLowering::LowerVectorXRINT(SDValue Op,
                                                SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  SDValue Src = Op.getOperand(0);
  SDLoc DL(Op);
 
  assert(VT.isVector() && "Expected vector type");
 
  EVT CastVT = VT.changeVectorElementType(
      *DAG.getContext(), Src.getValueType().getVectorElementType());
 
  // Round the floating-point value into a floating-point register with the
  // current rounding mode.
  SDValue FOp = DAG.getNode(ISD::FRINT, DL, CastVT, Src);
 
  // Truncate the rounded floating point to an integer.
  return DAG.getNode(ISD::FP_TO_SINT_SAT, DL, VT, FOp,
                     DAG.getValueType(VT.getVectorElementType()));
}
 
SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
                                                    SelectionDAG &DAG) const {
  // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
  // Any additional optimization in this function should be recorded
  // in the cost tables.
  bool IsStrict = Op->isStrictFPOpcode();
  EVT VT = Op.getValueType();
  SDLoc DL(Op);
  SDValue In = Op.getOperand(IsStrict ? 1 : 0);
  EVT InVT = In.getValueType();
  unsigned Opc = Op.getOpcode();
  bool IsSigned = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
 
  assert(!(IsStrict && VT.isScalableVector()) &&
         "Unimplemented SVE support for ISD:::STRICT_INT_TO_FP!");
 
  // NOTE: i1->bf16 does not require promotion to f32.
  if (VT.isScalableVector() && InVT.getVectorElementType() == MVT::i1) {
    SDValue FalseVal = DAG.getConstantFP(0.0, DL, VT);
    SDValue TrueVal = IsSigned ? DAG.getConstantFP(-1.0, DL, VT)
                               : DAG.getConstantFP(1.0, DL, VT);
    return DAG.getNode(ISD::VSELECT, DL, VT, In, TrueVal, FalseVal);
  }
 
  // Promote bf16 conversions to f32.
  if (VT.getVectorElementType() == MVT::bf16) {
    EVT F32 = VT.changeElementType(*DAG.getContext(), MVT::f32);
    if (IsStrict) {
      SDValue Val = DAG.getNode(Op.getOpcode(), DL, {F32, MVT::Other},
                                {Op.getOperand(0), In});
      return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
                         {Op.getValueType(), MVT::Other},
                         {Val.getValue(1), Val.getValue(0),
                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
    }
    return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(),
                       DAG.getNode(Op.getOpcode(), DL, F32, In),
                       DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
  }
 
  if (VT.isScalableVector()) {
    // Let common code split the operation.
    if (VT == MVT::nxv8f32)
      return Op;
 
    unsigned Opcode = IsSigned ? AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU
                               : AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU;
    return LowerToPredicatedOp(Op, DAG, Opcode);
  }
 
  if (useSVEForFixedLengthVectorVT(VT, !Subtarget->isNeonAvailable()) ||
      useSVEForFixedLengthVectorVT(InVT, !Subtarget->isNeonAvailable()))
    return LowerFixedLengthIntToFPToSVE(Op, DAG);
 
  uint64_t VTSize = VT.getFixedSizeInBits();
  uint64_t InVTSize = InVT.getFixedSizeInBits();
  if (VTSize < InVTSize) {
    // AArch64 doesn't have a direct vector instruction to convert
    // fixed point to floating point AND narrow it at the same time.
    // Additional rounding when the target is f32/f64 causes double
    // rounding issues. Conversion to f16 is fine due to narrow width.
    bool IsTargetf32 = VT.getVectorElementType() == MVT::f32;
    bool IsTargetf16 = false;
    if (Op.hasOneUse() &&
        Op->user_begin()->getOpcode() == ISD::CONCAT_VECTORS) {
      // Some vector types are split during legalization into half, followed by
      // concatenation, followed by rounding to the original vector type. If we
      // end up resolving to f16 type, we shouldn't worry about rounding errors.
      SDNode *U = *Op->user_begin();
      if (U->hasOneUse() && U->user_begin()->getOpcode() == ISD::FP_ROUND) {
        EVT TmpVT = U->user_begin()->getValueType(0);
        if (TmpVT.getScalarType() == MVT::f16)
          IsTargetf16 = true;
      }
    }
 
    if (IsTargetf32 && !IsTargetf16) {
      return !IsStrict ? DAG.UnrollVectorOp(Op.getNode()) : SDValue();
    }
 
    MVT CastVT =
        MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
                         InVT.getVectorNumElements());
    if (IsStrict) {
      In = DAG.getNode(Opc, DL, {CastVT, MVT::Other}, {Op.getOperand(0), In});
      return DAG.getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
                         {In.getValue(1), In.getValue(0),
                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
    }
    In = DAG.getNode(Opc, DL, CastVT, In);
    return DAG.getNode(ISD::FP_ROUND, DL, VT, In,
                       DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
  }
 
  if (VTSize > InVTSize) {
    unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    EVT CastVT = VT.changeVectorElementTypeToInteger();
    In = DAG.getNode(CastOpc, DL, CastVT, In);
    if (IsStrict)
      return DAG.getNode(Opc, DL, {VT, MVT::Other}, {Op.getOperand(0), In});
    return DAG.getNode(Opc, DL, VT, In);
  }
 
  // Use a scalar operation for conversions between single-element vectors of
  // the same size.
  if (VT.getVectorNumElements() == 1) {
    SDValue Extract =
        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InVT.getScalarType(), In,
                    DAG.getConstant(0, DL, MVT::i64));
    EVT ScalarVT = VT.getScalarType();
    if (IsStrict)
      return DAG.getNode(Op.getOpcode(), DL, {ScalarVT, MVT::Other},
                         {Op.getOperand(0), Extract});
    return DAG.getNode(Op.getOpcode(), DL, ScalarVT, Extract);
  }
 
  return Op;
}
 
SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
                                            SelectionDAG &DAG) const {
  if (Op.getValueType().isVector())
    return LowerVectorINT_TO_FP(Op, DAG);
 
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
 
  bool IsSigned = Op->getOpcode() == ISD::STRICT_SINT_TO_FP ||
                  Op->getOpcode() == ISD::SINT_TO_FP;
 
  auto IntToFpViaPromotion = [&](EVT PromoteVT) {
    SDLoc DL(Op);
    if (IsStrict) {
      SDValue Val = DAG.getNode(Op.getOpcode(), DL, {PromoteVT, MVT::Other},
                                {Op.getOperand(0), SrcVal});
      return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
                         {Op.getValueType(), MVT::Other},
                         {Val.getValue(1), Val.getValue(0),
                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
    }
    return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(),
                       DAG.getNode(Op.getOpcode(), DL, PromoteVT, SrcVal),
                       DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
  };
 
  if (Op.getValueType() == MVT::bf16) {
    unsigned MaxWidth = IsSigned
                            ? DAG.ComputeMaxSignificantBits(SrcVal)
                            : DAG.computeKnownBits(SrcVal).countMaxActiveBits();
    // bf16 conversions are promoted to f32 when converting from i16.
    if (MaxWidth <= 24) {
      return IntToFpViaPromotion(MVT::f32);
    }
 
    // bf16 conversions are promoted to f64 when converting from i32.
    if (MaxWidth <= 53) {
      return IntToFpViaPromotion(MVT::f64);
    }
 
    // We need to be careful about i64 -> bf16.
    // Consider an i32 22216703.
    // This number cannot be represented exactly as an f32 and so a itofp will
    // turn it into 22216704.0 fptrunc to bf16 will turn this into 22282240.0
    // However, the correct bf16 was supposed to be 22151168.0
    // We need to use sticky rounding to get this correct.
    if (SrcVal.getValueType() == MVT::i64) {
      SDLoc DL(Op);
      // This algorithm is equivalent to the following:
      // uint64_t SrcHi = SrcVal & ~0xfffull;
      // uint64_t SrcLo = SrcVal &  0xfffull;
      // uint64_t Highest = SrcVal >> 53;
      // bool HasHighest = Highest != 0;
      // uint64_t ToRound = HasHighest ? SrcHi : SrcVal;
      // double  Rounded = static_cast<double>(ToRound);
      // uint64_t RoundedBits = std::bit_cast<uint64_t>(Rounded);
      // uint64_t HasLo = SrcLo != 0;
      // bool NeedsAdjustment = HasHighest & HasLo;
      // uint64_t AdjustedBits = RoundedBits | uint64_t{NeedsAdjustment};
      // double Adjusted = std::bit_cast<double>(AdjustedBits);
      // return static_cast<__bf16>(Adjusted);
      //
      // Essentially, what happens is that SrcVal either fits perfectly in a
      // double-precision value or it is too big. If it is sufficiently small,
      // we should just go u64 -> double -> bf16 in a naive way. Otherwise, we
      // ensure that u64 -> double has no rounding error by only using the 52
      // MSB of the input. The low order bits will get merged into a sticky bit
      // which will avoid issues incurred by double rounding.
 
      // Signed conversion is more or less like so:
      // copysign((__bf16)abs(SrcVal), SrcVal)
      SDValue SignBit;
      if (IsSigned) {
        SignBit = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
                              DAG.getConstant(1ull << 63, DL, MVT::i64));
        SrcVal = DAG.getNode(ISD::ABS, DL, MVT::i64, SrcVal);
      }
      SDValue SrcHi = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
                                  DAG.getConstant(~0xfffull, DL, MVT::i64));
      SDValue SrcLo = DAG.getNode(ISD::AND, DL, MVT::i64, SrcVal,
                                  DAG.getConstant(0xfffull, DL, MVT::i64));
      SDValue Highest =
          DAG.getNode(ISD::SRL, DL, MVT::i64, SrcVal,
                      DAG.getShiftAmountConstant(53, MVT::i64, DL));
      SDValue Zero64 = DAG.getConstant(0, DL, MVT::i64);
      SDValue ToRound =
          DAG.getSelectCC(DL, Highest, Zero64, SrcHi, SrcVal, ISD::SETNE);
      SDValue Rounded =
          IsStrict ? DAG.getNode(Op.getOpcode(), DL, {MVT::f64, MVT::Other},
                                 {Op.getOperand(0), ToRound})
                   : DAG.getNode(Op.getOpcode(), DL, MVT::f64, ToRound);
 
      SDValue RoundedBits = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Rounded);
      if (SignBit) {
        RoundedBits = DAG.getNode(ISD::OR, DL, MVT::i64, RoundedBits, SignBit);
      }
 
      SDValue HasHighest = DAG.getSetCC(
          DL,
          getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
          Highest, Zero64, ISD::SETNE);
 
      SDValue HasLo = DAG.getSetCC(
          DL,
          getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
          SrcLo, Zero64, ISD::SETNE);
 
      SDValue NeedsAdjustment =
          DAG.getNode(ISD::AND, DL, HasLo.getValueType(), HasHighest, HasLo);
      NeedsAdjustment = DAG.getZExtOrTrunc(NeedsAdjustment, DL, MVT::i64);
 
      SDValue AdjustedBits =
          DAG.getNode(ISD::OR, DL, MVT::i64, RoundedBits, NeedsAdjustment);
      SDValue Adjusted = DAG.getNode(ISD::BITCAST, DL, MVT::f64, AdjustedBits);
      return IsStrict
                 ? DAG.getNode(
                       ISD::STRICT_FP_ROUND, DL,
                       {Op.getValueType(), MVT::Other},
                       {Rounded.getValue(1), Adjusted,
                        DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)})
                 : DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), Adjusted,
                               DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
    }
  }
 
  // f16 conversions are promoted to f32 when full fp16 is not supported.
  if (Op.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
    return IntToFpViaPromotion(MVT::f32);
  }
 
  // i128 conversions are libcalls.
  if (SrcVal.getValueType() == MVT::i128)
    return SDValue();
 
  // Other conversions are legal, unless it's to the completely software-based
  // fp128.
  if (Op.getValueType() != MVT::f128)
    return Op;
  return SDValue();
}
 
static MVT getSVEContainerType(EVT ContentTy);
 
SDValue
AArch64TargetLowering::LowerLOOP_DEPENDENCE_MASK(SDValue Op,
                                                 SelectionDAG &DAG) const {
  assert((Subtarget->hasSVE2() ||
          (Subtarget->hasSME() && Subtarget->isStreaming())) &&
         "Lowering loop_dependence_raw_mask or loop_dependence_war_mask "
         "requires SVE or SME");
 
  SDLoc DL(Op);
  EVT VT = Op.getValueType();
  unsigned LaneOffset = Op.getConstantOperandVal(3);
  unsigned NumElements = VT.getVectorMinNumElements();
  uint64_t EltSizeInBytes = Op.getConstantOperandVal(2);
  EVT AddrTy = Op->getOperand(0).getValueType();
 
  // Lane offsets and other element sizes are not supported by whilewr/rw.
  if (LaneOffset != 0 || !is_contained({1u, 2u, 4u, 8u}, EltSizeInBytes))
    return SDValue();
 
  EVT EltVT = MVT::getIntegerVT(EltSizeInBytes * 8);
  EVT PredVT =
      getPackedSVEVectorVT(EltVT).changeElementType(*DAG.getContext(), MVT::i1);
 
  if (PredVT == VT) {
    // Legal whilewr/rw (lowered by tablegen matcher).
    if (AddrTy == MVT::i64)
      return Op;
 
    // Almost legal whilewr/rw (addresses must be promoted to i64).
    assert(AddrTy == MVT::i32 && "Only expected i32 to be legal!");
    return DAG.getNode(
        Op.getOpcode(), DL, VT,
        DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Op->getOperand(0)),
        DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Op->getOperand(1)),
        DAG.getConstant(EltSizeInBytes, DL, MVT::i64), Op->getOperand(3));
  }
 
  // Expand if this mask needs splitting (this will produce a whilelo).
  if (NumElements > PredVT.getVectorMinNumElements())
    return SDValue();
 
  SDValue Mask =
      DAG.getNode(Op.getOpcode(), DL, PredVT, to_vector(Op->op_values()));
 
  if (VT.isFixedLengthVector()) {
    EVT WidePredVT =
        PredVT.changeElementType(*DAG.getContext(), VT.getScalarType());
    SDValue MaskAsInt = DAG.getNode(ISD::SIGN_EXTEND, DL, WidePredVT, Mask);
    return convertFromScalableVector(DAG, VT, MaskAsInt);
  }
 
  return DAG.getExtractSubvector(DL, VT, Mask, 0);
}
 
SDValue AArch64TargetLowering::LowerBITCAST(SDValue Op,
                                            SelectionDAG &DAG) const {
  EVT OpVT = Op.getValueType();
  EVT ArgVT = Op.getOperand(0).getValueType();
 
  if (useSVEForFixedLengthVectorVT(OpVT))
    return LowerFixedLengthBitcastToSVE(Op, DAG);
 
  if (OpVT.isScalableVector()) {
    assert(isTypeLegal(OpVT) && "Unexpected result type!");
 
    // Handle type legalisation first.
    if (!isTypeLegal(ArgVT)) {
      assert(OpVT.isFloatingPoint() && !ArgVT.isFloatingPoint() &&
             "Expected int->fp bitcast!");
 
      // Bitcasting between unpacked vector types of different element counts is
      // not a NOP because the live elements are laid out differently.
      //                01234567
      // e.g. nxv2i32 = XX??XX??
      //      nxv4f16 = X?X?X?X?
      if (OpVT.getVectorElementCount() != ArgVT.getVectorElementCount())
        return SDValue();
 
      SDValue ExtResult =
          DAG.getNode(ISD::ANY_EXTEND, SDLoc(Op), getSVEContainerType(ArgVT),
                      Op.getOperand(0));
      return getSVESafeBitCast(OpVT, ExtResult, DAG);
    }
 
    // Bitcasts between legal types with the same element count are legal.
    if (OpVT.getVectorElementCount() == ArgVT.getVectorElementCount())
      return Op;
 
    // getSVESafeBitCast does not support casting between unpacked types.
    if (!isPackedVectorType(OpVT, DAG))
      return SDValue();
 
    return getSVESafeBitCast(OpVT, Op.getOperand(0), DAG);
  }
 
  if (OpVT != MVT::f16 && OpVT != MVT::bf16)
    return SDValue();
 
  // Bitcasts between f16 and bf16 are legal.
  if (ArgVT == MVT::f16 || ArgVT == MVT::bf16)
    return Op;
 
  SDValue Src = Op.getOperand(0);
  SDLoc DL(Op);
  if (ArgVT.isVector() && ArgVT.getSizeInBits() == 16) {
    Src = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Src);
    ArgVT = MVT::i16;
  }
 
  assert(ArgVT == MVT::i16);
 
  // If the input from a vector, extract directly from it.
  if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
    Op = DAG.getNode(ISD::BITCAST, DL,
                     Src.getOperand(0).getValueType().changeElementType(
                         *DAG.getContext(), OpVT),
                     Src.getOperand(0));
    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Op,
                       Src.getOperand(1));
  }
 
  Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8i16, Src);
  Op = DAG.getNode(ISD::BITCAST, DL,
                   EVT::getVectorVT(*DAG.getContext(), OpVT, 8), Op);
  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Op,
                     DAG.getConstant(0, DL, MVT::i64));
}
 
// Returns lane if Op extracts from a two-element vector and lane is constant
// (i.e., extractelt(<2 x Ty> %v, ConstantLane)), and std::nullopt otherwise.
static std::optional<uint64_t>
getConstantLaneNumOfExtractHalfOperand(SDValue &Op) {
  SDNode *OpNode = Op.getNode();
  if (OpNode->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return std::nullopt;
 
  EVT VT = OpNode->getOperand(0).getValueType();
  ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpNode->getOperand(1));
  if (!VT.isFixedLengthVector() || VT.getVectorNumElements() != 2 || !C)
    return std::nullopt;
 
  return C->getZExtValue();
}
 
static bool isExtendedBUILD_VECTOR(SDValue N, SelectionDAG &DAG,
                                   bool isSigned) {
  EVT VT = N.getValueType();
 
  if (N.getOpcode() != ISD::BUILD_VECTOR)
    return false;
 
  for (const SDValue &Elt : N->op_values()) {
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
      unsigned EltSize = VT.getScalarSizeInBits();
      unsigned HalfSize = EltSize / 2;
      if (isSigned) {
        if (!isIntN(HalfSize, C->getSExtValue()))
          return false;
      } else {
        if (!isUIntN(HalfSize, C->getZExtValue()))
          return false;
      }
      continue;
    }
    return false;
  }
 
  return true;
}
 
static SDValue skipExtensionForVectorMULL(SDValue N, SelectionDAG &DAG) {
  EVT VT = N.getValueType();
  assert(VT.is128BitVector() && "Unexpected vector MULL size");
  EVT HalfVT = EVT::getVectorVT(
      *DAG.getContext(),
      VT.getScalarType().getHalfSizedIntegerVT(*DAG.getContext()),
      VT.getVectorElementCount());
  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), HalfVT, N);
}
 
static bool isSignExtended(SDValue N, SelectionDAG &DAG) {
  return N.getOpcode() == ISD::SIGN_EXTEND ||
         N.getOpcode() == ISD::ANY_EXTEND ||
         isExtendedBUILD_VECTOR(N, DAG, true);
}
 
static bool isZeroExtended(SDValue N, SelectionDAG &DAG) {
  return N.getOpcode() == ISD::ZERO_EXTEND ||
         N.getOpcode() == ISD::ANY_EXTEND ||
         isExtendedBUILD_VECTOR(N, DAG, false);
}
 
static bool isAddSubSExt(SDValue N, SelectionDAG &DAG) {
  unsigned Opcode = N.getOpcode();
  if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
    SDValue N0 = N.getOperand(0);
    SDValue N1 = N.getOperand(1);
    return N0->hasOneUse() && N1->hasOneUse() &&
      isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
  }
  return false;
}
 
static bool isAddSubZExt(SDValue N, SelectionDAG &DAG) {
  unsigned Opcode = N.getOpcode();
  if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
    SDValue N0 = N.getOperand(0);
    SDValue N1 = N.getOperand(1);
    return N0->hasOneUse() && N1->hasOneUse() &&
      isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
  }
  return false;
}
 
SDValue AArch64TargetLowering::LowerGET_ROUNDING(SDValue Op,
                                                 SelectionDAG &DAG) const {
  // The rounding mode is in bits 23:22 of the FPCR.
  // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
  // The formula we use to implement this is (((FPCR + (1 << 22)) >> 22) & 3)
  // so that the shift + and get folded into a bitfield extract.
  SDLoc DL(Op);
 
  SDValue Chain = Op.getOperand(0);
  SDValue FPCR_64 =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other},
                  {Chain, DAG.getTargetConstant(Intrinsic::aarch64_get_fpcr, DL,
                                                MVT::i64)});
  Chain = FPCR_64.getValue(1);
  SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPCR_64);
  SDValue FltRounds = DAG.getNode(ISD::ADD, DL, MVT::i32, FPCR_32,
                                  DAG.getConstant(1U << 22, DL, MVT::i32));
  SDValue RMODE = DAG.getNode(ISD::SRL, DL, MVT::i32, FltRounds,
                              DAG.getConstant(22, DL, MVT::i32));
  SDValue AND = DAG.getNode(ISD::AND, DL, MVT::i32, RMODE,
                            DAG.getConstant(3, DL, MVT::i32));
  return DAG.getMergeValues({AND, Chain}, DL);
}
 
SDValue AArch64TargetLowering::LowerSET_ROUNDING(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue RMValue = Op->getOperand(1);
 
  // The rounding mode is in bits 23:22 of the FPCR.
  // The llvm.set.rounding argument value to the rounding mode in FPCR mapping
  // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
  // ((arg - 1) & 3) << 22).
  //
  // The argument of llvm.set.rounding must be within the segment [0, 3], so
  // NearestTiesToAway (4) is not handled here. It is responsibility of the code
  // generated llvm.set.rounding to ensure this condition.
 
  // Calculate new value of FPCR[23:22].
  RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
                        DAG.getConstant(1, DL, MVT::i32));
  RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
                        DAG.getConstant(0x3, DL, MVT::i32));
  RMValue =
      DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
                  DAG.getConstant(AArch64::RoundingBitsPos, DL, MVT::i32));
  RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, RMValue);
 
  // Get current value of FPCR.
  SDValue Ops[] = {
      Chain, DAG.getTargetConstant(Intrinsic::aarch64_get_fpcr, DL, MVT::i64)};
  SDValue FPCR =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops);
  Chain = FPCR.getValue(1);
  FPCR = FPCR.getValue(0);
 
  // Put new rounding mode into FPCR[23:22].
  const int RMMask = ~(AArch64::Rounding::rmMask << AArch64::RoundingBitsPos);
  FPCR = DAG.getNode(ISD::AND, DL, MVT::i64, FPCR,
                     DAG.getConstant(RMMask, DL, MVT::i64));
  FPCR = DAG.getNode(ISD::OR, DL, MVT::i64, FPCR, RMValue);
  SDValue Ops2[] = {
      Chain, DAG.getTargetConstant(Intrinsic::aarch64_set_fpcr, DL, MVT::i64),
      FPCR};
  return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
}
 
SDValue AArch64TargetLowering::LowerGET_FPMODE(SDValue Op,
                                               SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
 
  // Get current value of FPCR.
  SDValue Ops[] = {
      Chain, DAG.getTargetConstant(Intrinsic::aarch64_get_fpcr, DL, MVT::i64)};
  SDValue FPCR =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops);
  Chain = FPCR.getValue(1);
  FPCR = FPCR.getValue(0);
 
  // Truncate FPCR to 32 bits.
  SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPCR);
 
  return DAG.getMergeValues({Result, Chain}, DL);
}
 
SDValue AArch64TargetLowering::LowerSET_FPMODE(SDValue Op,
                                               SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue Mode = Op->getOperand(1);
 
  // Extend the specified value to 64 bits.
  SDValue FPCR = DAG.getZExtOrTrunc(Mode, DL, MVT::i64);
 
  // Set new value of FPCR.
  SDValue Ops2[] = {
      Chain, DAG.getTargetConstant(Intrinsic::aarch64_set_fpcr, DL, MVT::i64),
      FPCR};
  return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
}
 
SDValue AArch64TargetLowering::LowerRESET_FPMODE(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
 
  // Get current value of FPCR.
  SDValue Ops[] = {
      Chain, DAG.getTargetConstant(Intrinsic::aarch64_get_fpcr, DL, MVT::i64)};
  SDValue FPCR =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i64, MVT::Other}, Ops);
  Chain = FPCR.getValue(1);
  FPCR = FPCR.getValue(0);
 
  // Clear bits that are not reserved.
  SDValue FPCRMasked = DAG.getNode(
      ISD::AND, DL, MVT::i64, FPCR,
      DAG.getConstant(AArch64::ReservedFPControlBits, DL, MVT::i64));
 
  // Set new value of FPCR.
  SDValue Ops2[] = {
      Chain, DAG.getTargetConstant(Intrinsic::aarch64_set_fpcr, DL, MVT::i64),
      FPCRMasked};
  return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
}
 
static unsigned selectUmullSmull(SDValue &N0, SDValue &N1, SelectionDAG &DAG,
                                 SDLoc DL, bool &IsMLA) {
  bool IsN0SExt = isSignExtended(N0, DAG);
  bool IsN1SExt = isSignExtended(N1, DAG);
  if (IsN0SExt && IsN1SExt)
    return AArch64ISD::SMULL;
 
  bool IsN0ZExt = isZeroExtended(N0, DAG);
  bool IsN1ZExt = isZeroExtended(N1, DAG);
 
  if (IsN0ZExt && IsN1ZExt)
    return AArch64ISD::UMULL;
 
  // Select UMULL if we can replace the other operand with an extend.
  EVT VT = N0.getValueType();
  unsigned EltSize = VT.getScalarSizeInBits();
  APInt Mask = APInt::getHighBitsSet(EltSize, EltSize / 2);
  if (IsN0ZExt || IsN1ZExt) {
    if (DAG.MaskedValueIsZero(IsN0ZExt ? N1 : N0, Mask))
      return AArch64ISD::UMULL;
  } else if (VT == MVT::v2i64 && DAG.MaskedValueIsZero(N0, Mask) &&
             DAG.MaskedValueIsZero(N1, Mask)) {
    // For v2i64 we look more aggressively at both operands being zero, to avoid
    // scalarization.
    return AArch64ISD::UMULL;
  }
 
  if (IsN0SExt || IsN1SExt) {
    if (DAG.ComputeNumSignBits(IsN0SExt ? N1 : N0) > EltSize / 2)
      return AArch64ISD::SMULL;
  } else if (VT == MVT::v2i64 && DAG.ComputeNumSignBits(N0) > EltSize / 2 &&
             DAG.ComputeNumSignBits(N1) > EltSize / 2) {
    return AArch64ISD::SMULL;
  }
 
  if (!IsN1SExt && !IsN1ZExt)
    return 0;
 
  // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
  // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
  if (IsN1SExt && isAddSubSExt(N0, DAG)) {
    IsMLA = true;
    return AArch64ISD::SMULL;
  }
  if (IsN1ZExt && isAddSubZExt(N0, DAG)) {
    IsMLA = true;
    return AArch64ISD::UMULL;
  }
  if (IsN0ZExt && isAddSubZExt(N1, DAG)) {
    std::swap(N0, N1);
    IsMLA = true;
    return AArch64ISD::UMULL;
  }
  return 0;
}
 
// Transform mul<v2i64, splat(const)> into a SHL and ADD/SUB
// (1) multiply-by-(power-of-2 +/- 1) into shift and add/sub.
// mul x, (2^N + 1) --> add (shl x, N), x
// mul x, (2^N - 1) --> sub (shl x, N), x
// Examples: x * 33 --> (x << 5) + x
//           x * 15 --> (x << 4) - x
//           x * -33 --> -((x << 5) + x)
//           x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
// (2) multiply-by-(power-of-2 +/- power-of-2) into shifts and add/sub.
// mul x, (2^N + 2^M) --> (add (shl x, N), (shl x, M))
// mul x, (2^N - 2^M) --> (sub (shl x, N), (shl x, M))
// Examples: x * 0x8800 --> (x << 15) + (x << 11)
//           x * 0xf800 --> (x << 16) - (x << 11)
//           x * -0x8800 --> -((x << 15) + (x << 11))
//           x * -0xf800 --> -((x << 16) - (x << 11)) ; (x << 11) - (x << 16)
static SDValue convertMulToShlAdd(SDNode *N, SelectionDAG &DAG) {
  const SDNode *Operand = N->getOperand(1).getNode();
  APInt SplatValue;
 
  // Not a constant splat so should just stay as a multiplication operation
  if (!ISD::isConstantSplatVector(Operand, SplatValue) ||
      !SplatValue.getBoolValue())
    return SDValue();
 
  bool IsNegative = SplatValue.isNegative();
  SplatValue = SplatValue.abs();
  // Placeholder for MathOp
  unsigned MathOp = ISD::DELETED_NODE;
  unsigned TZeros = SplatValue.countr_zero();
 
  // Shift the splat value by all the zeros, this won't affect the parity
  // this will help us find the first and second multiple to use.
  SplatValue.lshrInPlace(TZeros);
 
  if ((SplatValue - 1).isPowerOf2())
    MathOp = ISD::ADD;
  else if ((SplatValue + 1).isPowerOf2())
    MathOp = ISD::SUB;
 
  // If the constant is not (2^n + 1) or (2^n - 1), it would require
  // more than one addition/subtraction. For v2i64, the cost of
  // multiple vector adds/shifts often exceeds the cost of
  // scalarization (moving to GPRs to use a single MUL).
  if (MathOp != ISD::DELETED_NODE) {
    SDLoc DL(N);
    EVT VT = N->getValueType(0);
    SDValue LHS = N->getOperand(0);
 
    unsigned ShiftAmt = MathOp == ISD::ADD ? (SplatValue - 1).logBase2()
                                           : (SplatValue + 1).logBase2();
    ShiftAmt += TZeros;
 
    SDValue Shl =
        DAG.getNode(ISD::SHL, DL, VT, LHS, DAG.getConstant(ShiftAmt, DL, VT));
 
    SDValue NewLHS = TZeros ? DAG.getNode(ISD::SHL, DL, VT, LHS,
                                          DAG.getConstant(TZeros, DL, VT))
                            : LHS;
    SDValue Combined = DAG.getNode(MathOp, DL, VT, Shl, NewLHS);
    if (IsNegative)
      Combined = DAG.getNegative(Combined, DL, VT);
    return Combined;
  }
  return SDValue();
}
 
SDValue AArch64TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
 
  bool OverrideNEON = !Subtarget->isNeonAvailable();
  if (VT.isScalableVector() || useSVEForFixedLengthVectorVT(VT, OverrideNEON))
    return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED);
 
  // Multiplications are only custom-lowered for 128-bit and 64-bit vectors so
  // that VMULL can be detected.  Otherwise v2i64 multiplications are not legal.
  assert((VT.is128BitVector() || VT.is64BitVector()) && VT.isInteger() &&
         "unexpected type for custom-lowering ISD::MUL");
  SDValue N0 = Op.getOperand(0);
  SDValue N1 = Op.getOperand(1);
  bool isMLA = false;
  EVT OVT = VT;
  if (VT.is64BitVector()) {
    if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
        isNullConstant(N0.getOperand(1)) &&
        N0.getOperand(0).getValueType().is128BitVector() &&
        N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
        isNullConstant(N1.getOperand(1)) &&
        N1.getOperand(0).getValueType().is128BitVector()) {
      N0 = N0.getOperand(0);
      N1 = N1.getOperand(0);
      VT = N0.getValueType();
    } else {
      if (VT == MVT::v1i64) {
        if (Subtarget->hasSVE())
          return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED);
        // Fall through to expand this.  It is not legal.
        return SDValue();
      } else
        // Other vector multiplications are legal.
        return Op;
    }
  }
 
  SDLoc DL(Op);
  unsigned NewOpc = selectUmullSmull(N0, N1, DAG, DL, isMLA);
 
  if (!NewOpc) {
    if (VT.getVectorElementType() == MVT::i64) {
      // If SVE is available then i64 vector multiplications can also be made
      // legal.
      if (Subtarget->hasSVE())
        return LowerToPredicatedOp(Op, DAG, AArch64ISD::MUL_PRED);
      // Try to optimize the mul to a shift left and add instead of scalarizing.
      if (SDValue ShlAdd = convertMulToShlAdd(Op.getNode(), DAG))
        return ShlAdd;
      // Fall through to expanding as the mul is not legal.
      return SDValue();
    } else
      // Other vector multiplications are legal.
      return Op;
  }
 
  // Legalize to a S/UMULL instruction
  SDValue Op0;
  SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
  if (!isMLA) {
    Op0 = skipExtensionForVectorMULL(N0, DAG);
    assert(Op0.getValueType().is64BitVector() &&
           Op1.getValueType().is64BitVector() &&
           "unexpected types for extended operands to VMULL");
    return DAG.getExtractSubvector(DL, OVT,
                                   DAG.getNode(NewOpc, DL, VT, Op0, Op1), 0);
  }
  // Optimizing (zext A + zext B) * C, to (S/UMULL A, C) + (S/UMULL B, C) during
  // isel lowering to take advantage of no-stall back to back s/umul + s/umla.
  // This is true for CPUs with accumulate forwarding such as Cortex-A53/A57
  SDValue N00 = skipExtensionForVectorMULL(N0.getOperand(0), DAG);
  SDValue N01 = skipExtensionForVectorMULL(N0.getOperand(1), DAG);
  EVT Op1VT = Op1.getValueType();
  return DAG.getNode(
      ISD::EXTRACT_SUBVECTOR, DL, OVT,
      DAG.getNode(N0.getOpcode(), DL, VT,
                  DAG.getNode(NewOpc, DL, VT,
                              DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
                  DAG.getNode(NewOpc, DL, VT,
                              DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)),
      DAG.getConstant(0, DL, MVT::i64));
}
 
static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
                               int Pattern) {
  if (Pattern == AArch64SVEPredPattern::all)
    return DAG.getConstant(1, DL, VT);
 
  // When the number of active elements of a pattern matches the scalable vector
  // length, we can upgrade the pattern to ALL and emit a splat instead.
  if (unsigned PatNumElts = getNumElementsFromSVEPredPattern(Pattern)) {
    const AArch64Subtarget &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
    unsigned NumElts = VT.getVectorMinNumElements();
    unsigned VScale = Subtarget.getSVEVectorSizeInBits() / 128;
    if (PatNumElts == (NumElts * VScale))
      return DAG.getConstant(1, DL, VT);
  }
 
  return DAG.getNode(AArch64ISD::PTRUE, DL, VT,
                     DAG.getTargetConstant(Pattern, DL, MVT::i32));
}
 
static inline SDValue getPTrueAsCounter(SelectionDAG &DAG, SDLoc DL, EVT VT) {
  Intrinsic::ID IID;
 
  switch (VT.getScalarSizeInBits()) {
  default:
    llvm_unreachable("unsupported predicate element size");
  case 8:
    IID = Intrinsic::aarch64_sve_ptrue_c8;
    break;
  case 16:
    IID = Intrinsic::aarch64_sve_ptrue_c16;
    break;
  case 32:
    IID = Intrinsic::aarch64_sve_ptrue_c32;
    break;
  case 64:
    IID = Intrinsic::aarch64_sve_ptrue_c64;
    break;
  }
 
  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
                     DAG.getConstant(IID, DL, MVT::i64));
}
 
static SDValue optimizeIncrementingWhile(SDNode *N, SelectionDAG &DAG,
                                         bool IsSigned, bool IsEqual) {
  unsigned Op0 = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 1 : 0;
  unsigned Op1 = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 2 : 1;
 
  if (!N->getValueType(0).isScalableVector() ||
      !isa<ConstantSDNode>(N->getOperand(Op1)))
    return SDValue();
 
  SDLoc DL(N);
  APInt Y = N->getConstantOperandAPInt(Op1);
 
  // When the second operand is the maximum value, comparisons that include
  // equality can never fail and thus we can return an all active predicate.
  if (IsEqual)
    if (IsSigned ? Y.isMaxSignedValue() : Y.isMaxValue())
      return DAG.getConstant(1, DL, N->getValueType(0));
 
  if (!isa<ConstantSDNode>(N->getOperand(Op0)))
    return SDValue();
 
  APInt X = N->getConstantOperandAPInt(Op0);
 
  bool Overflow;
  APInt NumActiveElems =
      IsSigned ? Y.ssub_ov(X, Overflow) : Y.usub_ov(X, Overflow);
 
  if (Overflow)
    return SDValue();
 
  if (IsEqual) {
    APInt One(NumActiveElems.getBitWidth(), 1, IsSigned);
    NumActiveElems = IsSigned ? NumActiveElems.sadd_ov(One, Overflow)
                              : NumActiveElems.uadd_ov(One, Overflow);
    if (Overflow)
      return SDValue();
  }
 
  std::optional<unsigned> PredPattern =
      getSVEPredPatternFromNumElements(NumActiveElems.getZExtValue());
  unsigned MinSVEVectorSize = std::max(
      DAG.getSubtarget<AArch64Subtarget>().getMinSVEVectorSizeInBits(), 128u);
  unsigned ElementSize = 128 / N->getValueType(0).getVectorMinNumElements();
  if (PredPattern != std::nullopt &&
      NumActiveElems.getZExtValue() <= (MinSVEVectorSize / ElementSize))
    return getPTrue(DAG, DL, N->getValueType(0), *PredPattern);
 
  return SDValue();
}
 
// Match get.active.lane.mask(0, cttz.elts(x)) -> brkb(x)
// Match get.active.lane.mask(0, add(cttz.elts(x), 1)) -> brka(x)
static SDValue optimizeBrk(SDNode *N, SelectionDAG &DAG) {
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
  // Lower bound must be 0.
  if (!isZeroOrZeroSplat(N->getOperand(0)))
    return SDValue();
 
  SDValue Upper = N->getOperand(1);
 
  // Default to brkb, switch to brka if we find a +1.
  unsigned BrkID = Intrinsic::aarch64_sve_brkb_z;
  if (Upper->getOpcode() == ISD::ADD && isOneOrOneSplat(Upper.getOperand(1))) {
    Upper = Upper.getOperand(0);
    BrkID = Intrinsic::aarch64_sve_brka_z;
  }
 
  // We're looking for an upper bound based on CTTZ_ELTS; this would be selected
  // as a cntp(brk(Pg, Mask)), but if we're just going to make a whilelo based
  // on that then we just need the brk.
  if (Upper.getOpcode() != AArch64ISD::CTTZ_ELTS || !VT.isScalableVector() ||
      Upper.getOperand(0).getValueType() != VT)
    return SDValue();
 
  SDValue Pg = Upper->getOperand(0);
  SDValue Mask = Upper->getOperand(1);
 
  // brk{a,b} only support .b forms, so cast to make sure all our p regs match.
  Pg = getSVEPredicateBitCast(MVT::nxv16i1, Pg, DAG);
  SDValue MaskR =
      DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, MVT::nxv16i1, Mask);
  SDValue ID = DAG.getTargetConstant(BrkID, DL, MVT::i64);
  SDValue Brk =
      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv16i1, ID, Pg, MaskR);
  return DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Brk);
}
 
// Returns a safe bitcast between two scalable vector predicates, where
// any newly created lanes from a widening bitcast are defined as zero.
static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  EVT InVT = Op.getValueType();
 
  assert(InVT.getVectorElementType() == MVT::i1 &&
         VT.getVectorElementType() == MVT::i1 &&
         "Expected a predicate-to-predicate bitcast");
  assert(VT.isScalableVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
         InVT.isScalableVector() &&
         DAG.getTargetLoweringInfo().isTypeLegal(InVT) &&
         "Only expect to cast between legal scalable predicate types!");
 
  // Return the operand if the cast isn't changing type,
  if (InVT == VT)
    return Op;
 
  // Look through casts to <vscale x 16 x i1> when their input has more lanes
  // than VT. This will increase the chances of removing casts that introduce
  // new lanes, which have to be explicitly zero'd.
  if (Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
      Op.getConstantOperandVal(0) == Intrinsic::aarch64_sve_convert_to_svbool &&
      Op.getOperand(1).getValueType().bitsGT(VT))
    Op = Op.getOperand(1);
 
  SDValue Reinterpret = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Op);
 
  // We only have to zero the lanes if new lanes are being defined, e.g. when
  // casting from <vscale x 2 x i1> to <vscale x 16 x i1>. If this is not the
  // case (e.g. when casting from <vscale x 16 x i1> -> <vscale x 2 x i1>) then
  // we can return here.
  if (InVT.bitsGT(VT))
    return Reinterpret;
 
  // Check if the other lanes are already known to be zeroed by
  // construction.
  if (isZeroingInactiveLanes(Op))
    return Reinterpret;
 
  // Zero the newly introduced lanes.
  SDValue Mask = DAG.getConstant(1, DL, InVT);
  Mask = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL, VT, Mask);
  return DAG.getNode(ISD::AND, DL, VT, Reinterpret, Mask);
}
 
SDValue AArch64TargetLowering::getRuntimePStateSM(SelectionDAG &DAG,
                                                  SDValue Chain, SDLoc DL,
                                                  EVT VT) const {
  RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE;
  RTLIB::LibcallImpl LCImpl = DAG.getLibcalls().getLibcallImpl(LC);
  SDValue Callee =
      DAG.getExternalSymbol(LCImpl, getPointerTy(DAG.getDataLayout()));
  Type *Int64Ty = Type::getInt64Ty(*DAG.getContext());
  Type *RetTy = StructType::get(Int64Ty, Int64Ty);
  TargetLowering::CallLoweringInfo CLI(DAG);
  ArgListTy Args;
  CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
      DAG.getLibcalls().getLibcallImplCallingConv(LCImpl), RetTy, Callee,
      std::move(Args));
  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
  SDValue Mask = DAG.getConstant(/*PSTATE.SM*/ 1, DL, MVT::i64);
  return DAG.getNode(ISD::AND, DL, MVT::i64, CallResult.first.getOperand(0),
                     Mask);
}
 
// Lower an SME LDR/STR ZA intrinsic
// Case 1: If the vector number (vecnum) is an immediate in range, it gets
// folded into the instruction
//    ldr(%tileslice, %ptr, 11) -> ldr [%tileslice, 11], [%ptr, 11]
// Case 2: If the vecnum is not an immediate, then it is used to modify the base
// and tile slice registers
//    ldr(%tileslice, %ptr, %vecnum)
//    ->
//    %svl = rdsvl
//    %ptr2 = %ptr + %svl * %vecnum
//    %tileslice2 = %tileslice + %vecnum
//    ldr [%tileslice2, 0], [%ptr2, 0]
// Case 3: If the vecnum is an immediate out of range, then the same is done as
// case 2, but the base and slice registers are modified by the greatest
// multiple of 15 lower than the vecnum and the remainder is folded into the
// instruction. This means that successive loads and stores that are offset from
// each other can share the same base and slice register updates.
//    ldr(%tileslice, %ptr, 22)
//    ldr(%tileslice, %ptr, 23)
//    ->
//    %svl = rdsvl
//    %ptr2 = %ptr + %svl * 15
//    %tileslice2 = %tileslice + 15
//    ldr [%tileslice2, 7], [%ptr2, 7]
//    ldr [%tileslice2, 8], [%ptr2, 8]
// Case 4: If the vecnum is an add of an immediate, then the non-immediate
// operand and the immediate can be folded into the instruction, like case 2.
//    ldr(%tileslice, %ptr, %vecnum + 7)
//    ldr(%tileslice, %ptr, %vecnum + 8)
//    ->
//    %svl = rdsvl
//    %ptr2 = %ptr + %svl * %vecnum
//    %tileslice2 = %tileslice + %vecnum
//    ldr [%tileslice2, 7], [%ptr2, 7]
//    ldr [%tileslice2, 8], [%ptr2, 8]
// Case 5: The vecnum being an add of an immediate out of range is also handled,
// in which case the same remainder logic as case 3 is used.
static SDValue LowerSMELdrStr(SDValue N, SelectionDAG &DAG, bool IsLoad) {
  SDLoc DL(N);
 
  SDValue TileSlice = N->getOperand(2);
  SDValue Base = N->getOperand(3);
  SDValue VecNum = N->getOperand(4);
  int32_t ConstAddend = 0;
  SDValue VarAddend = VecNum;
 
  // If the vnum is an add of an immediate, we can fold it into the instruction
  if (VecNum.getOpcode() == ISD::ADD &&
      isa<ConstantSDNode>(VecNum.getOperand(1))) {
    ConstAddend = cast<ConstantSDNode>(VecNum.getOperand(1))->getSExtValue();
    VarAddend = VecNum.getOperand(0);
  } else if (auto ImmNode = dyn_cast<ConstantSDNode>(VecNum)) {
    ConstAddend = ImmNode->getSExtValue();
    VarAddend = SDValue();
  }
 
  int32_t ImmAddend = ConstAddend % 16;
  if (int32_t C = (ConstAddend - ImmAddend)) {
    SDValue CVal = DAG.getTargetConstant(C, DL, MVT::i32);
    VarAddend = VarAddend
                    ? DAG.getNode(ISD::ADD, DL, MVT::i32, {VarAddend, CVal})
                    : CVal;
  }
 
  if (VarAddend) {
    // Get the vector length that will be multiplied by vnum
    auto SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
                           DAG.getConstant(1, DL, MVT::i32));
 
    // Multiply SVL and vnum then add it to the base
    SDValue Mul = DAG.getNode(
        ISD::MUL, DL, MVT::i64,
        {SVL, DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, VarAddend)});
    Base = DAG.getNode(ISD::ADD, DL, MVT::i64, {Base, Mul});
    // Just add vnum to the tileslice
    TileSlice = DAG.getNode(ISD::ADD, DL, MVT::i32, {TileSlice, VarAddend});
  }
 
  return DAG.getNode(IsLoad ? AArch64ISD::SME_ZA_LDR : AArch64ISD::SME_ZA_STR,
                     DL, MVT::Other,
                     {/*Chain=*/N.getOperand(0), TileSlice, Base,
                      DAG.getTargetConstant(ImmAddend, DL, MVT::i32)});
}
 
static SDValue LowerVectorMatch(SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  auto Op1 = Op.getOperand(0);
  auto Op2 = Op.getOperand(1);
  auto Mask = Op.getOperand(2);
 
  EVT Op1VT = Op1.getValueType();
  EVT Op2VT = Op2.getValueType();
  EVT ResVT = Op.getValueType();
 
  assert((Op1VT.getVectorElementType() == MVT::i8 ||
          Op1VT.getVectorElementType() == MVT::i16) &&
         "Expected 8-bit or 16-bit characters.");
 
  SDValue ID =
      DAG.getTargetConstant(Intrinsic::aarch64_sve_match, DL, MVT::i64);
 
  // Scalable vector type used to wrap operands.
  // A single container is enough for both operands because ultimately the
  // operands will have to be wrapped to the same type (nxv16i8 or nxv8i16).
  EVT OpContainerVT = Op1VT.isScalableVector()
                          ? Op1VT
                          : getContainerForFixedLengthVector(DAG, Op1VT);
 
  if (Op2VT.is128BitVector()) {
    // If Op2 is a full 128-bit vector, wrap it trivially in a scalable vector.
    Op2 = convertToScalableVector(DAG, OpContainerVT, Op2);
    // Further, if the result is scalable, broadcast Op2 to a full SVE register.
    if (ResVT.isScalableVector())
      Op2 = DAG.getNode(AArch64ISD::DUPLANE128, DL, OpContainerVT, Op2,
                        DAG.getTargetConstant(0, DL, MVT::i64));
  } else {
    // If Op2 is not a full 128-bit vector, we always need to broadcast it.
    unsigned Op2BitWidth = Op2VT.getFixedSizeInBits();
    MVT Op2IntVT = MVT::getIntegerVT(Op2BitWidth);
    EVT Op2PromotedVT = getPackedSVEVectorVT(Op2IntVT);
    Op2 = DAG.getBitcast(MVT::getVectorVT(Op2IntVT, 1), Op2);
    Op2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op2IntVT, Op2,
                      DAG.getConstant(0, DL, MVT::i64));
    Op2 = DAG.getSplatVector(Op2PromotedVT, DL, Op2);
    Op2 = DAG.getBitcast(OpContainerVT, Op2);
  }
 
  // If the result is scalable, we just need to carry out the MATCH.
  if (ResVT.isScalableVector())
    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResVT, ID, Mask, Op1, Op2);
 
  // If the result is fixed, we can still use MATCH but we need to wrap the
  // first operand and the mask in scalable vectors before doing so.
 
  // Wrap the operands.
  Op1 = convertToScalableVector(DAG, OpContainerVT, Op1);
  Mask = DAG.getNode(ISD::SIGN_EXTEND, DL, Op1VT, Mask);
  Mask = convertFixedMaskToScalableVector(Mask, DAG);
 
  // Carry out the match.
  SDValue Match = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Mask.getValueType(),
                              ID, Mask, Op1, Op2);
 
  // Extract and promote the match result (nxv16i1/nxv8i1) to ResVT
  // (v16i8/v8i8).
  Match = DAG.getNode(ISD::SIGN_EXTEND, DL, OpContainerVT, Match);
  Match = convertFromScalableVector(DAG, Op1VT, Match);
  return DAG.getNode(ISD::TRUNCATE, DL, ResVT, Match);
}
 
SDValue AArch64TargetLowering::LowerINTRINSIC_VOID(SDValue Op,
                                                   SelectionDAG &DAG) const {
  unsigned IntNo = Op.getConstantOperandVal(1);
  SDLoc DL(Op);
  switch (IntNo) {
  default:
    return SDValue(); // Don't custom lower most intrinsics.
  case Intrinsic::aarch64_prefetch: {
    SDValue Chain = Op.getOperand(0);
    SDValue Addr = Op.getOperand(2);
 
    unsigned IsWrite = Op.getConstantOperandVal(3);
    unsigned Locality = Op.getConstantOperandVal(4);
    unsigned IsStream = Op.getConstantOperandVal(5);
    unsigned IsData = Op.getConstantOperandVal(6);
    unsigned PrfOp = (IsWrite << 4) |    // Load/Store bit
                     (!IsData << 3) |    // IsDataCache bit
                     (Locality << 1) |   // Cache level bits
                     (unsigned)IsStream; // Stream bit
 
    return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Chain,
                       DAG.getTargetConstant(PrfOp, DL, MVT::i32), Addr);
  }
  case Intrinsic::aarch64_range_prefetch: {
    SDValue Chain = Op.getOperand(0);
    SDValue Addr = Op.getOperand(2);
 
    unsigned IsWrite = Op.getConstantOperandVal(3);
    unsigned IsStream = Op.getConstantOperandVal(4);
    unsigned PrfOp = (IsStream << 2) | IsWrite;
 
    SDValue Metadata = Op.getOperand(5);
    return DAG.getNode(AArch64ISD::RANGE_PREFETCH, DL, MVT::Other, Chain,
                       DAG.getTargetConstant(PrfOp, DL, MVT::i32), Addr,
                       Metadata);
  }
  case Intrinsic::aarch64_prefetch_ir:
    return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other,
                       Op.getOperand(0),                        // Chain
                       DAG.getTargetConstant(24, DL, MVT::i32), // Rt
                       Op.getOperand(2));                       // Addr
  case Intrinsic::aarch64_sme_str:
  case Intrinsic::aarch64_sme_ldr: {
    return LowerSMELdrStr(Op, DAG, IntNo == Intrinsic::aarch64_sme_ldr);
  }
  case Intrinsic::aarch64_sme_za_enable:
    return DAG.getNode(
        AArch64ISD::SMSTART, DL, DAG.getVTList(MVT::Other, MVT::Glue),
        Op->getOperand(0), // Chain
        DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32));
  case Intrinsic::aarch64_sme_za_disable:
    return DAG.getNode(
        AArch64ISD::SMSTOP, DL, DAG.getVTList(MVT::Other, MVT::Glue),
        Op->getOperand(0), // Chain
        DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32));
  }
}
 
SDValue AArch64TargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
                                                      SelectionDAG &DAG) const {
  unsigned IntNo = Op.getConstantOperandVal(1);
  SDLoc DL(Op);
  switch (IntNo) {
  default:
    return SDValue(); // Don't custom lower most intrinsics.
  case Intrinsic::aarch64_svc:
  case Intrinsic::aarch64_hvc: {
    // The MSVC __svc/__hvc intrinsic takes the 16-bit instruction immediate as
    // their first operand and four further operands passed in X0-X3 (an unused
    // argument is passed as poison) and returns the value left in X0.  Matching
    // MSVC, the instruction is not treated as clobbering the caller-saved
    // registers; only X0 (the result) is defined.
    SDValue Chain = Op.getOperand(0);
    unsigned Imm = Op.getConstantOperandVal(2);
 
    static const MCPhysReg ArgGPRs[] = {AArch64::X0, AArch64::X1, AArch64::X2,
                                        AArch64::X3};
 
    SDValue Glue;
    SmallVector<SDValue, 4> RegOps;
    for (unsigned I = 0; I < std::size(ArgGPRs); ++I) {
      SDValue Arg = Op.getOperand(3 + I);
      if (Arg.isUndef())
        continue;
      Chain = DAG.getCopyToReg(Chain, DL, ArgGPRs[I], Arg, Glue);
      Glue = Chain.getValue(1);
      RegOps.push_back(DAG.getRegister(ArgGPRs[I], MVT::i64));
    }
 
    SmallVector<SDValue, 8> Ops;
    Ops.push_back(Chain);
    Ops.push_back(DAG.getTargetConstant(Imm, DL, MVT::i32));
    Ops.append(RegOps.begin(), RegOps.end());
    if (Glue.getNode())
      Ops.push_back(Glue);
 
    unsigned Opc =
        IntNo == Intrinsic::aarch64_svc ? AArch64ISD::SVC : AArch64ISD::HVC;
    SDValue Node =
        DAG.getNode(Opc, DL, DAG.getVTList(MVT::Other, MVT::Glue), Ops);
    Chain = Node.getValue(0);
    Glue = Node.getValue(1);
 
    SDValue Result = DAG.getCopyFromReg(Chain, DL, AArch64::X0, MVT::i64, Glue);
    return DAG.getMergeValues({Result.getValue(0), Result.getValue(1)}, DL);
  }
  case Intrinsic::aarch64_mops_memset_tag: {
    auto Node = cast<MemIntrinsicSDNode>(Op.getNode());
    SDValue Chain = Node->getChain();
    SDValue Dst = Op.getOperand(2);
    SDValue Val = Op.getOperand(3);
    Val = DAG.getAnyExtOrTrunc(Val, DL, MVT::i64);
    SDValue Size = Op.getOperand(4);
    auto Alignment = Node->getMemOperand()->getAlign();
    bool IsVol = Node->isVolatile();
    auto DstPtrInfo = Node->getPointerInfo();
 
    const auto &SDI =
        static_cast<const AArch64SelectionDAGInfo &>(DAG.getSelectionDAGInfo());
    SDValue MS = SDI.EmitMOPS(AArch64::MOPSMemorySetTaggingPseudo, DAG, DL,
                              Chain, Dst, Val, Size, Alignment, Alignment,
                              IsVol, DstPtrInfo, MachinePointerInfo{});
 
    // MOPS_MEMSET_TAGGING has 3 results (DstWb, SizeWb, Chain) whereas the
    // intrinsic has 2. So hide SizeWb using MERGE_VALUES. Otherwise
    // LowerOperationWrapper will complain that the number of results has
    // changed.
    return DAG.getMergeValues({MS.getValue(0), MS.getValue(2)}, DL);
  }
  }
}
 
SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                     SelectionDAG &DAG) const {
  unsigned IntNo = Op.getConstantOperandVal(0);
  SDLoc DL(Op);
  switch (IntNo) {
  default: return SDValue();    // Don't custom lower most intrinsics.
  case Intrinsic::thread_pointer: {
    EVT PtrVT = getPointerTy(DAG.getDataLayout());
    return DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
  }
  case Intrinsic::aarch64_sve_whilewr_b:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(1, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_sve_whilewr_h:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(2, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_sve_whilewr_s:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(4, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_sve_whilewr_d:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_WAR_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(8, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_sve_whilerw_b:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(1, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_sve_whilerw_h:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(2, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_sve_whilerw_s:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(4, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_sve_whilerw_d:
    return DAG.getNode(ISD::LOOP_DEPENDENCE_RAW_MASK, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(8, DL, MVT::i64),
                       DAG.getConstant(0, DL, MVT::i64));
  case Intrinsic::aarch64_neon_abs: {
    EVT Ty = Op.getValueType();
    if (Ty == MVT::i64) {
      SDValue Result =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i64, Op.getOperand(1));
      Result = DAG.getNode(ISD::ABS, DL, MVT::v1i64, Result);
      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, Result,
                         DAG.getConstant(0, DL, MVT::i64));
    } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
      return DAG.getNode(ISD::ABS, DL, Ty, Op.getOperand(1));
    } else {
      report_fatal_error("Unexpected type for AArch64 NEON intrinsic");
    }
  }
  case Intrinsic::aarch64_neon_addhn: {
    return DAG.getNode(AArch64ISD::ADDHN, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::aarch64_neon_pmull64: {
    SDValue LHS = Op.getOperand(1);
    SDValue RHS = Op.getOperand(2);
 
    std::optional<uint64_t> LHSLane =
        getConstantLaneNumOfExtractHalfOperand(LHS);
    std::optional<uint64_t> RHSLane =
        getConstantLaneNumOfExtractHalfOperand(RHS);
 
    assert((!LHSLane || *LHSLane < 2) && "Expect lane to be None or 0 or 1");
    assert((!RHSLane || *RHSLane < 2) && "Expect lane to be None or 0 or 1");
 
    // 'aarch64_neon_pmull64' takes i64 parameters; while pmull/pmull2
    // instructions execute on SIMD registers. So canonicalize i64 to v1i64,
    // which ISel recognizes better. For example, generate a ldr into d*
    // registers as opposed to a GPR load followed by a fmov.
    auto TryVectorizeOperand = [](SDValue N, std::optional<uint64_t> NLane,
                                  std::optional<uint64_t> OtherLane,
                                  const SDLoc &DL,
                                  SelectionDAG &DAG) -> SDValue {
      // If the operand is an higher half itself, rewrite it to
      // extract_high_v2i64; this way aarch64_neon_pmull64 could
      // re-use the dag-combiner function with aarch64_neon_{pmull,smull,umull}.
      if (NLane == 1)
        return DAG.getExtractSubvector(DL, MVT::v1i64, N.getOperand(0), 1);
 
      // Operand N is not a higher half but the other operand is.
      if (OtherLane == 1) {
        // If this operand is a lower half, rewrite it to
        // extract_high_v2i64(duplane(<2 x Ty>, 0)). This saves a roundtrip to
        // align lanes of two operands. A roundtrip sequence (to move from lane
        // 1 to lane 0) is like this:
        //   mov x8, v0.d[1]
        //   fmov d0, x8
        if (NLane == 0)
          return DAG.getExtractSubvector(
              DL, MVT::v1i64,
              DAG.getNode(AArch64ISD::DUPLANE64, DL, MVT::v2i64,
                          N.getOperand(0), DAG.getConstant(0, DL, MVT::i64)),
              1);
 
        // Otherwise just dup from main to all lanes.
        return DAG.getNode(AArch64ISD::DUP, DL, MVT::v1i64, N);
      }
 
      // Neither operand is an extract of higher half, so codegen may just use
      // the non-high version of PMULL instruction. Use v1i64 to represent i64.
      assert(N.getValueType() == MVT::i64 &&
             "Intrinsic aarch64_neon_pmull64 requires i64 parameters");
      return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i64, N);
    };
 
    LHS = TryVectorizeOperand(LHS, LHSLane, RHSLane, DL, DAG);
    RHS = TryVectorizeOperand(RHS, RHSLane, LHSLane, DL, DAG);
 
    return DAG.getNode(AArch64ISD::PMULL, DL, Op.getValueType(), LHS, RHS);
  }
  case Intrinsic::aarch64_neon_smax:
    return DAG.getNode(ISD::SMAX, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  case Intrinsic::aarch64_neon_umax:
    return DAG.getNode(ISD::UMAX, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  case Intrinsic::aarch64_neon_smin:
    return DAG.getNode(ISD::SMIN, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  case Intrinsic::aarch64_neon_umin:
    return DAG.getNode(ISD::UMIN, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  case Intrinsic::aarch64_neon_scalar_sqxtn:
  case Intrinsic::aarch64_neon_scalar_sqxtun:
  case Intrinsic::aarch64_neon_scalar_uqxtn: {
    assert(Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::f32);
    if (Op.getValueType() == MVT::i32)
      return DAG.getNode(ISD::BITCAST, DL, MVT::i32,
                         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::f32,
                                     Op.getOperand(0),
                                     DAG.getNode(ISD::BITCAST, DL, MVT::f64,
                                                 Op.getOperand(1))));
    return SDValue();
  }
  case Intrinsic::aarch64_neon_sqxtn:
    return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_neon_sqxtun:
    return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_neon_uqxtn:
    return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_neon_sqshrn:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, Op.getValueType(),
                         DAG.getNode(AArch64ISD::VASHR, DL,
                                     Op.getOperand(1).getValueType(),
                                     Op.getOperand(1), Op.getOperand(2)));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQSHRN, DAG,
                                 /*LastOperandIsImm=*/true);
  case Intrinsic::aarch64_neon_sqshrun:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, Op.getValueType(),
                         DAG.getNode(AArch64ISD::VASHR, DL,
                                     Op.getOperand(1).getValueType(),
                                     Op.getOperand(1), Op.getOperand(2)));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQSHRUN, DAG,
                                 /*LastOperandIsImm=*/true);
  case Intrinsic::aarch64_neon_uqshrn:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, Op.getValueType(),
                         DAG.getNode(AArch64ISD::VLSHR, DL,
                                     Op.getOperand(1).getValueType(),
                                     Op.getOperand(1), Op.getOperand(2)));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::UQSHRN, DAG,
                                 /*LastOperandIsImm=*/true);
  case Intrinsic::aarch64_neon_sqrshrn:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, Op.getValueType(),
                         DAG.getNode(AArch64ISD::SRSHR_I, DL,
                                     Op.getOperand(1).getValueType(),
                                     Op.getOperand(1), Op.getOperand(2)));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQRSHRN, DAG,
                                 /*LastOperandIsImm=*/true);
  case Intrinsic::aarch64_neon_sqrshrun:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, Op.getValueType(),
                         DAG.getNode(AArch64ISD::SRSHR_I, DL,
                                     Op.getOperand(1).getValueType(),
                                     Op.getOperand(1), Op.getOperand(2)));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQRSHRUN, DAG,
                                 /*LastOperandIsImm=*/true);
  case Intrinsic::aarch64_neon_uqrshrn:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, Op.getValueType(),
                         DAG.getNode(AArch64ISD::URSHR_I, DL,
                                     Op.getOperand(1).getValueType(),
                                     Op.getOperand(1), Op.getOperand(2)));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::UQRSHRN, DAG,
                                 /*LastOperandIsImm=*/true);
  case Intrinsic::aarch64_neon_sqdmulh:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQDMULH, DAG);
  case Intrinsic::aarch64_neon_sqrdmulh:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQRDMULH, DAG);
  case Intrinsic::aarch64_neon_sqrdmlah:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQRDMLAH, DAG);
  case Intrinsic::aarch64_neon_sqrdmlsh:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQRDMLSH, DAG);
  case Intrinsic::aarch64_neon_sqrshl:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQRSHL, DAG);
  case Intrinsic::aarch64_neon_sqshl:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQSHL, DAG);
  case Intrinsic::aarch64_neon_uqrshl:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::UQRSHL, DAG);
  case Intrinsic::aarch64_neon_uqshl:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::UQSHL, DAG);
  case Intrinsic::aarch64_neon_sqadd:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::SADDSAT, DL, Op.getValueType(), Op.getOperand(1),
                         Op.getOperand(2));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQADD, DAG);
 
  case Intrinsic::aarch64_neon_sqsub:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::SSUBSAT, DL, Op.getValueType(), Op.getOperand(1),
                         Op.getOperand(2));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQSUB, DAG);
 
  case Intrinsic::aarch64_neon_uqadd:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::UADDSAT, DL, Op.getValueType(), Op.getOperand(1),
                         Op.getOperand(2));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::UQADD, DAG);
  case Intrinsic::aarch64_neon_suqadd:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SUQADD, DAG);
  case Intrinsic::aarch64_neon_usqadd:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::USQADD, DAG);
  case Intrinsic::aarch64_neon_uqsub:
    if (Op.getValueType().isVector())
      return DAG.getNode(ISD::USUBSAT, DL, Op.getValueType(), Op.getOperand(1),
                         Op.getOperand(2));
    return lowerIntNeonIntrinsic(Op, AArch64ISD::UQSUB, DAG);
  case Intrinsic::aarch64_neon_sqdmulls_scalar:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQDMULL, DAG);
  case Intrinsic::aarch64_neon_sqabs:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQABS, DAG);
  case Intrinsic::aarch64_neon_sqneg:
    return lowerIntNeonIntrinsic(Op, AArch64ISD::SQNEG, DAG);
  case Intrinsic::aarch64_sve_whilelt:
    return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/true,
                                     /*IsEqual=*/false);
  case Intrinsic::aarch64_sve_whilels:
    return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/false,
                                     /*IsEqual=*/true);
  case Intrinsic::aarch64_sve_whilele:
    return optimizeIncrementingWhile(Op.getNode(), DAG, /*IsSigned=*/true,
                                     /*IsEqual=*/true);
  case Intrinsic::aarch64_sve_sunpkhi:
    return DAG.getNode(AArch64ISD::SUNPKHI, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_sunpklo:
    return DAG.getNode(AArch64ISD::SUNPKLO, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_uunpkhi:
    return DAG.getNode(AArch64ISD::UUNPKHI, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_uunpklo:
    return DAG.getNode(AArch64ISD::UUNPKLO, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_clasta_n:
    return DAG.getNode(AArch64ISD::CLASTA_N, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::aarch64_sve_clastb_n:
    return DAG.getNode(AArch64ISD::CLASTB_N, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::aarch64_sve_lasta:
    return DAG.getNode(AArch64ISD::LASTA, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_lastb:
    return DAG.getNode(AArch64ISD::LASTB, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_tbl:
    return DAG.getNode(AArch64ISD::TBL, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  case Intrinsic::aarch64_sve_trn1:
    return DAG.getNode(AArch64ISD::TRN1, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_trn2:
    return DAG.getNode(AArch64ISD::TRN2, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_uzp1:
    return DAG.getNode(AArch64ISD::UZP1, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_uzp2:
    return DAG.getNode(AArch64ISD::UZP2, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_zip1:
    return DAG.getNode(AArch64ISD::ZIP1, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_zip2:
    return DAG.getNode(AArch64ISD::ZIP2, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_splice:
    return DAG.getNode(AArch64ISD::SPLICE, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::aarch64_sve_ptrue:
    return getPTrue(DAG, DL, Op.getValueType(), Op.getConstantOperandVal(1));
  case Intrinsic::aarch64_sve_clz:
    return DAG.getNode(AArch64ISD::CTLZ_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sme_cntsd: {
    SDValue Bytes = DAG.getNode(AArch64ISD::RDSVL, DL, Op.getValueType(),
                                DAG.getConstant(1, DL, MVT::i32));
    return DAG.getNode(ISD::SRL, DL, Op.getValueType(), Bytes,
                       DAG.getConstant(3, DL, MVT::i32), SDNodeFlags::Exact);
  }
  case Intrinsic::aarch64_sve_cnt: {
    SDValue Data = Op.getOperand(3);
    // CTPOP only supports integer operands.
    if (Data.getValueType().isFloatingPoint())
      Data = DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Data);
    return DAG.getNode(AArch64ISD::CTPOP_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Data, Op.getOperand(1));
  }
  case Intrinsic::aarch64_sve_dupq_lane:
    return LowerDUPQLane(Op, DAG);
  case Intrinsic::aarch64_sve_convert_from_svbool:
    return getSVEPredicateBitCast(Op.getValueType(), Op.getOperand(1), DAG);
  case Intrinsic::aarch64_sve_convert_to_svbool:
    return getSVEPredicateBitCast(MVT::nxv16i1, Op.getOperand(1), DAG);
  case Intrinsic::aarch64_sve_convert_from_svcount:
  case Intrinsic::aarch64_sve_convert_to_svcount:
    return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Op.getOperand(1));
  case Intrinsic::aarch64_sve_fneg:
    return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frintp:
    return DAG.getNode(AArch64ISD::FCEIL_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frintm:
    return DAG.getNode(AArch64ISD::FFLOOR_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frinti:
    return DAG.getNode(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_frintx:
    return DAG.getNode(AArch64ISD::FRINT_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frint32x:
    return DAG.getNode(AArch64ISD::FRINT32_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_frint64x:
    return DAG.getNode(AArch64ISD::FRINT64_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_frinta:
    return DAG.getNode(AArch64ISD::FROUND_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frintn:
    return DAG.getNode(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_frintz:
    return DAG.getNode(AArch64ISD::FTRUNC_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frint32z:
    return DAG.getNode(AArch64ISD::FTRUNC32_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_frint64z:
    return DAG.getNode(AArch64ISD::FTRUNC64_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_ucvtf:
    return DAG.getNode(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_scvtf:
    return DAG.getNode(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_fcvtzu:
    return DAG.getNode(AArch64ISD::FCVTZU_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_fcvtzs:
    return DAG.getNode(AArch64ISD::FCVTZS_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_fsqrt:
    return DAG.getNode(AArch64ISD::FSQRT_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frecpx:
    return DAG.getNode(AArch64ISD::FRECPX_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_frecpe_x:
    return DAG.getNode(AArch64ISD::FRECPE, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_frecps_x:
    return DAG.getNode(AArch64ISD::FRECPS, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_frsqrte_x:
    return DAG.getNode(AArch64ISD::FRSQRTE, DL, Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_frsqrts_x:
    return DAG.getNode(AArch64ISD::FRSQRTS, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::aarch64_sve_fabs:
    return DAG.getNode(AArch64ISD::FABS_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_abs:
    return DAG.getNode(AArch64ISD::ABS_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_neg:
    return DAG.getNode(AArch64ISD::NEG_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_insr: {
    SDValue Scalar = Op.getOperand(2);
    EVT ScalarTy = Scalar.getValueType();
    if ((ScalarTy == MVT::i8) || (ScalarTy == MVT::i16))
      Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Scalar);
 
    return DAG.getNode(AArch64ISD::INSR, DL, Op.getValueType(),
                       Op.getOperand(1), Scalar);
  }
  case Intrinsic::aarch64_sve_rbit:
    return DAG.getNode(AArch64ISD::BITREVERSE_MERGE_PASSTHRU, DL,
                       Op.getValueType(), Op.getOperand(2), Op.getOperand(3),
                       Op.getOperand(1));
  case Intrinsic::aarch64_sve_revb:
    return DAG.getNode(AArch64ISD::BSWAP_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_revh:
    return DAG.getNode(AArch64ISD::REVH_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_revw:
    return DAG.getNode(AArch64ISD::REVW_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_revd:
    return DAG.getNode(AArch64ISD::REVD_MERGE_PASSTHRU, DL, Op.getValueType(),
                       Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
  case Intrinsic::aarch64_sve_sxtb:
    return DAG.getNode(
        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
        Op.getOperand(2), Op.getOperand(3),
        DAG.getValueType(Op.getValueType().changeVectorElementType(
            *DAG.getContext(), MVT::i8)),
        Op.getOperand(1));
  case Intrinsic::aarch64_sve_sxth:
    return DAG.getNode(
        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
        Op.getOperand(2), Op.getOperand(3),
        DAG.getValueType(Op.getValueType().changeVectorElementType(
            *DAG.getContext(), MVT::i16)),
        Op.getOperand(1));
  case Intrinsic::aarch64_sve_sxtw:
    return DAG.getNode(
        AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
        Op.getOperand(2), Op.getOperand(3),
        DAG.getValueType(Op.getValueType().changeVectorElementType(
            *DAG.getContext(), MVT::i32)),
        Op.getOperand(1));
  case Intrinsic::aarch64_sve_uxtb:
    return DAG.getNode(
        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
        Op.getOperand(2), Op.getOperand(3),
        DAG.getValueType(Op.getValueType().changeVectorElementType(
            *DAG.getContext(), MVT::i8)),
        Op.getOperand(1));
  case Intrinsic::aarch64_sve_uxth:
    return DAG.getNode(
        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
        Op.getOperand(2), Op.getOperand(3),
        DAG.getValueType(Op.getValueType().changeVectorElementType(
            *DAG.getContext(), MVT::i16)),
        Op.getOperand(1));
  case Intrinsic::aarch64_sve_uxtw:
    return DAG.getNode(
        AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU, DL, Op.getValueType(),
        Op.getOperand(2), Op.getOperand(3),
        DAG.getValueType(Op.getValueType().changeVectorElementType(
            *DAG.getContext(), MVT::i32)),
        Op.getOperand(1));
  case Intrinsic::localaddress: {
    const auto &MF = DAG.getMachineFunction();
    const auto *RegInfo = Subtarget->getRegisterInfo();
    unsigned Reg = RegInfo->getLocalAddressRegister(MF);
    return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg,
                              Op.getSimpleValueType());
  }
 
  case Intrinsic::eh_recoverfp: {
    // FIXME: This needs to be implemented to correctly handle highly aligned
    // stack objects. For now we simply return the incoming FP. Refer D53541
    // for more details.
    SDValue FnOp = Op.getOperand(1);
    SDValue IncomingFPOp = Op.getOperand(2);
    GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
    auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
    if (!Fn)
      report_fatal_error(
          "llvm.eh.recoverfp must take a function as the first argument");
    return IncomingFPOp;
  }
  case Intrinsic::aarch64_neon_vsri:
  case Intrinsic::aarch64_neon_vsli:
  case Intrinsic::aarch64_sve_sri:
  case Intrinsic::aarch64_sve_sli: {
    EVT Ty = Op.getValueType();
 
    if (!Ty.isVector())
      report_fatal_error("Unexpected type for aarch64_neon_vsli");
 
    assert(Op.getConstantOperandVal(3) <= Ty.getScalarSizeInBits());
 
    bool IsShiftRight = IntNo == Intrinsic::aarch64_neon_vsri ||
                        IntNo == Intrinsic::aarch64_sve_sri;
    unsigned Opcode = IsShiftRight ? AArch64ISD::VSRI : AArch64ISD::VSLI;
    return DAG.getNode(Opcode, DL, Ty, Op.getOperand(1), Op.getOperand(2),
                       Op.getOperand(3));
  }
 
  case Intrinsic::aarch64_neon_srhadd:
  case Intrinsic::aarch64_neon_urhadd:
  case Intrinsic::aarch64_neon_shadd:
  case Intrinsic::aarch64_neon_uhadd: {
    bool IsSignedAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
                        IntNo == Intrinsic::aarch64_neon_shadd);
    bool IsRoundingAdd = (IntNo == Intrinsic::aarch64_neon_srhadd ||
                          IntNo == Intrinsic::aarch64_neon_urhadd);
    unsigned Opcode = IsSignedAdd
                          ? (IsRoundingAdd ? ISD::AVGCEILS : ISD::AVGFLOORS)
                          : (IsRoundingAdd ? ISD::AVGCEILU : ISD::AVGFLOORU);
    return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  }
  case Intrinsic::aarch64_neon_saddlp:
  case Intrinsic::aarch64_neon_uaddlp: {
    unsigned Opcode = IntNo == Intrinsic::aarch64_neon_uaddlp
                          ? AArch64ISD::UADDLP
                          : AArch64ISD::SADDLP;
    return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(1));
  }
  case Intrinsic::aarch64_neon_sdot:
  case Intrinsic::aarch64_neon_udot:
  case Intrinsic::aarch64_sve_sdot:
  case Intrinsic::aarch64_sve_udot: {
    unsigned Opcode = (IntNo == Intrinsic::aarch64_neon_udot ||
                       IntNo == Intrinsic::aarch64_sve_udot)
                          ? AArch64ISD::UDOT
                          : AArch64ISD::SDOT;
    return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2), Op.getOperand(3));
  }
  case Intrinsic::aarch64_neon_usdot:
  case Intrinsic::aarch64_sve_usdot: {
    return DAG.getNode(AArch64ISD::USDOT, DL, Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  }
  case Intrinsic::aarch64_neon_saddlv:
  case Intrinsic::aarch64_neon_uaddlv: {
    EVT OpVT = Op.getOperand(1).getValueType();
    EVT ResVT = Op.getValueType();
    assert(
        ((ResVT == MVT::i32 && (OpVT == MVT::v8i8 || OpVT == MVT::v16i8 ||
                                OpVT == MVT::v8i16 || OpVT == MVT::v4i16)) ||
         (ResVT == MVT::i64 && (OpVT == MVT::v4i32 || OpVT == MVT::v2i32))) &&
        "Unexpected aarch64_neon_u/saddlv type");
    (void)OpVT;
    // In order to avoid insert_subvector, use v4i32 rather than v2i32.
    SDValue ADDLV = DAG.getNode(
        IntNo == Intrinsic::aarch64_neon_uaddlv ? AArch64ISD::UADDLV
                                                : AArch64ISD::SADDLV,
        DL, ResVT == MVT::i32 ? MVT::v4i32 : MVT::v2i64, Op.getOperand(1));
    SDValue EXTRACT_VEC_ELT = DAG.getNode(
        ISD::EXTRACT_VECTOR_ELT, DL, ResVT == MVT::i32 ? MVT::i32 : MVT::i64,
        ADDLV, DAG.getConstant(0, DL, MVT::i64));
    return EXTRACT_VEC_ELT;
  }
  case Intrinsic::aarch64_cls:
  case Intrinsic::aarch64_cls64: {
    SDValue Res = DAG.getNode(ISD::CTLS, DL, Op.getOperand(1).getValueType(),
                              Op.getOperand(1));
    return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
  }
  case Intrinsic::aarch64_neon_cls: {
    // Lower NEON CLS intrinsic to ISD::CTLS
    return DAG.getNode(ISD::CTLS, DL, Op.getValueType(), Op.getOperand(1));
  }
  case Intrinsic::aarch64_sve_pmul:
  case Intrinsic::aarch64_neon_pmul:
    return DAG.getNode(ISD::CLMUL, DL, Op.getValueType(), Op.getOperand(1),
                       Op.getOperand(2));
  case Intrinsic::aarch64_sve_psel: {
    if (Op.getValueType() != MVT::aarch64svcount)
      return Op;
    // Lower predicate-as-counter variants to reuse normal predicate isel.
    SmallVector<SDValue> Ops(Op->ops());
    Ops[1] = DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i1, Ops[1]);
    SDValue PSel = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::nxv16i1, Ops);
    return DAG.getNode(ISD::BITCAST, DL, MVT::aarch64svcount, PSel);
  }
  }
}
 
bool AArch64TargetLowering::shouldExtendGSIndex(EVT VT, EVT &EltTy) const {
  if (VT.getVectorElementType() == MVT::i8 ||
      VT.getVectorElementType() == MVT::i16) {
    EltTy = MVT::i32;
    return true;
  }
  return false;
}
 
bool AArch64TargetLowering::shouldRemoveExtendFromGSIndex(SDValue Extend,
                                                          EVT DataVT) const {
  const EVT IndexVT = Extend.getOperand(0).getValueType();
  // SVE only supports implicit extension of 32-bit indices.
  if (!Subtarget->hasSVE() || IndexVT.getVectorElementType() != MVT::i32)
    return false;
 
  // Indices cannot be smaller than the main data type.
  if (IndexVT.getScalarSizeInBits() < DataVT.getScalarSizeInBits())
    return false;
 
  // Scalable vectors with "vscale * 2" or fewer elements sit within a 64-bit
  // element container type, which would violate the previous clause.
  return DataVT.isFixedLengthVector() || DataVT.getVectorMinNumElements() > 2;
}
 
/// Helper function to check if a small vector load can be optimized.
static bool isEligibleForSmallVectorLoadOpt(LoadSDNode *LD,
                                            const AArch64Subtarget &Subtarget) {
  if (!Subtarget.isNeonAvailable())
    return false;
  if (LD->isVolatile())
    return false;
 
  EVT MemVT = LD->getMemoryVT();
  if (MemVT != MVT::v2i8 && MemVT != MVT::v4i8 && MemVT != MVT::v2i16)
    return false;
 
  Align Alignment = LD->getAlign();
  Align RequiredAlignment = Align(MemVT.getStoreSize().getFixedValue());
  if (Subtarget.requiresStrictAlign() && Alignment < RequiredAlignment)
    return false;
 
  return true;
}
 
bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
  EVT ExtVT = ExtVal.getValueType();
  // Small, illegal vectors can be extended inreg.
  if (auto *Load = dyn_cast<LoadSDNode>(ExtVal.getOperand(0))) {
    if (ExtVT.isFixedLengthVector() && ExtVT.getStoreSizeInBits() <= 128 &&
        isEligibleForSmallVectorLoadOpt(Load, *Subtarget))
      return true;
  }
  if (!ExtVT.isScalableVector() && !Subtarget->useSVEForFixedLengthVectors())
    return false;
 
  // It may be worth creating extending masked loads if there are multiple