ARMISelLowering.cpp

Go to the documentation of this file.

//===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the interfaces that ARM uses to lower LLVM code into a
// selection DAG.
//
//===----------------------------------------------------------------------===//
 
#include "ARMISelLowering.h"
#include "ARMBaseInstrInfo.h"
#include "ARMBaseRegisterInfo.h"
#include "ARMCallingConv.h"
#include "ARMConstantPoolValue.h"
#include "ARMMachineFunctionInfo.h"
#include "ARMPerfectShuffle.h"
#include "ARMRegisterInfo.h"
#include "ARMSelectionDAGInfo.h"
#include "ARMSubtarget.h"
#include "ARMTargetTransformInfo.h"
#include "MCTargetDesc/ARMAddressingModes.h"
#include "MCTargetDesc/ARMBaseInfo.h"
#include "Utils/ARMBaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcallUtil.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/MC/MCInstrItineraries.h"
#include "llvm/MC/MCSchedule.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>
 
using namespace llvm;
 
#define DEBUG_TYPE "arm-isel"
 
STATISTIC(NumTailCalls, "Number of tail calls");
STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
STATISTIC(NumConstpoolPromoted,
  "Number of constants with their storage promoted into constant pools");
 
static cl::opt<bool>
ARMInterworking("arm-interworking", cl::Hidden,
  cl::desc("Enable / disable ARM interworking (for debugging only)"),
  cl::init(true));
 
static cl::opt<bool> EnableConstpoolPromotion(
    "arm-promote-constant", cl::Hidden,
    cl::desc("Enable / disable promotion of unnamed_addr constants into "
             "constant pools"),
    cl::init(false)); // FIXME: set to true by default once PR32780 is fixed
static cl::opt<unsigned> ConstpoolPromotionMaxSize(
    "arm-promote-constant-max-size", cl::Hidden,
    cl::desc("Maximum size of constant to promote into a constant pool"),
    cl::init(64));
static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
    "arm-promote-constant-max-total", cl::Hidden,
    cl::desc("Maximum size of ALL constants to promote into a constant pool"),
    cl::init(128));
 
cl::opt<unsigned>
MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
  cl::desc("Maximum interleave factor for MVE VLDn to generate."),
  cl::init(2));
 
cl::opt<unsigned> ArmMaxBaseUpdatesToCheck(
    "arm-max-base-updates-to-check", cl::Hidden,
    cl::desc("Maximum number of base-updates to check generating postindex."),
    cl::init(64));
 
/// Value type used for "flags" operands / results (either CPSR or FPSCR_NZCV).
constexpr MVT FlagsVT = MVT::i32;
 
// The APCS parameter registers.
static const MCPhysReg GPRArgRegs[] = {
  ARM::R0, ARM::R1, ARM::R2, ARM::R3
};
 
static SDValue handleCMSEValue(const SDValue &Value, const ISD::InputArg &Arg,
                               SelectionDAG &DAG, const SDLoc &DL) {
  assert(Arg.ArgVT.isScalarInteger());
  assert(Arg.ArgVT.bitsLT(MVT::i32));
  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, Arg.ArgVT, Value);
  SDValue Ext =
      DAG.getNode(Arg.Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
                  MVT::i32, Trunc);
  return Ext;
}
 
void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT) {
  if (VT != PromotedLdStVT) {
    setOperationAction(ISD::LOAD, VT, Promote);
    AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT);
 
    setOperationAction(ISD::STORE, VT, Promote);
    AddPromotedToType (ISD::STORE, VT, PromotedLdStVT);
  }
 
  MVT ElemTy = VT.getVectorElementType();
  if (ElemTy != MVT::f64)
    setOperationAction(ISD::SETCC, VT, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
  if (ElemTy == MVT::i32) {
    setOperationAction(ISD::SINT_TO_FP, VT, Custom);
    setOperationAction(ISD::UINT_TO_FP, VT, Custom);
    setOperationAction(ISD::FP_TO_SINT, VT, Custom);
    setOperationAction(ISD::FP_TO_UINT, VT, Custom);
  } else {
    setOperationAction(ISD::SINT_TO_FP, VT, Expand);
    setOperationAction(ISD::UINT_TO_FP, VT, Expand);
    setOperationAction(ISD::FP_TO_SINT, VT, Expand);
    setOperationAction(ISD::FP_TO_UINT, VT, Expand);
  }
  setOperationAction(ISD::BUILD_VECTOR,      VT, Custom);
  setOperationAction(ISD::VECTOR_SHUFFLE,    VT, Custom);
  setOperationAction(ISD::CONCAT_VECTORS,    VT, Legal);
  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
  setOperationAction(ISD::SELECT,            VT, Expand);
  setOperationAction(ISD::SELECT_CC,         VT, Expand);
  setOperationAction(ISD::VSELECT,           VT, Expand);
  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
  if (VT.isInteger()) {
    setOperationAction(ISD::SHL, VT, Custom);
    setOperationAction(ISD::SRA, VT, Custom);
    setOperationAction(ISD::SRL, VT, Custom);
  }
 
  // Neon does not support vector divide/remainder operations.
  setOperationAction(ISD::SDIV, VT, Expand);
  setOperationAction(ISD::UDIV, VT, Expand);
  setOperationAction(ISD::FDIV, VT, Expand);
  setOperationAction(ISD::SREM, VT, Expand);
  setOperationAction(ISD::UREM, VT, Expand);
  setOperationAction(ISD::FREM, VT, Expand);
  setOperationAction(ISD::SDIVREM, VT, Expand);
  setOperationAction(ISD::UDIVREM, VT, Expand);
 
  if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
    for (auto Opcode : {ISD::ABS, ISD::ABDS, ISD::ABDU, ISD::SMIN, ISD::SMAX,
                        ISD::UMIN, ISD::UMAX, ISD::CTLS})
      setOperationAction(Opcode, VT, Legal);
  if (!VT.isFloatingPoint())
    for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
      setOperationAction(Opcode, VT, Legal);
}
 
void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
  addRegisterClass(VT, &ARM::DPRRegClass);
  addTypeForNEON(VT, MVT::f64);
}
 
void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
  addRegisterClass(VT, &ARM::DPairRegClass);
  addTypeForNEON(VT, MVT::v2f64);
}
 
void ARMTargetLowering::setAllExpand(MVT VT) {
  for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
    setOperationAction(Opc, VT, Expand);
 
  // We support these really simple operations even on types where all
  // the actual arithmetic has to be broken down into simpler
  // operations or turned into library calls.
  setOperationAction(ISD::BITCAST, VT, Legal);
  setOperationAction(ISD::LOAD, VT, Legal);
  setOperationAction(ISD::STORE, VT, Legal);
  setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Legal);
}
 
void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
                                       LegalizeAction Action) {
  setLoadExtAction(ISD::EXTLOAD,  From, To, Action);
  setLoadExtAction(ISD::ZEXTLOAD, From, To, Action);
  setLoadExtAction(ISD::SEXTLOAD, From, To, Action);
}
 
void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
  const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
 
  for (auto VT : IntTypes) {
    addRegisterClass(VT, &ARM::MQPRRegClass);
    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
    setOperationAction(ISD::SHL, VT, Custom);
    setOperationAction(ISD::SRA, VT, Custom);
    setOperationAction(ISD::SRL, VT, Custom);
    setOperationAction(ISD::SMIN, VT, Legal);
    setOperationAction(ISD::SMAX, VT, Legal);
    setOperationAction(ISD::UMIN, VT, Legal);
    setOperationAction(ISD::UMAX, VT, Legal);
    setOperationAction(ISD::ABS, VT, Legal);
    setOperationAction(ISD::CTLS, VT, Legal);
    setOperationAction(ISD::SETCC, VT, Custom);
    setOperationAction(ISD::MLOAD, VT, Custom);
    setOperationAction(ISD::MSTORE, VT, Legal);
    setOperationAction(ISD::CTLZ, VT, Legal);
    setOperationAction(ISD::CTTZ, VT, Custom);
    setOperationAction(ISD::BITREVERSE, VT, Legal);
    setOperationAction(ISD::BSWAP, VT, Legal);
    setOperationAction(ISD::SADDSAT, VT, Legal);
    setOperationAction(ISD::UADDSAT, VT, Legal);
    setOperationAction(ISD::SSUBSAT, VT, Legal);
    setOperationAction(ISD::USUBSAT, VT, Legal);
    setOperationAction(ISD::ABDS, VT, Legal);
    setOperationAction(ISD::ABDU, VT, Legal);
    setOperationAction(ISD::AVGFLOORS, VT, Legal);
    setOperationAction(ISD::AVGFLOORU, VT, Legal);
    setOperationAction(ISD::AVGCEILS, VT, Legal);
    setOperationAction(ISD::AVGCEILU, VT, Legal);
 
    // No native support for these.
    setOperationAction(ISD::UDIV, VT, Expand);
    setOperationAction(ISD::SDIV, VT, Expand);
    setOperationAction(ISD::UREM, VT, Expand);
    setOperationAction(ISD::SREM, VT, Expand);
    setOperationAction(ISD::UDIVREM, VT, Expand);
    setOperationAction(ISD::SDIVREM, VT, Expand);
    setOperationAction(ISD::CTPOP, VT, Expand);
    setOperationAction(ISD::SELECT, VT, Expand);
    setOperationAction(ISD::SELECT_CC, VT, Expand);
 
    // Vector reductions
    setOperationAction(ISD::VECREDUCE_ADD, VT, Legal);
    setOperationAction(ISD::VECREDUCE_SMAX, VT, Legal);
    setOperationAction(ISD::VECREDUCE_UMAX, VT, Legal);
    setOperationAction(ISD::VECREDUCE_SMIN, VT, Legal);
    setOperationAction(ISD::VECREDUCE_UMIN, VT, Legal);
    setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
    setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
    setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
    setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
 
    if (!HasMVEFP) {
      setOperationAction(ISD::SINT_TO_FP, VT, Expand);
      setOperationAction(ISD::UINT_TO_FP, VT, Expand);
      setOperationAction(ISD::FP_TO_SINT, VT, Expand);
      setOperationAction(ISD::FP_TO_UINT, VT, Expand);
    } else {
      setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
      setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
    }
 
    // Pre and Post inc are supported on loads and stores
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im, VT, Legal);
      setIndexedStoreAction(im, VT, Legal);
      setIndexedMaskedLoadAction(im, VT, Legal);
      setIndexedMaskedStoreAction(im, VT, Legal);
    }
  }
 
  const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
  for (auto VT : FloatTypes) {
    addRegisterClass(VT, &ARM::MQPRRegClass);
    if (!HasMVEFP)
      setAllExpand(VT);
 
    // These are legal or custom whether we have MVE.fp or not
    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getVectorElementType(), Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT.getVectorElementType(), Custom);
    setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal);
    setOperationAction(ISD::SETCC, VT, Custom);
    setOperationAction(ISD::MLOAD, VT, Custom);
    setOperationAction(ISD::MSTORE, VT, Legal);
    setOperationAction(ISD::SELECT, VT, Expand);
    setOperationAction(ISD::SELECT_CC, VT, Expand);
 
    // Pre and Post inc are supported on loads and stores
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im, VT, Legal);
      setIndexedStoreAction(im, VT, Legal);
      setIndexedMaskedLoadAction(im, VT, Legal);
      setIndexedMaskedStoreAction(im, VT, Legal);
    }
 
    if (HasMVEFP) {
      setOperationAction(ISD::FMINNUM, VT, Legal);
      setOperationAction(ISD::FMAXNUM, VT, Legal);
      for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
                      ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
                      ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
                      ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
        setOperationAction(Op, VT, Legal);
      }
      setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
 
      // No native support for these.
      setOperationAction(ISD::FDIV, VT, Expand);
      setOperationAction(ISD::FREM, VT, Expand);
      setOperationAction(ISD::FSQRT, VT, Expand);
      setOperationAction(ISD::FSIN, VT, Expand);
      setOperationAction(ISD::FCOS, VT, Expand);
      setOperationAction(ISD::FTAN, VT, Expand);
      setOperationAction(ISD::FPOW, VT, Expand);
      setOperationAction(ISD::FLOG, VT, Expand);
      setOperationAction(ISD::FLOG2, VT, Expand);
      setOperationAction(ISD::FLOG10, VT, Expand);
      setOperationAction(ISD::FEXP, VT, Expand);
      setOperationAction(ISD::FEXP2, VT, Expand);
      setOperationAction(ISD::FEXP10, VT, Expand);
      setOperationAction(ISD::FNEARBYINT, VT, Expand);
    }
  }
 
  // Custom Expand smaller than legal vector reductions to prevent false zero
  // items being added.
  setOperationAction(ISD::VECREDUCE_FADD, MVT::v4f16, Custom);
  setOperationAction(ISD::VECREDUCE_FMUL, MVT::v4f16, Custom);
  setOperationAction(ISD::VECREDUCE_FMIN, MVT::v4f16, Custom);
  setOperationAction(ISD::VECREDUCE_FMAX, MVT::v4f16, Custom);
  setOperationAction(ISD::VECREDUCE_FADD, MVT::v2f16, Custom);
  setOperationAction(ISD::VECREDUCE_FMUL, MVT::v2f16, Custom);
  setOperationAction(ISD::VECREDUCE_FMIN, MVT::v2f16, Custom);
  setOperationAction(ISD::VECREDUCE_FMAX, MVT::v2f16, Custom);
 
  // We 'support' these types up to bitcast/load/store level, regardless of
  // MVE integer-only / float support. Only doing FP data processing on the FP
  // vector types is inhibited at integer-only level.
  const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
  for (auto VT : LongTypes) {
    addRegisterClass(VT, &ARM::MQPRRegClass);
    setAllExpand(VT);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
    setOperationAction(ISD::VSELECT, VT, Legal);
    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
  }
  setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
 
  // We can do bitwise operations on v2i64 vectors
  setOperationAction(ISD::AND, MVT::v2i64, Legal);
  setOperationAction(ISD::OR, MVT::v2i64, Legal);
  setOperationAction(ISD::XOR, MVT::v2i64, Legal);
 
  // It is legal to extload from v4i8 to v4i16 or v4i32.
  addAllExtLoads(MVT::v8i16, MVT::v8i8, Legal);
  addAllExtLoads(MVT::v4i32, MVT::v4i16, Legal);
  addAllExtLoads(MVT::v4i32, MVT::v4i8, Legal);
 
  // It is legal to sign extend from v4i8/v4i16 to v4i32 or v8i8 to v8i16.
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8,  Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i8,  Legal);
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i16, Legal);
 
  // Some truncating stores are legal too.
  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
  setTruncStoreAction(MVT::v4i32, MVT::v4i8,  Legal);
  setTruncStoreAction(MVT::v8i16, MVT::v8i8,  Legal);
 
  // Pre and Post inc on these are legal, given the correct extends
  for (unsigned im = (unsigned)ISD::PRE_INC;
       im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
    for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
      setIndexedLoadAction(im, VT, Legal);
      setIndexedStoreAction(im, VT, Legal);
      setIndexedMaskedLoadAction(im, VT, Legal);
      setIndexedMaskedStoreAction(im, VT, Legal);
    }
  }
 
  // Predicate types
  const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1};
  for (auto VT : pTypes) {
    addRegisterClass(VT, &ARM::VCCRRegClass);
    setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
    setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
    setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
    setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
    setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    setOperationAction(ISD::SETCC, VT, Custom);
    setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
    setOperationAction(ISD::LOAD, VT, Custom);
    setOperationAction(ISD::STORE, VT, Custom);
    setOperationAction(ISD::TRUNCATE, VT, Custom);
    setOperationAction(ISD::VSELECT, VT, Expand);
    setOperationAction(ISD::SELECT, VT, Expand);
    setOperationAction(ISD::SELECT_CC, VT, Expand);
 
    if (!HasMVEFP) {
      setOperationAction(ISD::SINT_TO_FP, VT, Expand);
      setOperationAction(ISD::UINT_TO_FP, VT, Expand);
      setOperationAction(ISD::FP_TO_SINT, VT, Expand);
      setOperationAction(ISD::FP_TO_UINT, VT, Expand);
    }
  }
  setOperationAction(ISD::SETCC, MVT::v2i1, Expand);
  setOperationAction(ISD::TRUNCATE, MVT::v2i1, Expand);
  setOperationAction(ISD::AND, MVT::v2i1, Expand);
  setOperationAction(ISD::OR, MVT::v2i1, Expand);
  setOperationAction(ISD::XOR, MVT::v2i1, Expand);
  setOperationAction(ISD::SINT_TO_FP, MVT::v2i1, Expand);
  setOperationAction(ISD::UINT_TO_FP, MVT::v2i1, Expand);
  setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Expand);
  setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Expand);
 
  setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom);
  setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom);
  setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
  setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom);
  setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom);
  setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
  setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
  setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
}
 
const ARMBaseTargetMachine &ARMTargetLowering::getTM() const {
  return static_cast<const ARMBaseTargetMachine &>(getTargetMachine());
}
 
ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
                                     const ARMSubtarget &STI)
    : TargetLowering(TM_, STI), Subtarget(&STI),
      RegInfo(Subtarget->getRegisterInfo()),
      Itins(Subtarget->getInstrItineraryData()) {
  const auto &TM = static_cast<const ARMBaseTargetMachine &>(TM_);
 
  setBooleanContents(ZeroOrOneBooleanContent);
  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
  const Triple &TT = TM.getTargetTriple();
 
  if (Subtarget->isThumb1Only())
    addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
  else
    addRegisterClass(MVT::i32, &ARM::GPRRegClass);
 
  if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
      Subtarget->hasFPRegs()) {
    addRegisterClass(MVT::f32, &ARM::SPRRegClass);
    addRegisterClass(MVT::f64, &ARM::DPRRegClass);
 
    if (!Subtarget->hasVFP2Base()) {
      setAllExpand(MVT::f32);
    } else {
      setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom);
      setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom);
 
      for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                      ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
        setOperationAction(Op, MVT::f32, Legal);
    }
    if (!Subtarget->hasFP64()) {
      setAllExpand(MVT::f64);
    } else {
      for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                      ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
        setOperationAction(Op, MVT::f64, Legal);
 
      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
    }
  }
 
  if (Subtarget->hasFullFP16()) {
    for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                    ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT})
      setOperationAction(Op, MVT::f16, Legal);
 
    addRegisterClass(MVT::f16, &ARM::HPRRegClass);
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
    setOperationAction(ISD::BITCAST, MVT::f16, Custom);
 
    setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
    setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
    setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Legal);
    setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Legal);
  }
 
  if (Subtarget->hasBF16()) {
    addRegisterClass(MVT::bf16, &ARM::HPRRegClass);
    setAllExpand(MVT::bf16);
    if (!Subtarget->hasFullFP16())
      setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
    setOperationAction(ISD::FP_ROUND, MVT::bf16, Custom);
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::bf16, Custom);
  } else {
    setOperationAction(ISD::BF16_TO_FP, MVT::f32, Expand);
    setOperationAction(ISD::BF16_TO_FP, MVT::f64, Expand);
    setOperationAction(ISD::FP_TO_BF16, MVT::f32, Custom);
    setOperationAction(ISD::FP_TO_BF16, MVT::f64, Custom);
  }
 
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
      setTruncStoreAction(VT, InnerVT, Expand);
      addAllExtLoads(VT, InnerVT, Expand);
    }
 
    setOperationAction(ISD::SMUL_LOHI, VT, Expand);
    setOperationAction(ISD::UMUL_LOHI, VT, Expand);
 
    setOperationAction(ISD::BSWAP, VT, Expand);
  }
 
  if (!Subtarget->isThumb1Only() && !Subtarget->hasV8_1MMainlineOps())
    setOperationAction(ISD::SCMP, MVT::i32, Custom);
 
  if (!Subtarget->hasV8_1MMainlineOps())
    setOperationAction(ISD::UCMP, MVT::i32, Custom);
 
  if (!Subtarget->isThumb1Only())
    setOperationAction(ISD::ABS, MVT::i32, Custom);
 
  setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
  setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
 
  setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
  setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);
 
  if (Subtarget->hasMVEIntegerOps())
    addMVEVectorTypes(Subtarget->hasMVEFloatOps());
 
  // Combine low-overhead loop intrinsics so that we can lower i1 types.
  if (Subtarget->hasLOB()) {
    setTargetDAGCombine({ISD::BRCOND, ISD::BR_CC});
  }
 
  if (Subtarget->hasNEON()) {
    addDRTypeForNEON(MVT::v2f32);
    addDRTypeForNEON(MVT::v8i8);
    addDRTypeForNEON(MVT::v4i16);
    addDRTypeForNEON(MVT::v2i32);
    addDRTypeForNEON(MVT::v1i64);
 
    addQRTypeForNEON(MVT::v4f32);
    addQRTypeForNEON(MVT::v2f64);
    addQRTypeForNEON(MVT::v16i8);
    addQRTypeForNEON(MVT::v8i16);
    addQRTypeForNEON(MVT::v4i32);
    addQRTypeForNEON(MVT::v2i64);
 
    if (Subtarget->hasFullFP16()) {
      addQRTypeForNEON(MVT::v8f16);
      addDRTypeForNEON(MVT::v4f16);
    }
 
    if (Subtarget->hasBF16()) {
      addQRTypeForNEON(MVT::v8bf16);
      addDRTypeForNEON(MVT::v4bf16);
    }
  }
 
  if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
    // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
    // none of Neon, MVE or VFP supports any arithmetic operations on it.
    setOperationAction(ISD::FADD, MVT::v2f64, Expand);
    setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
    setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
    // FIXME: Code duplication: FDIV and FREM are expanded always, see
    // ARMTargetLowering::addTypeForNEON method for details.
    setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
    setOperationAction(ISD::FREM, MVT::v2f64, Expand);
    // FIXME: Create unittest.
    // In another words, find a way when "copysign" appears in DAG with vector
    // operands.
    setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand);
    // FIXME: Code duplication: SETCC has custom operation action, see
    // ARMTargetLowering::addTypeForNEON method for details.
    setOperationAction(ISD::SETCC, MVT::v2f64, Expand);
    // FIXME: Create unittest for FNEG and for FABS.
    setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
    setOperationAction(ISD::FABS, MVT::v2f64, Expand);
    setOperationAction(ISD::FSQRT, MVT::v2f64, Expand);
    setOperationAction(ISD::FSIN, MVT::v2f64, Expand);
    setOperationAction(ISD::FCOS, MVT::v2f64, Expand);
    setOperationAction(ISD::FTAN, MVT::v2f64, Expand);
    setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
    setOperationAction(ISD::FLOG, MVT::v2f64, Expand);
    setOperationAction(ISD::FLOG2, MVT::v2f64, Expand);
    setOperationAction(ISD::FLOG10, MVT::v2f64, Expand);
    setOperationAction(ISD::FEXP, MVT::v2f64, Expand);
    setOperationAction(ISD::FEXP2, MVT::v2f64, Expand);
    setOperationAction(ISD::FEXP10, MVT::v2f64, Expand);
    setOperationAction(ISD::FCEIL, MVT::v2f64, Expand);
    setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand);
    setOperationAction(ISD::FRINT, MVT::v2f64, Expand);
    setOperationAction(ISD::FROUNDEVEN, MVT::v2f64, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand);
    setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand);
    setOperationAction(ISD::FMA, MVT::v2f64, Expand);
  }
 
  if (Subtarget->hasNEON()) {
    // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
    // supported for v4f32.
    setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
    setOperationAction(ISD::FSIN, MVT::v4f32, Expand);
    setOperationAction(ISD::FCOS, MVT::v4f32, Expand);
    setOperationAction(ISD::FTAN, MVT::v4f32, Expand);
    setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
    setOperationAction(ISD::FLOG, MVT::v4f32, Expand);
    setOperationAction(ISD::FLOG2, MVT::v4f32, Expand);
    setOperationAction(ISD::FLOG10, MVT::v4f32, Expand);
    setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
    setOperationAction(ISD::FEXP2, MVT::v4f32, Expand);
    setOperationAction(ISD::FEXP10, MVT::v4f32, Expand);
    setOperationAction(ISD::FCEIL, MVT::v4f32, Expand);
    setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand);
    setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
    setOperationAction(ISD::FROUNDEVEN, MVT::v4f32, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
    setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand);
 
    // Mark v2f32 intrinsics.
    setOperationAction(ISD::FSQRT, MVT::v2f32, Expand);
    setOperationAction(ISD::FSIN, MVT::v2f32, Expand);
    setOperationAction(ISD::FCOS, MVT::v2f32, Expand);
    setOperationAction(ISD::FTAN, MVT::v2f32, Expand);
    setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
    setOperationAction(ISD::FLOG, MVT::v2f32, Expand);
    setOperationAction(ISD::FLOG2, MVT::v2f32, Expand);
    setOperationAction(ISD::FLOG10, MVT::v2f32, Expand);
    setOperationAction(ISD::FEXP, MVT::v2f32, Expand);
    setOperationAction(ISD::FEXP2, MVT::v2f32, Expand);
    setOperationAction(ISD::FEXP10, MVT::v2f32, Expand);
    setOperationAction(ISD::FCEIL, MVT::v2f32, Expand);
    setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand);
    setOperationAction(ISD::FRINT, MVT::v2f32, Expand);
    setOperationAction(ISD::FROUNDEVEN, MVT::v2f32, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand);
    setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand);
 
    for (ISD::NodeType Op : {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL,
                              ISD::FRINT, ISD::FTRUNC, ISD::FROUNDEVEN}) {
      setOperationAction(Op, MVT::v4f16, Expand);
      setOperationAction(Op, MVT::v8f16, Expand);
    }
 
    // Neon does not support some operations on v1i64 and v2i64 types.
    setOperationAction(ISD::MUL, MVT::v1i64, Expand);
    // Custom handling for some quad-vector types to detect VMULL.
    setOperationAction(ISD::MUL, MVT::v8i16, Custom);
    setOperationAction(ISD::MUL, MVT::v4i32, Custom);
    setOperationAction(ISD::MUL, MVT::v2i64, Custom);
    // Custom handling for some vector types to avoid expensive expansions
    setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
    setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
    setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
    setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
    // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
    // a destination type that is wider than the source, and nor does
    // it have a FP_TO_[SU]INT instruction with a narrower destination than
    // source.
    setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
    setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
 
    setOperationAction(ISD::FP_ROUND,   MVT::v2f32, Expand);
    setOperationAction(ISD::FP_EXTEND,  MVT::v2f64, Expand);
 
    // NEON does not have single instruction CTPOP for vectors with element
    // types wider than 8-bits.  However, custom lowering can leverage the
    // v8i8/v16i8 vcnt instruction.
    setOperationAction(ISD::CTPOP,      MVT::v2i32, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v4i32, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v4i16, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v8i16, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v1i64, Custom);
    setOperationAction(ISD::CTPOP,      MVT::v2i64, Custom);
 
    setOperationAction(ISD::CTLZ,       MVT::v1i64, Expand);
    setOperationAction(ISD::CTLZ,       MVT::v2i64, Expand);
 
    // NEON does not have single instruction CTTZ for vectors.
    setOperationAction(ISD::CTTZ, MVT::v8i8, Custom);
    setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
    setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
    setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
 
    setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
    setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
    setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
    setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
 
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v8i8, Custom);
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v4i16, Custom);
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v2i32, Custom);
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v1i64, Custom);
 
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v16i8, Custom);
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v8i16, Custom);
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v4i32, Custom);
    setOperationAction(ISD::CTTZ_ZERO_POISON, MVT::v2i64, Custom);
 
    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
      setOperationAction(ISD::MULHS, VT, Expand);
      setOperationAction(ISD::MULHU, VT, Expand);
    }
 
    // NEON only has FMA instructions as of VFP4.
    if (!Subtarget->hasVFP4Base()) {
      setOperationAction(ISD::FMA, MVT::v2f32, Expand);
      setOperationAction(ISD::FMA, MVT::v4f32, Expand);
    }
 
    setTargetDAGCombine({ISD::SHL, ISD::SRL, ISD::SRA, ISD::FP_TO_SINT,
                         ISD::FP_TO_UINT, ISD::FMUL, ISD::LOAD});
 
    // It is legal to extload from v4i8 to v4i16 or v4i32.
    for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
                   MVT::v2i32}) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
        setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal);
        setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal);
        setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal);
      }
    }
 
    for (auto VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
                    MVT::v4i32}) {
      setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
    }
  }
 
  if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
    setTargetDAGCombine(
        {ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE, ISD::INSERT_SUBVECTOR,
         ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
         ISD::SIGN_EXTEND_INREG, ISD::STORE, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,
         ISD::ANY_EXTEND, ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
         ISD::INTRINSIC_VOID, ISD::VECREDUCE_ADD, ISD::ADD, ISD::BITCAST});
  }
  if (Subtarget->hasMVEIntegerOps()) {
    setTargetDAGCombine({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
                         ISD::FP_EXTEND, ISD::SELECT, ISD::SELECT_CC,
                         ISD::SETCC});
  }
  if (Subtarget->hasMVEFloatOps()) {
    setTargetDAGCombine(ISD::FADD);
  }
 
  if (!Subtarget->hasFP64()) {
    // When targeting a floating-point unit with only single-precision
    // operations, f64 is legal for the few double-precision instructions which
    // are present However, no double-precision operations other than moves,
    // loads and stores are provided by the hardware.
    setOperationAction(ISD::FADD,       MVT::f64, Expand);
    setOperationAction(ISD::FSUB,       MVT::f64, Expand);
    setOperationAction(ISD::FMUL,       MVT::f64, Expand);
    setOperationAction(ISD::FMA,        MVT::f64, Expand);
    setOperationAction(ISD::FDIV,       MVT::f64, Expand);
    setOperationAction(ISD::FREM, MVT::f64, LibCall);
    setOperationAction(ISD::FCOPYSIGN,  MVT::f64, Expand);
    setOperationAction(ISD::FGETSIGN,   MVT::f64, Expand);
    setOperationAction(ISD::FNEG,       MVT::f64, Expand);
    setOperationAction(ISD::FABS,       MVT::f64, Expand);
    setOperationAction(ISD::FSQRT,      MVT::f64, Expand);
    setOperationAction(ISD::FSIN,       MVT::f64, Expand);
    setOperationAction(ISD::FCOS,       MVT::f64, Expand);
    setOperationAction(ISD::FPOW,       MVT::f64, Expand);
    setOperationAction(ISD::FLOG,       MVT::f64, Expand);
    setOperationAction(ISD::FLOG2,      MVT::f64, Expand);
    setOperationAction(ISD::FLOG10,     MVT::f64, Expand);
    setOperationAction(ISD::FEXP,       MVT::f64, Expand);
    setOperationAction(ISD::FEXP2,      MVT::f64, Expand);
    setOperationAction(ISD::FEXP10,      MVT::f64, Expand);
    setOperationAction(ISD::FCEIL,      MVT::f64, Expand);
    setOperationAction(ISD::FTRUNC,     MVT::f64, Expand);
    setOperationAction(ISD::FRINT,      MVT::f64, Expand);
    setOperationAction(ISD::FROUNDEVEN, MVT::f64, Expand);
    setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand);
    setOperationAction(ISD::FFLOOR,     MVT::f64, Expand);
    setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom);
    setOperationAction(ISD::FP_ROUND,   MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FP_ROUND,   MVT::f32, Custom);
  }
 
  // STRICT_(U/S)INT_TO_FP specifically use the input MVT to register with
  // setOperationAction() as opposed to other opcodes that use the output MVT
  // All inputs should be i32 due to type legalization
  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
 
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
 
  if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
    setOperationAction(ISD::FP_EXTEND,  MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
    if (Subtarget->hasFullFP16()) {
      setOperationAction(ISD::FP_ROUND,  MVT::f16, Custom);
      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
    }
  } else {
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
  }
 
  if (!Subtarget->hasFP16()) {
    setOperationAction(ISD::FP_EXTEND,  MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
  } else {
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
  }
 
  computeRegisterProperties(Subtarget->getRegisterInfo());
 
  // ARM does not have floating-point extending loads.
  for (MVT VT : MVT::fp_valuetypes()) {
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
  }
 
  // ... or truncating stores
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
  setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
  setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
 
  // ARM does not have i1 sign extending load.
  for (MVT VT : MVT::integer_valuetypes())
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
 
  // ARM supports all 4 flavors of integer indexed load / store.
  if (!Subtarget->isThumb1Only()) {
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im,  MVT::i1,  Legal);
      setIndexedLoadAction(im,  MVT::i8,  Legal);
      setIndexedLoadAction(im,  MVT::i16, Legal);
      setIndexedLoadAction(im,  MVT::i32, Legal);
      setIndexedStoreAction(im, MVT::i1,  Legal);
      setIndexedStoreAction(im, MVT::i8,  Legal);
      setIndexedStoreAction(im, MVT::i16, Legal);
      setIndexedStoreAction(im, MVT::i32, Legal);
    }
  } else {
    // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
    setIndexedLoadAction(ISD::POST_INC, MVT::i32,  Legal);
    setIndexedStoreAction(ISD::POST_INC, MVT::i32,  Legal);
  }
 
  // Custom loads/stores to possible use __aeabi_uread/write*
  if (TT.isTargetAEABI() && !Subtarget->allowsUnalignedMem()) {
    setOperationAction(ISD::STORE, MVT::i32, Custom);
    setOperationAction(ISD::STORE, MVT::i64, Custom);
    setOperationAction(ISD::LOAD, MVT::i32, Custom);
    setOperationAction(ISD::LOAD, MVT::i64, Custom);
  }
 
  setOperationAction(ISD::SADDO, MVT::i32, Custom);
  setOperationAction(ISD::UADDO, MVT::i32, Custom);
  setOperationAction(ISD::SSUBO, MVT::i32, Custom);
  setOperationAction(ISD::USUBO, MVT::i32, Custom);
 
  if (!Subtarget->isThumb1Only()) {
    setOperationAction(ISD::UMULO, MVT::i32, Custom);
    setOperationAction(ISD::SMULO, MVT::i32, Custom);
  }
 
  setOperationAction(ISD::UADDO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::USUBO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom);
  if (Subtarget->hasDSP()) {
    setOperationAction(ISD::SADDSAT, MVT::i8, Custom);
    setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
    setOperationAction(ISD::SADDSAT, MVT::i16, Custom);
    setOperationAction(ISD::SSUBSAT, MVT::i16, Custom);
    setOperationAction(ISD::UADDSAT, MVT::i8, Custom);
    setOperationAction(ISD::USUBSAT, MVT::i8, Custom);
    setOperationAction(ISD::UADDSAT, MVT::i16, Custom);
    setOperationAction(ISD::USUBSAT, MVT::i16, Custom);
  }
  if (Subtarget->hasBaseDSP()) {
    setOperationAction(ISD::SADDSAT, MVT::i32, Legal);
    setOperationAction(ISD::SSUBSAT, MVT::i32, Legal);
  }
 
  // i64 operation support.
  setOperationAction(ISD::MUL,     MVT::i64, Expand);
  setOperationAction(ISD::MULHU,   MVT::i32, Expand);
  if (Subtarget->isThumb1Only()) {
    setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
    setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
  }
  if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
      || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
    setOperationAction(ISD::MULHS, MVT::i32, Expand);
 
  setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
  setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
  setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
  setOperationAction(ISD::SRL,       MVT::i64, Custom);
  setOperationAction(ISD::SRA,       MVT::i64, Custom);
  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
  setOperationAction(ISD::LOAD, MVT::i64, Custom);
  setOperationAction(ISD::STORE, MVT::i64, Custom);
 
  // MVE lowers 64 bit shifts to lsll and lsrl
  // assuming that ISD::SRL and SRA of i64 are already marked custom
  if (Subtarget->hasMVEIntegerOps())
    setOperationAction(ISD::SHL, MVT::i64, Custom);
 
  // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
  if (Subtarget->isThumb1Only()) {
    setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
    setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
    setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
  }
 
  if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
    setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
 
  // ARM does not have ROTL.
  setOperationAction(ISD::ROTL, MVT::i32, Expand);
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    setOperationAction(ISD::ROTL, VT, Expand);
    setOperationAction(ISD::ROTR, VT, Expand);
  }
  setOperationAction(ISD::CTTZ,  MVT::i32, Custom);
  // TODO: These two should be set to LibCall, but this currently breaks
  //   the Linux kernel build. See #101786.
  setOperationAction(ISD::CTPOP, MVT::i32, Expand);
  setOperationAction(ISD::CTPOP, MVT::i64, Expand);
  if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
    setOperationAction(ISD::CTLZ, MVT::i32, Expand);
    setOperationAction(ISD::CTLZ_ZERO_POISON, MVT::i32, LibCall);
  }
 
  // @llvm.readcyclecounter requires the Performance Monitors extension.
  // Default to the 0 expansion on unsupported platforms.
  // FIXME: Technically there are older ARM CPUs that have
  // implementation-specific ways of obtaining this information.
  if (Subtarget->hasPerfMon())
    setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
 
  // Only ARMv6 has BSWAP.
  if (!Subtarget->hasV6Ops())
    setOperationAction(ISD::BSWAP, MVT::i32, Expand);
 
  bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
                                        : Subtarget->hasDivideInARMMode();
  if (!hasDivide) {
    // These are expanded into libcalls if the cpu doesn't have HW divider.
    setOperationAction(ISD::SDIV,  MVT::i32, LibCall);
    setOperationAction(ISD::UDIV,  MVT::i32, LibCall);
  }
 
  if (TT.isOSWindows() && !Subtarget->hasDivideInThumbMode()) {
    setOperationAction(ISD::SDIV, MVT::i32, Custom);
    setOperationAction(ISD::UDIV, MVT::i32, Custom);
 
    setOperationAction(ISD::SDIV, MVT::i64, Custom);
    setOperationAction(ISD::UDIV, MVT::i64, Custom);
  }
 
  setOperationAction(ISD::SREM,  MVT::i32, Expand);
  setOperationAction(ISD::UREM,  MVT::i32, Expand);
 
  // Register based DivRem for AEABI (RTABI 4.2)
  if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
      TT.isTargetMuslAEABI() || TT.isOSFuchsia() || TT.isOSWindows()) {
    setOperationAction(ISD::SREM, MVT::i64, Custom);
    setOperationAction(ISD::UREM, MVT::i64, Custom);
    HasStandaloneRem = false;
 
    setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
    setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
    setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
    setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
  } else {
    setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
    setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
  }
 
  setOperationAction(ISD::GlobalAddress, MVT::i32,   Custom);
  setOperationAction(ISD::ConstantPool,  MVT::i32,   Custom);
  setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
  setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
 
  setOperationAction(ISD::TRAP, MVT::Other, Legal);
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
 
  // Use the default implementation.
  setOperationAction(ISD::VASTART,            MVT::Other, Custom);
  setOperationAction(ISD::VAARG,              MVT::Other, Expand);
  setOperationAction(ISD::VACOPY,             MVT::Other, Expand);
  setOperationAction(ISD::VAEND,              MVT::Other, Expand);
  setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
  setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
 
  if (TT.isOSWindows())
    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
  else
    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
 
  // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
  // the default expansion.
  InsertFencesForAtomic = false;
  if (Subtarget->hasAnyDataBarrier() &&
      (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
    // ATOMIC_FENCE needs custom lowering; the others should have been expanded
    // to ldrex/strex loops already.
    setOperationAction(ISD::ATOMIC_FENCE,     MVT::Other, Custom);
    if (!Subtarget->isThumb() || !Subtarget->isMClass())
      setOperationAction(ISD::ATOMIC_CMP_SWAP,  MVT::i64, Custom);
 
    // On v8, we have particularly efficient implementations of atomic fences
    // if they can be combined with nearby atomic loads and stores.
    if (!Subtarget->hasAcquireRelease() ||
        getTargetMachine().getOptLevel() == CodeGenOptLevel::None) {
      // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
      InsertFencesForAtomic = true;
    }
  } else {
    // If there's anything we can use as a barrier, go through custom lowering
    // for ATOMIC_FENCE.
    // If target has DMB in thumb, Fences can be inserted.
    if (Subtarget->hasDataBarrier())
      InsertFencesForAtomic = true;
 
    setOperationAction(ISD::ATOMIC_FENCE,   MVT::Other,
                       Subtarget->hasAnyDataBarrier() ? Custom : Expand);
 
    // Set them all for libcall, which will force libcalls.
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, LibCall);
    // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
    // Unordered/Monotonic case.
    if (!InsertFencesForAtomic) {
      setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
      setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
    }
  }
 
  // Compute supported atomic widths.
  if (TT.isOSLinux() || (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
    // For targets where __sync_* routines are reliably available, we use them
    // if necessary.
    //
    // ARM Linux always supports 64-bit atomics through kernel-assisted atomic
    // routines (kernel 3.1 or later). FIXME: Not with compiler-rt?
    //
    // ARMv6 targets have native instructions in ARM mode. For Thumb mode,
    // such targets should provide __sync_* routines, which use the ARM mode
    // instructions. (ARMv6 doesn't have dmb, but it has an equivalent
    // encoding; see ARMISD::MEMBARRIER_MCR.)
    setMaxAtomicSizeInBitsSupported(64);
  } else if ((Subtarget->isMClass() && Subtarget->hasV8MBaselineOps()) ||
             Subtarget->hasForced32BitAtomics()) {
    // Cortex-M (besides Cortex-M0) have 32-bit atomics.
    setMaxAtomicSizeInBitsSupported(32);
  } else {
    // We can't assume anything about other targets; just use libatomic
    // routines.
    setMaxAtomicSizeInBitsSupported(0);
  }
 
  setMaxDivRemBitWidthSupported(64);
 
  setOperationAction(ISD::PREFETCH,         MVT::Other, Custom);
 
  // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
  if (!Subtarget->hasV6Ops()) {
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8,  Expand);
  }
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 
  if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
      !Subtarget->isThumb1Only()) {
    // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
    // iff target supports vfp2.
    setOperationAction(ISD::BITCAST, MVT::i64, Custom);
    setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
    setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
    setOperationAction(ISD::GET_FPENV, MVT::i32, Legal);
    setOperationAction(ISD::SET_FPENV, MVT::i32, Legal);
    setOperationAction(ISD::RESET_FPENV, MVT::Other, Legal);
    setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal);
    setOperationAction(ISD::SET_FPMODE, MVT::i32, Custom);
    setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
  }
 
  // We want to custom lower some of our intrinsics.
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
  setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
 
  setOperationAction(ISD::SETCC,     MVT::i32, Expand);
  setOperationAction(ISD::SETCC,     MVT::f32, Expand);
  setOperationAction(ISD::SETCC,     MVT::f64, Expand);
  setOperationAction(ISD::SELECT,    MVT::i32, Custom);
  setOperationAction(ISD::SELECT,    MVT::f32, Custom);
  setOperationAction(ISD::SELECT,    MVT::f64, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
  if (Subtarget->hasFullFP16()) {
    setOperationAction(ISD::SETCC,     MVT::f16, Expand);
    setOperationAction(ISD::SELECT,    MVT::f16, Custom);
    setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
  }
 
  setOperationAction(ISD::SETCCCARRY, MVT::i32, Custom);
 
  setOperationAction(ISD::BRCOND,    MVT::Other, Custom);
  setOperationAction(ISD::BR_CC,     MVT::i32,   Custom);
  if (Subtarget->hasFullFP16())
      setOperationAction(ISD::BR_CC, MVT::f16,   Custom);
  setOperationAction(ISD::BR_CC,     MVT::f32,   Custom);
  setOperationAction(ISD::BR_CC,     MVT::f64,   Custom);
  setOperationAction(ISD::BR_JT,     MVT::Other, Custom);
 
  // We don't support sin/cos/fmod/copysign/pow
  setOperationAction(ISD::FSIN,      MVT::f64, Expand);
  setOperationAction(ISD::FSIN,      MVT::f32, Expand);
  setOperationAction(ISD::FCOS,      MVT::f32, Expand);
  setOperationAction(ISD::FCOS,      MVT::f64, Expand);
  setOperationAction(ISD::FSINCOS,   MVT::f64, Expand);
  setOperationAction(ISD::FSINCOS,   MVT::f32, Expand);
  setOperationAction(ISD::FREM, MVT::f64, LibCall);
  setOperationAction(ISD::FREM, MVT::f32, LibCall);
  if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
      !Subtarget->isThumb1Only()) {
    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
  }
  setOperationAction(ISD::FPOW,      MVT::f64, Expand);
  setOperationAction(ISD::FPOW,      MVT::f32, Expand);
 
  if (!Subtarget->hasVFP4Base()) {
    setOperationAction(ISD::FMA, MVT::f64, Expand);
    setOperationAction(ISD::FMA, MVT::f32, Expand);
  }
 
  // Various VFP goodness
  if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
    // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
    if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
      setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
      setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
      setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
      setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Expand);
    }
 
    // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
    if (!Subtarget->hasFP16()) {
      setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
      setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
      setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Expand);
      setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Expand);
    }
 
    // Strict floating-point comparisons need custom lowering.
    setOperationAction(ISD::STRICT_FSETCC,  MVT::f16, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FSETCC,  MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
  }
 
  // FP-ARMv8 implements a lot of rounding-like FP operations.
  if (Subtarget->hasFPARMv8Base()) {
    for (auto Op :
         {ISD::FFLOOR,            ISD::FCEIL,             ISD::FROUND,
          ISD::FTRUNC,            ISD::FNEARBYINT,        ISD::FRINT,
          ISD::FROUNDEVEN,        ISD::FMINNUM,           ISD::FMAXNUM,
          ISD::STRICT_FFLOOR,     ISD::STRICT_FCEIL,      ISD::STRICT_FROUND,
          ISD::STRICT_FTRUNC,     ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
          ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM,    ISD::STRICT_FMAXNUM}) {
      setOperationAction(Op, MVT::f32, Legal);
 
      if (Subtarget->hasFP64())
        setOperationAction(Op, MVT::f64, Legal);
    }
 
    if (Subtarget->hasNEON()) {
      setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal);
      setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
    }
  }
 
  // FP16 often need to be promoted to call lib functions
  // clang-format off
  if (Subtarget->hasFullFP16()) {
    setOperationAction(ISD::LRINT, MVT::f16, Expand);
    setOperationAction(ISD::LROUND, MVT::f16, Expand);
    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
 
    for (auto Op : {ISD::FREM,          ISD::FPOW,         ISD::FPOWI,
                  ISD::FCOS,          ISD::FSIN,         ISD::FSINCOS,
                  ISD::FSINCOSPI,     ISD::FMODF,        ISD::FACOS,
                  ISD::FASIN,         ISD::FATAN,        ISD::FATAN2,
                  ISD::FCOSH,         ISD::FSINH,        ISD::FTANH,
                  ISD::FTAN,          ISD::FEXP,         ISD::FEXP2,
                  ISD::FEXP10,        ISD::FLOG,         ISD::FLOG2,
                  ISD::FLOG10,        ISD::STRICT_FREM,  ISD::STRICT_FPOW,
                  ISD::STRICT_FPOWI,  ISD::STRICT_FCOS,  ISD::STRICT_FSIN,
                  ISD::STRICT_FACOS,  ISD::STRICT_FASIN, ISD::STRICT_FATAN,
                  ISD::STRICT_FATAN2, ISD::STRICT_FCOSH, ISD::STRICT_FSINH,
                  ISD::STRICT_FTANH,  ISD::STRICT_FEXP,  ISD::STRICT_FEXP2,
                  ISD::STRICT_FLOG,   ISD::STRICT_FLOG2, ISD::STRICT_FLOG10,
                  ISD::STRICT_FTAN}) {
        setOperationAction(Op, MVT::f16, Promote);
    }
 
    // Round-to-integer need custom lowering for fp16, as Promote doesn't work
    // because the result type is integer.
    for (auto Op : {ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
      setOperationAction(Op, MVT::f16, Custom);
 
    for (auto Op : {ISD::FROUND,         ISD::FROUNDEVEN,        ISD::FTRUNC,
                    ISD::FNEARBYINT,     ISD::FRINT,             ISD::FFLOOR,
                    ISD::FCEIL,          ISD::STRICT_FROUND,     ISD::STRICT_FROUNDEVEN,
                    ISD::STRICT_FTRUNC,  ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
                    ISD::STRICT_FFLOOR,  ISD::STRICT_FCEIL}) {
      setOperationAction(Op, MVT::f16, Legal);
    }
    // clang-format on
  }
 
  if (Subtarget->hasNEON()) {
    // vmin and vmax aren't available in a scalar form, so we can use
    // a NEON instruction with an undef lane instead.
    setOperationAction(ISD::FMINIMUM, MVT::f32, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal);
    setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
    setOperationAction(ISD::FMINIMUM, MVT::v2f32, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::v2f32, Legal);
    setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
    setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
 
    if (Subtarget->hasV8Ops()) {
      for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
                      ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
                      ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
                      ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
        setOperationAction(Op, MVT::v2f32, Legal);
        setOperationAction(Op, MVT::v4f32, Legal);
      }
    }
 
    if (Subtarget->hasFullFP16()) {
      setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMINNUM, MVT::v8f16, Legal);
      setOperationAction(ISD::FMAXNUM, MVT::v8f16, Legal);
 
      setOperationAction(ISD::FMINIMUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMAXIMUM, MVT::v4f16, Legal);
      setOperationAction(ISD::FMINIMUM, MVT::v8f16, Legal);
      setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Legal);
 
      for (auto Op : {ISD::FROUND, ISD::STRICT_FROUND, ISD::FROUNDEVEN,
                      ISD::STRICT_FROUNDEVEN, ISD::FTRUNC, ISD::STRICT_FTRUNC,
                      ISD::FRINT, ISD::STRICT_FRINT, ISD::FFLOOR,
                      ISD::STRICT_FFLOOR, ISD::FCEIL, ISD::STRICT_FCEIL}) {
        setOperationAction(Op, MVT::v4f16, Legal);
        setOperationAction(Op, MVT::v8f16, Legal);
      }
    }
  }
 
  // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined.  MinGW has
  // it, but it's just a wrapper around ldexp.
  if (TT.isOSWindows()) {
    for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
      if (isOperationExpand(Op, MVT::f32))
        setOperationAction(Op, MVT::f32, Promote);
  }
 
  // LegalizeDAG currently can't expand fp16 LDEXP/FREXP on targets where i16
  // isn't legal.
  for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
    if (isOperationExpand(Op, MVT::f16))
      setOperationAction(Op, MVT::f16, Promote);
 
  // We have target-specific dag combine patterns for the following nodes:
  // ARMISD::VMOVRRD  - No need to call setTargetDAGCombine
  setTargetDAGCombine(
      {ISD::ADD, ISD::SUB, ISD::MUL, ISD::AND, ISD::OR, ISD::XOR});
 
  if (Subtarget->hasMVEIntegerOps())
    setTargetDAGCombine(ISD::VSELECT);
 
  if (Subtarget->hasV6Ops())
    setTargetDAGCombine(ISD::SRL);
  if (Subtarget->isThumb1Only())
    setTargetDAGCombine(ISD::SHL);
  // Attempt to lower smin/smax to ssat/usat
  if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) ||
      Subtarget->isThumb2()) {
    setTargetDAGCombine({ISD::SMIN, ISD::SMAX});
  }
 
  setStackPointerRegisterToSaveRestore(ARM::SP);
 
  if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
      !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
    setSchedulingPreference(Sched::RegPressure);
  else
    setSchedulingPreference(Sched::Hybrid);
 
  //// temporary - rewrite interface to use type
  MaxStoresPerMemset = 8;
  MaxStoresPerMemsetOptSize = 4;
  MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
  MaxStoresPerMemcpyOptSize = 2;
  MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
  MaxStoresPerMemmoveOptSize = 2;
 
  // On ARM arguments smaller than 4 bytes are extended, so all arguments
  // are at least 4 bytes aligned.
  setMinStackArgumentAlignment(Align(4));
 
  // Prefer likely predicted branches to selects on out-of-order cores.
  PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
 
  setPrefLoopAlignment(Align(1ULL << Subtarget->getPreferBranchLogAlignment()));
  setPrefFunctionAlignment(
      Align(1ULL << Subtarget->getPreferBranchLogAlignment()));
 
  setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
 
  IsStrictFPEnabled = true;
}
 
bool ARMTargetLowering::useSoftFloat() const {
  return Subtarget->useSoftFloat();
}
 
bool ARMTargetLowering::preferSelectsOverBooleanArithmetic(EVT VT) const {
  return !Subtarget->isThumb1Only() && VT.getSizeInBits() <= 32;
}
 
// FIXME: It might make sense to define the representative register class as the
// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
// SPR's representative would be DPR_VFP2. This should work well if register
// pressure tracking were modified such that a register use would increment the
// pressure of the register class's representative and all of it's super
// classes' representatives transitively. We have not implemented this because
// of the difficulty prior to coalescing of modeling operand register classes
// due to the common occurrence of cross class copies and subregister insertions
// and extractions.
std::pair<const TargetRegisterClass *, uint8_t>
ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
                                           MVT VT) const {
  const TargetRegisterClass *RRC = nullptr;
  uint8_t Cost = 1;
  switch (VT.SimpleTy) {
  default:
    return TargetLowering::findRepresentativeClass(TRI, VT);
  // Use DPR as representative register class for all floating point
  // and vector types. Since there are 32 SPR registers and 32 DPR registers so
  // the cost is 1 for both f32 and f64.
  case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
  case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
    RRC = &ARM::DPRRegClass;
    // When NEON is used for SP, only half of the register file is available
    // because operations that define both SP and DP results will be constrained
    // to the VFP2 class (D0-D15). We currently model this constraint prior to
    // coalescing by double-counting the SP regs. See the FIXME above.
    if (Subtarget->useNEONForSinglePrecisionFP())
      Cost = 2;
    break;
  case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
  case MVT::v4f32: case MVT::v2f64:
    RRC = &ARM::DPRRegClass;
    Cost = 2;
    break;
  case MVT::v4i64:
    RRC = &ARM::DPRRegClass;
    Cost = 4;
    break;
  case MVT::v8i64:
    RRC = &ARM::DPRRegClass;
    Cost = 8;
    break;
  }
  return std::make_pair(RRC, Cost);
}
 
EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
                                          EVT VT) const {
  if (!VT.isVector())
    return getPointerTy(DL);
 
  // MVE has a predicate register.
  if (Subtarget->hasMVEIntegerOps())
    return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
 
  return VT.changeVectorElementTypeToInteger();
}
 
/// getRegClassFor - Return the register class that should be used for the
/// specified value type.
const TargetRegisterClass *
ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
  (void)isDivergent;
  // Map v4i64 to QQ registers but do not make the type legal. Similarly map
  // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
  // load / store 4 to 8 consecutive NEON D registers, or 2 to 4 consecutive
  // MVE Q registers.
  if (Subtarget->hasNEON()) {
    if (VT == MVT::v4i64)
      return &ARM::QQPRRegClass;
    if (VT == MVT::v8i64)
      return &ARM::QQQQPRRegClass;
  }
  if (Subtarget->hasMVEIntegerOps()) {
    if (VT == MVT::v4i64)
      return &ARM::MQQPRRegClass;
    if (VT == MVT::v8i64)
      return &ARM::MQQQQPRRegClass;
  }
  return TargetLowering::getRegClassFor(VT);
}
 
// memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
// source/dest is aligned and the copy size is large enough. We therefore want
// to align such objects passed to memory intrinsics.
bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
                                               Align &PrefAlign) const {
  if (!isa<MemIntrinsic>(CI))
    return false;
  MinSize = 8;
  // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
  // cycle faster than 4-byte aligned LDM.
  PrefAlign =
      (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? Align(8) : Align(4));
  return true;
}
 
// Create a fast isel object.
FastISel *ARMTargetLowering::createFastISel(
    FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo,
    const LibcallLoweringInfo *libcallLowering) const {
  return ARM::createFastISel(funcInfo, libInfo, libcallLowering);
}
 
Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
  unsigned NumVals = N->getNumValues();
  if (!NumVals)
    return Sched::RegPressure;
 
  for (unsigned i = 0; i != NumVals; ++i) {
    EVT VT = N->getValueType(i);
    if (VT == MVT::Glue || VT == MVT::Other)
      continue;
    if (VT.isFloatingPoint() || VT.isVector())
      return Sched::ILP;
  }
 
  if (!N->isMachineOpcode())
    return Sched::RegPressure;
 
  // Load are scheduled for latency even if there instruction itinerary
  // is not available.
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
 
  if (MCID.getNumDefs() == 0)
    return Sched::RegPressure;
  if (!Itins->isEmpty() &&
      Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2U)
    return Sched::ILP;
 
  return Sched::RegPressure;
}
 
//===----------------------------------------------------------------------===//
// Lowering Code
//===----------------------------------------------------------------------===//
 
static bool isSRL16(const SDValue &Op) {
  if (Op.getOpcode() != ISD::SRL)
    return false;
  if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
    return Const->getZExtValue() == 16;
  return false;
}
 
static bool isSRA16(const SDValue &Op) {
  if (Op.getOpcode() != ISD::SRA)
    return false;
  if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
    return Const->getZExtValue() == 16;
  return false;
}
 
static bool isSHL16(const SDValue &Op) {
  if (Op.getOpcode() != ISD::SHL)
    return false;
  if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
    return Const->getZExtValue() == 16;
  return false;
}
 
// Check for a signed 16-bit value. We special case SRA because it makes it
// more simple when also looking for SRAs that aren't sign extending a
// smaller value. Without the check, we'd need to take extra care with
// checking order for some operations.
static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
  if (isSRA16(Op))
    return isSHL16(Op.getOperand(0));
  return DAG.ComputeNumSignBits(Op) == 17;
}
 
/// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
  switch (CC) {
  default: llvm_unreachable("Unknown condition code!");
  case ISD::SETNE:  return ARMCC::NE;
  case ISD::SETEQ:  return ARMCC::EQ;
  case ISD::SETGT:  return ARMCC::GT;
  case ISD::SETGE:  return ARMCC::GE;
  case ISD::SETLT:  return ARMCC::LT;
  case ISD::SETLE:  return ARMCC::LE;
  case ISD::SETUGT: return ARMCC::HI;
  case ISD::SETUGE: return ARMCC::HS;
  case ISD::SETULT: return ARMCC::LO;
  case ISD::SETULE: return ARMCC::LS;
  }
}
 
/// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
                        ARMCC::CondCodes &CondCode2) {
  CondCode2 = ARMCC::AL;
  switch (CC) {
  default: llvm_unreachable("Unknown FP condition!");
  case ISD::SETEQ:
  case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
  case ISD::SETGT:
  case ISD::SETOGT: CondCode = ARMCC::GT; break;
  case ISD::SETGE:
  case ISD::SETOGE: CondCode = ARMCC::GE; break;
  case ISD::SETOLT: CondCode = ARMCC::MI; break;
  case ISD::SETOLE: CondCode = ARMCC::LS; break;
  case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
  case ISD::SETO:   CondCode = ARMCC::VC; break;
  case ISD::SETUO:  CondCode = ARMCC::VS; break;
  case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
  case ISD::SETUGT: CondCode = ARMCC::HI; break;
  case ISD::SETUGE: CondCode = ARMCC::PL; break;
  case ISD::SETLT:
  case ISD::SETULT: CondCode = ARMCC::LT; break;
  case ISD::SETLE:
  case ISD::SETULE: CondCode = ARMCC::LE; break;
  case ISD::SETNE:
  case ISD::SETUNE: CondCode = ARMCC::NE; break;
  }
}
 
//===----------------------------------------------------------------------===//
//                      Calling Convention Implementation
//===----------------------------------------------------------------------===//
 
/// getEffectiveCallingConv - Get the effective calling convention, taking into
/// account presence of floating point hardware and calling convention
/// limitations, such as support for variadic functions.
CallingConv::ID
ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
                                           bool isVarArg) const {
  switch (CC) {
  default:
    // Unknown CCs are rejected when calling convention lowering is required.
  case CallingConv::ARM_AAPCS:
  case CallingConv::ARM_APCS:
  case CallingConv::GHC:
  case CallingConv::CFGuard_Check:
    return CC;
  case CallingConv::PreserveMost:
    return CallingConv::PreserveMost;
  case CallingConv::PreserveAll:
    return CallingConv::PreserveAll;
  case CallingConv::ARM_AAPCS_VFP:
  case CallingConv::Swift:
  case CallingConv::SwiftTail:
    return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
  case CallingConv::C:
  case CallingConv::Tail:
    if (!Subtarget->isAAPCS_ABI())
      return CallingConv::ARM_APCS;
    else if (Subtarget->isTargetHardFloat() && !isVarArg)
      return CallingConv::ARM_AAPCS_VFP;
    else
      return CallingConv::ARM_AAPCS;
  case CallingConv::Fast:
  case CallingConv::CXX_FAST_TLS:
    if (!Subtarget->isAAPCS_ABI()) {
      if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() && !isVarArg)
        return CallingConv::Fast;
      return CallingConv::ARM_APCS;
    } else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() &&
               !isVarArg)
      return CallingConv::ARM_AAPCS_VFP;
    else
      return CallingConv::ARM_AAPCS;
  }
}
 
CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
                                                 bool isVarArg) const {
  return CCAssignFnForNode(CC, false, isVarArg);
}
 
CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
                                                   bool isVarArg) const {
  return CCAssignFnForNode(CC, true, isVarArg);
}
 
/// CCAssignFnForNode - Selects the correct CCAssignFn for the given
/// CallingConvention.
CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
                                                 bool Return,
                                                 bool isVarArg) const {
  switch (getEffectiveCallingConv(CC, isVarArg)) {
  default:
    report_fatal_error("Unsupported calling convention");
  case CallingConv::ARM_APCS:
    return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
  case CallingConv::ARM_AAPCS:
    return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
  case CallingConv::ARM_AAPCS_VFP:
    return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
  case CallingConv::Fast:
    return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
  case CallingConv::GHC:
    return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
  case CallingConv::PreserveMost:
    return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
  case CallingConv::PreserveAll:
    return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
  case CallingConv::CFGuard_Check:
    return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
  }
}
 
SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
                                     MVT LocVT, MVT ValVT, SDValue Val) const {
  Val = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocVT.getSizeInBits()),
                    Val);
  if (Subtarget->hasFullFP16()) {
    Val = DAG.getNode(ARMISD::VMOVhr, dl, ValVT, Val);
  } else {
    Val = DAG.getNode(ISD::TRUNCATE, dl,
                      MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
    Val = DAG.getNode(ISD::BITCAST, dl, ValVT, Val);
  }
  return Val;
}
 
SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
                                       MVT LocVT, MVT ValVT,
                                       SDValue Val) const {
  if (Subtarget->hasFullFP16()) {
    Val = DAG.getNode(ARMISD::VMOVrh, dl,
                      MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
  } else {
    Val = DAG.getNode(ISD::BITCAST, dl,
                      MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
    Val = DAG.getNode(ISD::ZERO_EXTEND, dl,
                      MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
  }
  return DAG.getNode(ISD::BITCAST, dl, LocVT, Val);
}
 
/// LowerCallResult - Lower the result values of a call into the
/// appropriate copies out of appropriate physical registers.
SDValue ARMTargetLowering::LowerCallResult(
    SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
    SDValue ThisVal, bool isCmseNSCall) const {
  // Assign locations to each value returned by this call.
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
  CCInfo.AnalyzeCallResult(Ins, CCAssignFnForReturn(CallConv, isVarArg));
 
  // Copy all of the result registers out of their specified physreg.
  for (unsigned i = 0; i != RVLocs.size(); ++i) {
    CCValAssign VA = RVLocs[i];
 
    // Pass 'this' value directly from the argument to return value, to avoid
    // reg unit interference
    if (i == 0 && isThisReturn) {
      assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
             "unexpected return calling convention register assignment");
      InVals.push_back(ThisVal);
      continue;
    }
 
    SDValue Val;
    if (VA.needsCustom() &&
        (VA.getLocVT() == MVT::f64 || VA.getLocVT() == MVT::v2f64)) {
      // Handle f64 or half of a v2f64.
      SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
                                      InGlue);
      Chain = Lo.getValue(1);
      InGlue = Lo.getValue(2);
      VA = RVLocs[++i]; // skip ahead to next loc
      SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
                                      InGlue);
      Chain = Hi.getValue(1);
      InGlue = Hi.getValue(2);
      if (!Subtarget->isLittle())
        std::swap (Lo, Hi);
      Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
 
      if (VA.getLocVT() == MVT::v2f64) {
        SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
        Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
                          DAG.getConstant(0, dl, MVT::i32));
 
        VA = RVLocs[++i]; // skip ahead to next loc
        Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InGlue);
        Chain = Lo.getValue(1);
        InGlue = Lo.getValue(2);
        VA = RVLocs[++i]; // skip ahead to next loc
        Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InGlue);
        Chain = Hi.getValue(1);
        InGlue = Hi.getValue(2);
        if (!Subtarget->isLittle())
          std::swap (Lo, Hi);
        Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
        Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
                          DAG.getConstant(1, dl, MVT::i32));
      }
    } else {
      Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
                               InGlue);
      Chain = Val.getValue(1);
      InGlue = Val.getValue(2);
    }
 
    switch (VA.getLocInfo()) {
    default: llvm_unreachable("Unknown loc info!");
    case CCValAssign::Full: break;
    case CCValAssign::BCvt:
      Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val);
      break;
    }
 
    // f16 arguments have their size extended to 4 bytes and passed as if they
    // had been copied to the LSBs of a 32-bit register.
    // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
    if (VA.needsCustom() &&
        (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
      Val = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Val);
 
    // On CMSE Non-secure Calls, call results (returned values) whose bitwidth
    // is less than 32 bits must be sign- or zero-extended after the call for
    // security reasons. Although the ABI mandates an extension done by the
    // callee, the latter cannot be trusted to follow the rules of the ABI.
    const ISD::InputArg &Arg = Ins[VA.getValNo()];
    if (isCmseNSCall && Arg.ArgVT.isScalarInteger() &&
        VA.getLocVT().isScalarInteger() && Arg.ArgVT.bitsLT(MVT::i32))
      Val = handleCMSEValue(Val, Arg, DAG, dl);
 
    InVals.push_back(Val);
  }
 
  return Chain;
}
 
std::pair<SDValue, MachinePointerInfo> ARMTargetLowering::computeAddrForCallArg(
    const SDLoc &dl, SelectionDAG &DAG, const CCValAssign &VA, SDValue StackPtr,
    bool IsTailCall, int SPDiff) const {
  SDValue DstAddr;
  MachinePointerInfo DstInfo;
  int32_t Offset = VA.getLocMemOffset();
  MachineFunction &MF = DAG.getMachineFunction();
 
  if (IsTailCall) {
        Offset += SPDiff;
        auto PtrVT = getPointerTy(DAG.getDataLayout());
        int Size = VA.getLocVT().getFixedSizeInBits() / 8;
        int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
        DstAddr = DAG.getFrameIndex(FI, PtrVT);
        DstInfo =
            MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
  } else {
        SDValue PtrOff = DAG.getIntPtrConstant(Offset, dl);
        DstAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
                              StackPtr, PtrOff);
        DstInfo =
            MachinePointerInfo::getStack(DAG.getMachineFunction(), Offset);
  }
 
  return std::make_pair(DstAddr, DstInfo);
}
 
// Returns the type of copying which is required to set up a byval argument to
// a tail-called function. This isn't needed for non-tail calls, because they
// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
// avoid clobbering another argument (CopyViaTemp), and sometimes can be
// optimised to zero copies when forwarding an argument from the caller's
// caller (NoCopy).
ARMTargetLowering::ByValCopyKind ARMTargetLowering::ByValNeedsCopyForTailCall(
    SelectionDAG &DAG, SDValue Src, SDValue Dst, ISD::ArgFlagsTy Flags) const {
  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
  ARMFunctionInfo *AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
 
  // Globals are always safe to copy from.
  if (isa<GlobalAddressSDNode>(Src) || isa<ExternalSymbolSDNode>(Src))
    return CopyOnce;
 
  // Can only analyse frame index nodes, conservatively assume we need a
  // temporary.
  auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Src);
  auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Dst);
  if (!SrcFrameIdxNode || !DstFrameIdxNode)
    return CopyViaTemp;
 
  int SrcFI = SrcFrameIdxNode->getIndex();
  int DstFI = DstFrameIdxNode->getIndex();
  assert(MFI.isFixedObjectIndex(DstFI) &&
         "byval passed in non-fixed stack slot");
 
  int64_t SrcOffset = MFI.getObjectOffset(SrcFI);
  int64_t DstOffset = MFI.getObjectOffset(DstFI);
 
  // If the source is in the local frame, then the copy to the argument memory
  // is always valid.
  bool FixedSrc = MFI.isFixedObjectIndex(SrcFI);
  if (!FixedSrc ||
      (FixedSrc && SrcOffset < -(int64_t)AFI->getArgRegsSaveSize()))
    return CopyOnce;
 
  // In the case of byval arguments split between registers and the stack,
  // computeAddrForCallArg returns a FrameIndex which corresponds only to the
  // stack portion, but the Src SDValue will refer to the full value, including
  // the local stack memory that the register portion gets stored into. We only
  // need to compare them for equality, so normalise on the full value version.
  uint64_t RegSize = Flags.getByValSize() - MFI.getObjectSize(DstFI);
  DstOffset -= RegSize;
 
  // If the value is already in the correct location, then no copying is
  // needed. If not, then we need to copy via a temporary.
  if (SrcOffset == DstOffset)
    return NoCopy;
  else
    return CopyViaTemp;
}
 
void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
                                         SDValue Chain, SDValue &Arg,
                                         RegsToPassVector &RegsToPass,
                                         CCValAssign &VA, CCValAssign &NextVA,
                                         SDValue &StackPtr,
                                         SmallVectorImpl<SDValue> &MemOpChains,
                                         bool IsTailCall,
                                         int SPDiff) const {
  SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
                              DAG.getVTList(MVT::i32, MVT::i32), Arg);
  unsigned id = Subtarget->isLittle() ? 0 : 1;
  RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id)));
 
  if (NextVA.isRegLoc())
    RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id)));
  else {
    assert(NextVA.isMemLoc());
    if (!StackPtr.getNode())
      StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
                                    getPointerTy(DAG.getDataLayout()));
 
    SDValue DstAddr;
    MachinePointerInfo DstInfo;
    std::tie(DstAddr, DstInfo) =
        computeAddrForCallArg(dl, DAG, NextVA, StackPtr, IsTailCall, SPDiff);
    MemOpChains.push_back(
        DAG.getStore(Chain, dl, fmrrd.getValue(1 - id), DstAddr, DstInfo));
  }
}
 
static bool canGuaranteeTCO(CallingConv::ID CC, bool GuaranteeTailCalls) {
  return (CC == CallingConv::Fast && GuaranteeTailCalls) ||
         CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
}
 
/// LowerCall - Lowering a call into a callseq_start <-
/// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
/// nodes.
SDValue
ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                             SmallVectorImpl<SDValue> &InVals) const {
  SelectionDAG &DAG                     = CLI.DAG;
  SDLoc &dl                             = CLI.DL;
  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
  SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
  SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
  SDValue Chain                         = CLI.Chain;
  SDValue Callee                        = CLI.Callee;
  bool &isTailCall                      = CLI.IsTailCall;
  CallingConv::ID CallConv              = CLI.CallConv;
  bool doesNotRet                       = CLI.DoesNotReturn;
  bool isVarArg                         = CLI.IsVarArg;
  const CallBase *CB = CLI.CB;
 
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
  MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
  MachineFunction::CallSiteInfo CSInfo;
  bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
  bool isThisReturn = false;
  bool isCmseNSCall   = false;
  bool isSibCall = false;
  bool PreferIndirect = false;
  bool GuardWithBTI = false;
 
  // Analyze operands of the call, assigning locations to each operand.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                 *DAG.getContext());
  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CallConv, isVarArg));
 
  // Lower 'returns_twice' calls to a pseudo-instruction.
  if (CLI.CB && CLI.CB->getAttributes().hasFnAttr(Attribute::ReturnsTwice) &&
      !Subtarget->noBTIAtReturnTwice())
    GuardWithBTI = AFI->branchTargetEnforcement();
 
  // Set type id for call site info.
  setTypeIdForCallsiteInfo(CB, MF, CSInfo);
 
  // Determine whether this is a non-secure function call.
  if (CLI.CB && CLI.CB->getAttributes().hasFnAttr("cmse_nonsecure_call"))
    isCmseNSCall = true;
 
  // Disable tail calls if they're not supported.
  if (!Subtarget->supportsTailCall())
    isTailCall = false;
 
  // For both the non-secure calls and the returns from a CMSE entry function,
  // the function needs to do some extra work after the call, or before the
  // return, respectively, thus it cannot end with a tail call
  if (isCmseNSCall || AFI->isCmseNSEntryFunction())
    isTailCall = false;
 
  if (isa<GlobalAddressSDNode>(Callee)) {
    // If we're optimizing for minimum size and the function is called three or
    // more times in this block, we can improve codesize by calling indirectly
    // as BLXr has a 16-bit encoding.
    auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
    if (CLI.CB) {
      auto *BB = CLI.CB->getParent();
      PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
                       count_if(GV->users(), [&BB](const User *U) {
                         return isa<Instruction>(U) &&
                                cast<Instruction>(U)->getParent() == BB;
                       }) > 2;
    }
  }
  if (isTailCall) {
    // Check if it's really possible to do a tail call.
    isTailCall =
        IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs, PreferIndirect);
 
    if (isTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt &&
        CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
      isSibCall = true;
 
    // We don't support GuaranteedTailCallOpt for ARM, only automatically
    // detected sibcalls.
    if (isTailCall)
      ++NumTailCalls;
  }
 
  if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
    report_fatal_error("failed to perform tail call elimination on a call "
                       "site marked musttail");
 
  // Get a count of how many bytes are to be pushed on the stack.
  unsigned NumBytes = CCInfo.getStackSize();
 
  // SPDiff is the byte offset of the call's argument area from the callee's.
  // Stores to callee stack arguments will be placed in FixedStackSlots offset
  // by this amount for a tail call. In a sibling call it must be 0 because the
  // caller will deallocate the entire stack and the callee still expects its
  // arguments to begin at SP+0. Completely unused for non-tail calls.
  int SPDiff = 0;
 
  if (isTailCall && !isSibCall) {
    auto FuncInfo = MF.getInfo<ARMFunctionInfo>();
    unsigned NumReusableBytes = FuncInfo->getArgumentStackSize();
 
    // Since callee will pop argument stack as a tail call, we must keep the
    // popped size 16-byte aligned.
    MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
    assert(StackAlign && "data layout string is missing stack alignment");
    NumBytes = alignTo(NumBytes, *StackAlign);
 
    // SPDiff will be negative if this tail call requires more space than we
    // would automatically have in our incoming argument space. Positive if we
    // can actually shrink the stack.
    SPDiff = NumReusableBytes - NumBytes;
 
    // If this call requires more stack than we have available from
    // LowerFormalArguments, tell FrameLowering to reserve space for it.
    if (SPDiff < 0 && AFI->getArgRegsSaveSize() < (unsigned)-SPDiff)
      AFI->setArgRegsSaveSize(-SPDiff);
  }
 
  if (isSibCall) {
    // For sibling tail calls, memory operands are available in our caller's stack.
    NumBytes = 0;
  } else {
    // Adjust the stack pointer for the new arguments...
    // These operations are automatically eliminated by the prolog/epilog pass
    Chain = DAG.getCALLSEQ_START(Chain, isTailCall ? 0 : NumBytes, 0, dl);
  }
 
  SDValue StackPtr =
      DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
 
  RegsToPassVector RegsToPass;
  SmallVector<SDValue, 8> MemOpChains;
 
  // If we are doing a tail-call, any byval arguments will be written to stack
  // space which was used for incoming arguments. If any the values being used
  // are incoming byval arguments to this function, then they might be
  // overwritten by the stores of the outgoing arguments. To avoid this, we
  // need to make a temporary copy of them in local stack space, then copy back
  // to the argument area.
  DenseMap<unsigned, SDValue> ByValTemporaries;
  SDValue ByValTempChain;
  if (isTailCall) {
    SmallVector<SDValue, 8> ByValCopyChains;
    for (const CCValAssign &VA : ArgLocs) {
      unsigned ArgIdx = VA.getValNo();
      SDValue Src = OutVals[ArgIdx];
      ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
 
      if (!Flags.isByVal())
        continue;
 
      SDValue Dst;
      MachinePointerInfo DstInfo;
      std::tie(Dst, DstInfo) =
          computeAddrForCallArg(dl, DAG, VA, SDValue(), true, SPDiff);
      ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
 
      if (Copy == NoCopy) {
        // If the argument is already at the correct offset on the stack
        // (because we are forwarding a byval argument from our caller), we
        // don't need any copying.
        continue;
      } else if (Copy == CopyOnce) {
        // If the argument is in our local stack frame, no other argument
        // preparation can clobber it, so we can copy it to the final location
        // later.
        ByValTemporaries[ArgIdx] = Src;
      } else {
        assert(Copy == CopyViaTemp && "unexpected enum value");
        // If we might be copying this argument from the outgoing argument
        // stack area, we need to copy via a temporary in the local stack
        // frame.
        int TempFrameIdx = MFI.CreateStackObject(
            Flags.getByValSize(), Flags.getNonZeroByValAlign(), false);
        SDValue Temp =
            DAG.getFrameIndex(TempFrameIdx, getPointerTy(DAG.getDataLayout()));
 
        SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
        SDValue AlignNode =
            DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
 
        SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
        SDValue Ops[] = {Chain, Temp, Src, SizeNode, AlignNode};
        ByValCopyChains.push_back(
            DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs, Ops));
        ByValTemporaries[ArgIdx] = Temp;
      }
    }
    if (!ByValCopyChains.empty())
      ByValTempChain =
          DAG.getNode(ISD::TokenFactor, dl, MVT::Other, ByValCopyChains);
  }
 
  // During a tail call, stores to the argument area must happen after all of
  // the function's incoming arguments have been loaded because they may alias.
  // This is done by folding in a TokenFactor from LowerFormalArguments, but
  // there's no point in doing so repeatedly so this tracks whether that's
  // happened yet.
  bool AfterFormalArgLoads = false;
 
  // Walk the register/memloc assignments, inserting copies/loads.  In the case
  // of tail call optimization, arguments are handled later.
  for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
       i != e;
       ++i, ++realArgIdx) {
    CCValAssign &VA = ArgLocs[i];
    SDValue Arg = OutVals[realArgIdx];
    ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
    bool isByVal = Flags.isByVal();
 
    // Promote the value if needed.
    switch (VA.getLocInfo()) {
    default: llvm_unreachable("Unknown loc info!");
    case CCValAssign::Full: break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    case CCValAssign::BCvt:
      Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
      break;
    }
 
    if (isTailCall && VA.isMemLoc() && !AfterFormalArgLoads) {
      Chain = DAG.getStackArgumentTokenFactor(Chain);
      if (ByValTempChain) {
        // In case of large byval copies, re-using the stackframe for tail-calls
        // can lead to overwriting incoming arguments on the stack. Force
        // loading these stack arguments before the copy to avoid that.
        SmallVector<SDValue, 8> IncomingLoad;
        for (unsigned I = 0; I < OutVals.size(); ++I) {
          if (Outs[I].Flags.isByVal())
            continue;
 
          SDValue OutVal = OutVals[I];
          LoadSDNode *OutLN = dyn_cast_or_null<LoadSDNode>(OutVal);
          if (!OutLN)
            continue;
 
          FrameIndexSDNode *FIN =
              dyn_cast_or_null<FrameIndexSDNode>(OutLN->getBasePtr());
          if (!FIN)
            continue;
 
          if (!MFI.isFixedObjectIndex(FIN->getIndex()))
            continue;
 
          for (const CCValAssign &VA : ArgLocs) {
            if (VA.isMemLoc())
              IncomingLoad.push_back(OutVal.getValue(1));
          }
        }
 
        // Update the chain to force loads for potentially clobbered argument
        // loads to happen before the byval copy.
        if (!IncomingLoad.empty()) {
          IncomingLoad.push_back(Chain);
          Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, IncomingLoad);
        }
 
        Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chain,
                            ByValTempChain);
      }
      AfterFormalArgLoads = true;
    }
 
    // f16 arguments have their size extended to 4 bytes and passed as if they
    // had been copied to the LSBs of a 32-bit register.
    // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
    if (VA.needsCustom() &&
        (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
      Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
    } else {
      // f16 arguments could have been extended prior to argument lowering.
      // Mask them arguments if this is a CMSE nonsecure call.
      auto ArgVT = Outs[realArgIdx].ArgVT;
      if (isCmseNSCall && (ArgVT == MVT::f16)) {
        auto LocBits = VA.getLocVT().getSizeInBits();
        auto MaskValue = APInt::getLowBitsSet(LocBits, ArgVT.getSizeInBits());
        SDValue Mask =
            DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
        Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
        Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
        Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
      }
    }
 
    // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
    if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
      SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
                                DAG.getConstant(0, dl, MVT::i32));
      SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
                                DAG.getConstant(1, dl, MVT::i32));
 
      PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
                       StackPtr, MemOpChains, isTailCall, SPDiff);
 
      VA = ArgLocs[++i]; // skip ahead to next loc
      if (VA.isRegLoc()) {
        PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
                         StackPtr, MemOpChains, isTailCall, SPDiff);
      } else {
        assert(VA.isMemLoc());
        SDValue DstAddr;
        MachinePointerInfo DstInfo;
        std::tie(DstAddr, DstInfo) =
            computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
        MemOpChains.push_back(DAG.getStore(Chain, dl, Op1, DstAddr, DstInfo));
      }
    } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
      PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
                       StackPtr, MemOpChains, isTailCall, SPDiff);
    } else if (VA.isRegLoc()) {
      if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
          Outs[0].VT == MVT::i32) {
        assert(VA.getLocVT() == MVT::i32 &&
               "unexpected calling convention register assignment");
        assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
               "unexpected use of 'returned'");
        isThisReturn = true;
      }
      const TargetOptions &Options = DAG.getTarget().Options;
      if (Options.EmitCallSiteInfo)
        CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), i);
      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
    } else if (isByVal) {
      assert(VA.isMemLoc());
      unsigned offset = 0;
 
      // True if this byval aggregate will be split between registers
      // and memory.
      unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
      unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
 
      SDValue ByValSrc;
      bool NeedsStackCopy;
      if (auto It = ByValTemporaries.find(realArgIdx);
          It != ByValTemporaries.end()) {
        ByValSrc = It->second;
        NeedsStackCopy = true;
      } else {
        ByValSrc = Arg;
        NeedsStackCopy = !isTailCall;
      }
 
      // If part of the argument is in registers, load them.
      if (CurByValIdx < ByValArgsCount) {
        unsigned RegBegin, RegEnd;
        CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
 
        EVT PtrVT = getPointerTy(DAG.getDataLayout());
        unsigned int i, j;
        for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
          SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
          SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, Const);
          SDValue Load =
              DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
                          DAG.InferPtrAlign(AddArg));
          MemOpChains.push_back(Load.getValue(1));
          RegsToPass.push_back(std::make_pair(j, Load));
        }
 
        // If parameter size outsides register area, "offset" value
        // helps us to calculate stack slot for remained part properly.
        offset = RegEnd - RegBegin;
 
        CCInfo.nextInRegsParam();
      }
 
      // If the memory part of the argument isn't already in the correct place
      // (which can happen with tail calls), copy it into the argument area.
      if (NeedsStackCopy && Flags.getByValSize() > 4 * offset) {
        auto PtrVT = getPointerTy(DAG.getDataLayout());
        SDValue Dst;
        MachinePointerInfo DstInfo;
        std::tie(Dst, DstInfo) =
            computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
        SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
        SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, ByValSrc, SrcOffset);
        SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
                                           MVT::i32);
        SDValue AlignNode =
            DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
 
        SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
        SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
        MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
                                          Ops));
      }
    } else {
      assert(VA.isMemLoc());
      SDValue DstAddr;
      MachinePointerInfo DstInfo;
      std::tie(DstAddr, DstInfo) =
          computeAddrForCallArg(dl, DAG, VA, StackPtr, isTailCall, SPDiff);
 
      SDValue Store = DAG.getStore(Chain, dl, Arg, DstAddr, DstInfo);
      MemOpChains.push_back(Store);
    }
  }
 
  if (!MemOpChains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
 
  // Build a sequence of copy-to-reg nodes chained together with token chain
  // and flag operands which copy the outgoing args into the appropriate regs.
  SDValue InGlue;
  for (const auto &[Reg, N] : RegsToPass) {
    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
    InGlue = Chain.getValue(1);
  }
 
  // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
  // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
  // node so that legalize doesn't hack it.
  bool isDirect = false;
 
  const TargetMachine &TM = getTargetMachine();
  const Triple &TT = TM.getTargetTriple();
  const GlobalValue *GVal = nullptr;
  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
    GVal = G->getGlobal();
  bool isStub = !TM.shouldAssumeDSOLocal(GVal) && TT.isOSBinFormatMachO();
 
  bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
  bool isLocalARMFunc = false;
  auto PtrVt = getPointerTy(DAG.getDataLayout());
 
  if (Subtarget->genLongCalls()) {
    bool isPIC = isPositionIndependent() && !TT.isOSWindows();
    if (isPIC && Subtarget->genExecuteOnly())
      reportFatalUsageError("long-calls with execute-only and "
                            "position-independent code is not supported");
    if (Subtarget->isROPI())
      reportFatalUsageError("long-calls with ROPI is not currently supported");
 
    // Handle a global address or an external symbol. If it's not one of
    // those, the target's already in a register, so we don't need to do
    // anything extra.
    if (isa<GlobalAddressSDNode>(Callee)) {
      if (Subtarget->genExecuteOnly()) {
        // Execute-only forbids constant pools in .text, so use movw/movt.
        // fPIC is not supported with execute-only.
        if (Subtarget->useMovt())
          ++NumMovwMovt;
        Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
                             DAG.getTargetGlobalAddress(GVal, dl, PtrVt));
      } else if (isPIC) {
        // PIC without execute-only: use GOT-based addressing.
        // DSO-local symbols use a plain PC-relative WrapperPIC;
        // non-DSO-local symbols additionally load the address from the GOT.
        SDValue G = DAG.getTargetGlobalAddress(
            GVal, dl, PtrVt, 0, GVal->isDSOLocal() ? 0 : ARMII::MO_GOT);
        Callee = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVt, G);
        if (!GVal->isDSOLocal())
          Callee =
              DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee,
                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
      } else {
        // Neither execute-only nor PIC: load the address from a constant pool.
        unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
        ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
            GVal, ARMPCLabelIndex, ARMCP::CPValue, 0);
 
        // Get the address of the callee into a register
        SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
        Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
        Callee = DAG.getLoad(
            PtrVt, dl, DAG.getEntryNode(), Addr,
            MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
      }
    } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) {
      const char *Sym = S->getSymbol();
 
      if (Subtarget->genExecuteOnly()) {
        // Execute-only forbids constant pools in .text, so use movw/movt.
        // fPIC is not supported with execute-only.
        if (Subtarget->useMovt())
          ++NumMovwMovt;
        Callee = DAG.getNode(ARMISD::Wrapper, dl, PtrVt,
                             DAG.getTargetExternalSymbol(Sym, PtrVt, 0));
      } else if (isPIC) {
        // PIC without execute-only: load the symbol's address from the GOT via
        // a GOT_PREL constant pool entry consumed by a PICLDR.
        unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
        unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
        ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
            *DAG.getContext(), Sym, ARMPCLabelIndex, PCAdj, ARMCP::GOT_PREL,
            /*AddCurrentAddress=*/true);
        SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
        CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
        SDValue GOTOffset = DAG.getLoad(
            PtrVt, dl, DAG.getEntryNode(), CPAddr,
            MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
        SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
        Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, GOTOffset, PICLabel);
        Callee =
            DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee,
                        MachinePointerInfo::getGOT(DAG.getMachineFunction()));
      } else {
        // Neither execute-only nor PIC: load the address from a constant pool.
        unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
        ARMConstantPoolValue *CPV = ARMConstantPoolSymbol::Create(
            *DAG.getContext(), Sym, ARMPCLabelIndex, 0);
 
        // Get the address of the callee into a register
        SDValue Addr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
        Addr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Addr);
        Callee = DAG.getLoad(
            PtrVt, dl, DAG.getEntryNode(), Addr,
            MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
      }
    }
  } else if (isa<GlobalAddressSDNode>(Callee)) {
    if (!PreferIndirect) {
      isDirect = true;
      bool isDef = GVal->isStrongDefinitionForLinker();
 
      // ARM call to a local ARM function is predicable.
      isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
      // tBX takes a register source operand.
      if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
        assert(TT.isOSBinFormatMachO() && "WrapperPIC use on non-MachO?");
        Callee = DAG.getNode(
            ARMISD::WrapperPIC, dl, PtrVt,
            DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, ARMII::MO_NONLAZY));
        Callee = DAG.getLoad(
            PtrVt, dl, DAG.getEntryNode(), Callee,
            MachinePointerInfo::getGOT(DAG.getMachineFunction()), MaybeAlign(),
            MachineMemOperand::MODereferenceable |
                MachineMemOperand::MOInvariant);
      } else if (Subtarget->isTargetCOFF()) {
        assert(Subtarget->isTargetWindows() &&
               "Windows is the only supported COFF target");
        unsigned TargetFlags = ARMII::MO_NO_FLAG;
        if (GVal->hasDLLImportStorageClass())
          TargetFlags = ARMII::MO_DLLIMPORT;
        else if (!TM.shouldAssumeDSOLocal(GVal))
          TargetFlags = ARMII::MO_COFFSTUB;
        Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, /*offset=*/0,
                                            TargetFlags);
        if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
          Callee =
              DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
                          DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
                          MachinePointerInfo::getGOT(DAG.getMachineFunction()));
      } else {
        Callee = DAG.getTargetGlobalAddress(GVal, dl, PtrVt, 0, 0);
      }
    }
  } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
    isDirect = true;
    // tBX takes a register source operand.
    const char *Sym = S->getSymbol();
    if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
      unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
      ARMConstantPoolValue *CPV =
        ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
                                      ARMPCLabelIndex, 4);
      SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
      CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
      Callee = DAG.getLoad(
          PtrVt, dl, DAG.getEntryNode(), CPAddr,
          MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
      SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
      Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
    } else {
      Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0);
    }
  }
 
  if (isCmseNSCall) {
    assert(!isARMFunc && !isDirect &&
           "Cannot handle call to ARM function or direct call");
    if (NumBytes > 0) {
      DAG.getContext()->diagnose(
          DiagnosticInfoUnsupported(DAG.getMachineFunction().getFunction(),
                                    "call to non-secure function would require "
                                    "passing arguments on stack",
                                    dl.getDebugLoc()));
    }
    if (isStructRet) {
      DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
          DAG.getMachineFunction().getFunction(),
          "call to non-secure function would return value through pointer",
          dl.getDebugLoc()));
    }
  }
 
  // FIXME: handle tail calls differently.
  unsigned CallOpc;
  if (Subtarget->isThumb()) {
    if (GuardWithBTI)
      CallOpc = ARMISD::t2CALL_BTI;
    else if (isCmseNSCall)
      CallOpc = ARMISD::tSECALL;
    else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
      CallOpc = ARMISD::CALL_NOLINK;
    else
      CallOpc = ARMISD::CALL;
  } else {
    if (!isDirect && !Subtarget->hasV5TOps())
      CallOpc = ARMISD::CALL_NOLINK;
    else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
             // Emit regular call when code size is the priority
             !Subtarget->hasMinSize())
      // "mov lr, pc; b _foo" to avoid confusing the RSP
      CallOpc = ARMISD::CALL_NOLINK;
    else
      CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
  }
 
  // We don't usually want to end the call-sequence here because we would tidy
  // the frame up *after* the call, however in the ABI-changing tail-call case
  // we've carefully laid out the parameters so that when sp is reset they'll be
  // in the correct location.
  if (isTailCall && !isSibCall) {
    Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, dl);
    InGlue = Chain.getValue(1);
  }
 
  std::vector<SDValue> Ops;
  Ops.push_back(Chain);
  Ops.push_back(Callee);
 
  if (isTailCall) {
    Ops.push_back(DAG.getSignedTargetConstant(SPDiff, dl, MVT::i32));
  }
 
  // Add argument registers to the end of the list so that they are known live
  // into the call.
  for (const auto &[Reg, N] : RegsToPass)
    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
  // Add a register mask operand representing the call-preserved registers.
  const uint32_t *Mask;
  const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
  if (isThisReturn) {
    // For 'this' returns, use the R0-preserving mask if applicable
    Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
    if (!Mask) {
      // Set isThisReturn to false if the calling convention is not one that
      // allows 'returned' to be modeled in this way, so LowerCallResult does
      // not try to pass 'this' straight through
      isThisReturn = false;
      Mask = ARI->getCallPreservedMask(MF, CallConv);
    }
  } else
    Mask = ARI->getCallPreservedMask(MF, CallConv);
 
  assert(Mask && "Missing call preserved mask for calling convention");
  Ops.push_back(DAG.getRegisterMask(Mask));
 
  if (InGlue.getNode())
    Ops.push_back(InGlue);
 
  if (isTailCall) {
    MF.getFrameInfo().setHasTailCall();
    SDValue Ret = DAG.getNode(ARMISD::TC_RETURN, dl, MVT::Other, Ops);
    if (CLI.CFIType)
      Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
    DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
    DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
    return Ret;
  }
 
  // Returns a chain and a flag for retval copy to use.
  Chain = DAG.getNode(CallOpc, dl, {MVT::Other, MVT::Glue}, Ops);
  if (CLI.CFIType)
    Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
  DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
  InGlue = Chain.getValue(1);
  DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
 
  // If we're guaranteeing tail-calls will be honoured, the callee must
  // pop its own argument stack on return. But this call is *not* a tail call so
  // we need to undo that after it returns to restore the status-quo.
  bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
  uint64_t CalleePopBytes =
      canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1U;
 
  Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
  if (!Ins.empty())
    InGlue = Chain.getValue(1);
 
  // Handle result values, copying them out of physregs into vregs that we
  // return.
  return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
                         InVals, isThisReturn,
                         isThisReturn ? OutVals[0] : SDValue(), isCmseNSCall);
}
 
/// HandleByVal - Every parameter *after* a byval parameter is passed
/// on the stack.  Remember the next parameter register to allocate,
/// and then confiscate the rest of the parameter registers to insure
/// this.
void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
                                    Align Alignment) const {
  // Byval (as with any stack) slots are always at least 4 byte aligned.
  Alignment = std::max(Alignment, Align(4));
 
  MCRegister Reg = State->AllocateReg(GPRArgRegs);
  if (!Reg)
    return;
 
  unsigned AlignInRegs = Alignment.value() / 4;
  unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
  for (unsigned i = 0; i < Waste; ++i)
    Reg = State->AllocateReg(GPRArgRegs);
 
  if (!Reg)
    return;
 
  unsigned Excess = 4 * (ARM::R4 - Reg);
 
  // Special case when NSAA != SP and parameter size greater than size of
  // all remained GPR regs. In that case we can't split parameter, we must
  // send it to stack. We also must set NCRN to R4, so waste all
  // remained registers.
  const unsigned NSAAOffset = State->getStackSize();
  if (NSAAOffset != 0 && Size > Excess) {
    while (State->AllocateReg(GPRArgRegs))
      ;
    return;
  }
 
  // First register for byval parameter is the first register that wasn't
  // allocated before this method call, so it would be "reg".
  // If parameter is small enough to be saved in range [reg, r4), then
  // the end (first after last) register would be reg + param-size-in-regs,
  // else parameter would be splitted between registers and stack,
  // end register would be r4 in this case.
  unsigned ByValRegBegin = Reg;
  unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4);
  State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd);
  // Note, first register is allocated in the beginning of function already,
  // allocate remained amount of registers we need.
  for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
    State->AllocateReg(GPRArgRegs);
  // A byval parameter that is split between registers and memory needs its
  // size truncated here.
  // In the case where the entire structure fits in registers, we set the
  // size in memory to zero.
  Size = std::max<int>(Size - Excess, 0);
}
 
/// IsEligibleForTailCallOptimization - Check whether the call is eligible
/// for tail call optimization. Targets which want to do tail call
/// optimization should implement this function. Note that this function also
/// processes musttail calls, so when this function returns false on a valid
/// musttail call, a fatal backend error occurs.
bool ARMTargetLowering::IsEligibleForTailCallOptimization(
    TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,
    SmallVectorImpl<CCValAssign> &ArgLocs, const bool isIndirect) const {
  CallingConv::ID CalleeCC = CLI.CallConv;
  SDValue Callee = CLI.Callee;
  bool isVarArg = CLI.IsVarArg;
  const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
  const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
  const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
  const SelectionDAG &DAG = CLI.DAG;
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &CallerF = MF.getFunction();
  CallingConv::ID CallerCC = CallerF.getCallingConv();
 
  assert(Subtarget->supportsTailCall());
 
  // Indirect tail-calls require a register to hold the target address. That
  // register must be:
  // * Allocatable (i.e. r0-r7 if the target is Thumb1).
  // * Not callee-saved, so must be one of r0-r3 or r12.
  // * Not used to hold an argument to the tail-called function, which might be
  //   in r0-r3.
  // * Not used to hold the return address authentication code, which is in r12
  //   if enabled.
  // Sometimes, no register matches all of these conditions, so we can't do a
  // tail-call.
  if (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect) {
    SmallSet<MCPhysReg, 5> AddressRegisters = {ARM::R0, ARM::R1, ARM::R2,
                                               ARM::R3};
    if (!(Subtarget->isThumb1Only() ||
          MF.getInfo<ARMFunctionInfo>()->shouldSignReturnAddress(true)))
      AddressRegisters.insert(ARM::R12);
    for (const CCValAssign &AL : ArgLocs)
      if (AL.isRegLoc())
        AddressRegisters.erase(AL.getLocReg());
    if (AddressRegisters.empty()) {
      LLVM_DEBUG(dbgs() << "false (no reg to hold function pointer)\n");
      return false;
    }
  }
 
  // Look for obvious safe cases to perform tail call optimization that do not
  // require ABI changes. This is what gcc calls sibcall.
 
  // Exception-handling functions need a special set of instructions to indicate
  // a return to the hardware. Tail-calling another function would probably
  // break this.
  if (CallerF.hasFnAttribute("interrupt")) {
    LLVM_DEBUG(dbgs() << "false (interrupt attribute)\n");
    return false;
  }
 
  if (canGuaranteeTCO(CalleeCC,
                      getTargetMachine().Options.GuaranteedTailCallOpt)) {
    LLVM_DEBUG(dbgs() << (CalleeCC == CallerCC ? "true" : "false")
                      << " (guaranteed tail-call CC)\n");
    return CalleeCC == CallerCC;
  }
 
  // Also avoid sibcall optimization if either caller or callee uses struct
  // return semantics.
  bool isCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
  bool isCallerStructRet = MF.getFunction().hasStructRetAttr();
  if (isCalleeStructRet != isCallerStructRet) {
    LLVM_DEBUG(dbgs() << "false (struct-ret)\n");
    return false;
  }
 
  // Externally-defined functions with weak linkage should not be
  // tail-called on ARM when the OS does not support dynamic
  // pre-emption of symbols, as the AAELF spec requires normal calls
  // to undefined weak functions to be replaced with a NOP or jump to the
  // next instruction. The behaviour of branch instructions in this
  // situation (as used for tail calls) is implementation-defined, so we
  // cannot rely on the linker replacing the tail call with a return.
  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
    const GlobalValue *GV = G->getGlobal();
    const Triple &TT = getTargetMachine().getTargetTriple();
    if (GV->hasExternalWeakLinkage() &&
        (!TT.isOSWindows() || TT.isOSBinFormatELF() ||
         TT.isOSBinFormatMachO())) {
      LLVM_DEBUG(dbgs() << "false (external weak linkage)\n");
      return false;
    }
  }
 
  // Check that the call results are passed in the same way.
  LLVMContext &C = *DAG.getContext();
  if (!CCState::resultsCompatible(
          getEffectiveCallingConv(CalleeCC, isVarArg),
          getEffectiveCallingConv(CallerCC, CallerF.isVarArg()), MF, C, Ins,
          CCAssignFnForReturn(CalleeCC, isVarArg),
          CCAssignFnForReturn(CallerCC, CallerF.isVarArg()))) {
    LLVM_DEBUG(dbgs() << "false (incompatible results)\n");
    return false;
  }
  // The callee has to preserve all registers the caller needs to preserve.
  const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
  if (CalleeCC != CallerCC) {
    const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved)) {
      LLVM_DEBUG(dbgs() << "false (not all registers preserved)\n");
      return false;
    }
  }
 
  // If Caller's vararg argument has been split between registers and stack, do
  // not perform tail call, since part of the argument is in caller's local
  // frame.
  const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
  if (CLI.IsVarArg && AFI_Caller->getArgRegsSaveSize()) {
    LLVM_DEBUG(dbgs() << "false (arg reg save area)\n");
    return false;
  }
 
  // If the callee takes no arguments then go on to check the results of the
  // call.
  const MachineRegisterInfo &MRI = MF.getRegInfo();
  if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals)) {
    LLVM_DEBUG(dbgs() << "false (parameters in CSRs do not match)\n");
    return false;
  }
 
  // If the stack arguments for this call do not fit into our own save area then
  // the call cannot be made tail.
  if (CCInfo.getStackSize() > AFI_Caller->getArgumentStackSize())
    return false;
 
  LLVM_DEBUG(dbgs() << "true\n");
  return true;
}
 
bool
ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
                                  MachineFunction &MF, bool isVarArg,
                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
                                  LLVMContext &Context, const Type *RetTy) const {
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
  return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
}
 
static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
                                    const SDLoc &DL, SelectionDAG &DAG) {
  const MachineFunction &MF = DAG.getMachineFunction();
  const Function &F = MF.getFunction();
 
  StringRef IntKind = F.getFnAttribute("interrupt").getValueAsString();
 
  // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
  // version of the "preferred return address". These offsets affect the return
  // instruction if this is a return from PL1 without hypervisor extensions.
  //    IRQ/FIQ: +4     "subs pc, lr, #4"
  //    SWI:     0      "subs pc, lr, #0"
  //    ABORT:   +4     "subs pc, lr, #4"
  //    UNDEF:   +4/+2  "subs pc, lr, #0"
  // UNDEF varies depending on where the exception came from ARM or Thumb
  // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
 
  int64_t LROffset;
  if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
      IntKind == "ABORT")
    LROffset = 4;
  else if (IntKind == "SWI" || IntKind == "UNDEF")
    LROffset = 0;
  else
    report_fatal_error("Unsupported interrupt attribute. If present, value "
                       "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
 
  RetOps.insert(RetOps.begin() + 1,
                DAG.getConstant(LROffset, DL, MVT::i32, false));
 
  return DAG.getNode(ARMISD::INTRET_GLUE, DL, MVT::Other, RetOps);
}
 
SDValue
ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                               bool isVarArg,
                               const SmallVectorImpl<ISD::OutputArg> &Outs,
                               const SmallVectorImpl<SDValue> &OutVals,
                               const SDLoc &dl, SelectionDAG &DAG) const {
  // CCValAssign - represent the assignment of the return value to a location.
  SmallVector<CCValAssign, 16> RVLocs;
 
  // CCState - Info about the registers and stack slots.
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
 
  // Analyze outgoing return values.
  CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
 
  SDValue Glue;
  SmallVector<SDValue, 4> RetOps;
  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
  bool isLittleEndian = Subtarget->isLittle();
 
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
  AFI->setReturnRegsCount(RVLocs.size());
 
 // Report error if cmse entry function returns structure through first ptr arg.
  if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) {
    // Note: using an empty SDLoc(), as the first line of the function is a
    // better place to report than the last line.
    DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
        DAG.getMachineFunction().getFunction(),
        "secure entry function would return value through pointer",
        SDLoc().getDebugLoc()));
  }
 
  // Copy the result values into the output registers.
  for (unsigned i = 0, realRVLocIdx = 0;
       i != RVLocs.size();
       ++i, ++realRVLocIdx) {
    CCValAssign &VA = RVLocs[i];
    assert(VA.isRegLoc() && "Can only return in registers!");
 
    SDValue Arg = OutVals[realRVLocIdx];
    bool ReturnF16 = false;
 
    if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
      // Half-precision return values can be returned like this:
      //
      // t11 f16 = fadd ...
      // t12: i16 = bitcast t11
      //   t13: i32 = zero_extend t12
      // t14: f32 = bitcast t13  <~~~~~~~ Arg
      //
      // to avoid code generation for bitcasts, we simply set Arg to the node
      // that produces the f16 value, t11 in this case.
      //
      if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) {
        SDValue ZE = Arg.getOperand(0);
        if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) {
          SDValue BC = ZE.getOperand(0);
          if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) {
            Arg = BC.getOperand(0);
            ReturnF16 = true;
          }
        }
      }
    }
 
    switch (VA.getLocInfo()) {
    default: llvm_unreachable("Unknown loc info!");
    case CCValAssign::Full: break;
    case CCValAssign::BCvt:
      if (!ReturnF16)
        Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
      break;
    }
 
    // Mask f16 arguments if this is a CMSE nonsecure entry.
    auto RetVT = Outs[realRVLocIdx].ArgVT;
    if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
      if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
        Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
      } else {
        auto LocBits = VA.getLocVT().getSizeInBits();
        auto MaskValue = APInt::getLowBitsSet(LocBits, RetVT.getSizeInBits());
        SDValue Mask =
            DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
        Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
        Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
        Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
      }
    }
 
    if (VA.needsCustom() &&
        (VA.getLocVT() == MVT::v2f64 || VA.getLocVT() == MVT::f64)) {
      if (VA.getLocVT() == MVT::v2f64) {
        // Extract the first half and return it in two registers.
        SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
                                   DAG.getConstant(0, dl, MVT::i32));
        SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
                                       DAG.getVTList(MVT::i32, MVT::i32), Half);
 
        Chain =
            DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
                             HalfGPRs.getValue(isLittleEndian ? 0 : 1), Glue);
        Glue = Chain.getValue(1);
        RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
        VA = RVLocs[++i]; // skip ahead to next loc
        Chain =
            DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
                             HalfGPRs.getValue(isLittleEndian ? 1 : 0), Glue);
        Glue = Chain.getValue(1);
        RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
        VA = RVLocs[++i]; // skip ahead to next loc
 
        // Extract the 2nd half and fall through to handle it as an f64 value.
        Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
                          DAG.getConstant(1, dl, MVT::i32));
      }
      // Legalize ret f64 -> ret 2 x i32.  We always have fmrrd if f64 is
      // available.
      SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
                                  DAG.getVTList(MVT::i32, MVT::i32), Arg);
      Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
                               fmrrd.getValue(isLittleEndian ? 0 : 1), Glue);
      Glue = Chain.getValue(1);
      RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
      VA = RVLocs[++i]; // skip ahead to next loc
      Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
                               fmrrd.getValue(isLittleEndian ? 1 : 0), Glue);
    } else
      Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Glue);
 
    // Guarantee that all emitted copies are
    // stuck together, avoiding something bad.
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(
        VA.getLocReg(), ReturnF16 ? Arg.getValueType() : VA.getLocVT()));
  }
  const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
  const MCPhysReg *I =
      TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
  if (I) {
    for (; *I; ++I) {
      if (ARM::GPRRegClass.contains(*I))
        RetOps.push_back(DAG.getRegister(*I, MVT::i32));
      else if (ARM::DPRRegClass.contains(*I))
        RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
      else
        llvm_unreachable("Unexpected register class in CSRsViaCopy!");
    }
  }
 
  // Update chain and glue.
  RetOps[0] = Chain;
  if (Glue.getNode())
    RetOps.push_back(Glue);
 
  // CPUs which aren't M-class use a special sequence to return from
  // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
  // though we use "subs pc, lr, #N").
  //
  // M-class CPUs actually use a normal return sequence with a special
  // (hardware-provided) value in LR, so the normal code path works.
  if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt") &&
      !Subtarget->isMClass()) {
    if (Subtarget->isThumb1Only())
      report_fatal_error("interrupt attribute is not supported in Thumb1");
    return LowerInterruptReturn(RetOps, dl, DAG);
  }
 
  unsigned RetNode =
      AFI->isCmseNSEntryFunction() ? ARMISD::SERET_GLUE : ARMISD::RET_GLUE;
  return DAG.getNode(RetNode, dl, MVT::Other, RetOps);
}
 
bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
  if (N->getNumValues() != 1)
    return false;
  if (!N->hasNUsesOfValue(1, 0))
    return false;
 
  SDValue TCChain = Chain;
  SDNode *Copy = *N->user_begin();
  if (Copy->getOpcode() == ISD::CopyToReg) {
    // If the copy has a glue operand, we conservatively assume it isn't safe to
    // perform a tail call.
    if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
      return false;
    TCChain = Copy->getOperand(0);
  } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
    SDNode *VMov = Copy;
    // f64 returned in a pair of GPRs.
    SmallPtrSet<SDNode*, 2> Copies;
    for (SDNode *U : VMov->users()) {
      if (U->getOpcode() != ISD::CopyToReg)
        return false;
      Copies.insert(U);
    }
    if (Copies.size() > 2)
      return false;
 
    for (SDNode *U : VMov->users()) {
      SDValue UseChain = U->getOperand(0);
      if (Copies.count(UseChain.getNode()))
        // Second CopyToReg
        Copy = U;
      else {
        // We are at the top of this chain.
        // If the copy has a glue operand, we conservatively assume it
        // isn't safe to perform a tail call.
        if (U->getOperand(U->getNumOperands() - 1).getValueType() == MVT::Glue)
          return false;
        // First CopyToReg
        TCChain = UseChain;
      }
    }
  } else if (Copy->getOpcode() == ISD::BITCAST) {
    // f32 returned in a single GPR.
    if (!Copy->hasOneUse())
      return false;
    Copy = *Copy->user_begin();
    if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0))
      return false;
    // If the copy has a glue operand, we conservatively assume it isn't safe to
    // perform a tail call.
    if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
      return false;
    TCChain = Copy->getOperand(0);
  } else {
    return false;
  }
 
  bool HasRet = false;
  for (const SDNode *U : Copy->users()) {
    if (U->getOpcode() != ARMISD::RET_GLUE &&
        U->getOpcode() != ARMISD::INTRET_GLUE)
      return false;
    HasRet = true;
  }
 
  if (!HasRet)
    return false;
 
  Chain = TCChain;
  return true;
}
 
bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
  if (!Subtarget->supportsTailCall())
    return false;
 
  if (!CI->isTailCall())
    return false;
 
  return true;
}
 
// Trying to write a 64 bit value so need to split into two 32 bit values first,
// and pass the lower and high parts through.
static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  SDValue WriteValue = Op->getOperand(2);
 
  // This function is only supposed to be called for i64 type argument.
  assert(WriteValue.getValueType() == MVT::i64
          && "LowerWRITE_REGISTER called for non-i64 type argument.");
 
  SDValue Lo, Hi;
  std::tie(Lo, Hi) = DAG.SplitScalar(WriteValue, DL, MVT::i32, MVT::i32);
  SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
  return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
}
 
// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
// their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
// one of the above mentioned nodes. It has to be wrapped because otherwise
// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
// be used to form addressing mode. These wrapped nodes will be selected
// into MOVi.
SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
                                             SelectionDAG &DAG) const {
  EVT PtrVT = Op.getValueType();
  // FIXME there is no actual debug info here
  SDLoc dl(Op);
  ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
  SDValue Res;
 
  // When generating execute-only code Constant Pools must be promoted to the
  // global data section. It's a bit ugly that we can't share them across basic
  // blocks, but this way we guarantee that execute-only behaves correct with
  // position-independent addressing modes.
  if (Subtarget->genExecuteOnly()) {
    auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
    auto *T = CP->getType();
    auto C = const_cast<Constant*>(CP->getConstVal());
    auto M = DAG.getMachineFunction().getFunction().getParent();
    auto GV = new GlobalVariable(
        *M, T, /*isConstant=*/true, GlobalVariable::InternalLinkage, C,
        Twine(DAG.getDataLayout().getInternalSymbolPrefix()) + "CP" +
            Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
            Twine(AFI->createPICLabelUId()));
    SDValue GA = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
    return LowerGlobalAddress(GA, DAG);
  }
 
  // The 16-bit ADR instruction can only encode offsets that are multiples of 4,
  // so we need to align to at least 4 bytes when we don't have 32-bit ADR.
  Align CPAlign = CP->getAlign();
  if (Subtarget->isThumb1Only())
    CPAlign = std::max(CPAlign, Align(4));
  if (CP->isMachineConstantPoolEntry())
    Res =
        DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CPAlign);
  else
    Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CPAlign);
  return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
}
 
unsigned ARMTargetLowering::getJumpTableEncoding() const {
  // If we don't have a 32-bit pc-relative branch instruction then the jump
  // table consists of block addresses. Usually this is inline, but for
  // execute-only it must be placed out-of-line.
  if (Subtarget->genExecuteOnly() && !Subtarget->hasV8MBaselineOps())
    return MachineJumpTableInfo::EK_BlockAddress;
  return MachineJumpTableInfo::EK_Inline;
}
 
SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
                                             SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
  unsigned ARMPCLabelIndex = 0;
  SDLoc DL(Op);
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
  SDValue CPAddr;
  bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
  if (!IsPositionIndependent) {
    CPAddr = DAG.getTargetConstantPool(BA, PtrVT, Align(4));
  } else {
    unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
    ARMPCLabelIndex = AFI->createPICLabelUId();
    ARMConstantPoolValue *CPV =
      ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex,
                                      ARMCP::CPBlockAddress, PCAdj);
    CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
  }
  CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
  SDValue Result = DAG.getLoad(
      PtrVT, DL, DAG.getEntryNode(), CPAddr,
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
  if (!IsPositionIndependent)
    return Result;
  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
  return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
}
 
/// Convert a TLS address reference into the correct sequence of loads
/// and calls to compute the variable's address for Darwin, and return an
/// SDValue containing the final node.
 
/// Darwin only has one TLS scheme which must be capable of dealing with the
/// fully general situation, in the worst case. This means:
///     + "extern __thread" declaration.
///     + Defined in a possibly unknown dynamic library.
///
/// The general system is that each __thread variable has a [3 x i32] descriptor
/// which contains information used by the runtime to calculate the address. The
/// only part of this the compiler needs to know about is the first word, which
/// contains a function pointer that must be called with the address of the
/// entire descriptor in "r0".
///
/// Since this descriptor may be in a different unit, in general access must
/// proceed along the usual ARM rules. A common sequence to produce is:
///
///     movw rT1, :lower16:_var$non_lazy_ptr
///     movt rT1, :upper16:_var$non_lazy_ptr
///     ldr r0, [rT1]
///     ldr rT2, [r0]
///     blx rT2
///     [...address now in r0...]
SDValue
ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
                                               SelectionDAG &DAG) const {
  assert(getTargetMachine().getTargetTriple().isOSDarwin() &&
         "This function expects a Darwin target");
  SDLoc DL(Op);
 
  // First step is to get the address of the actua global symbol. This is where
  // the TLS descriptor lives.
  SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
 
  // The first entry in the descriptor is a function pointer that we must call
  // to obtain the address of the variable.
  SDValue Chain = DAG.getEntryNode();
  SDValue FuncTLVGet = DAG.getLoad(
      MVT::i32, DL, Chain, DescAddr,
      MachinePointerInfo::getGOT(DAG.getMachineFunction()), Align(4),
      MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable |
          MachineMemOperand::MOInvariant);
  Chain = FuncTLVGet.getValue(1);
 
  MachineFunction &F = DAG.getMachineFunction();
  MachineFrameInfo &MFI = F.getFrameInfo();
  MFI.setAdjustsStack(true);
 
  // TLS calls preserve all registers except those that absolutely must be
  // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
  // silly).
  auto TRI =
      getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo();
  auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
  const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction());
 
  // Finally, we can make the call. This is just a degenerate version of a
  // normal AArch64 call node: r0 takes the address of the descriptor, and
  // returns the address of the variable in this thread.
  Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue());
  Chain =
      DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
                  Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32),
                  DAG.getRegisterMask(Mask), Chain.getValue(1));
  return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1));
}
 
SDValue
ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
                                                SelectionDAG &DAG) const {
  assert(getTargetMachine().getTargetTriple().isOSWindows() &&
         "Windows specific TLS lowering");
 
  SDValue Chain = DAG.getEntryNode();
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  SDLoc DL(Op);
 
  // Load the current TEB (thread environment block)
  SDValue Ops[] = {Chain,
                   DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
                   DAG.getTargetConstant(15, DL, MVT::i32),
                   DAG.getTargetConstant(0, DL, MVT::i32),
                   DAG.getTargetConstant(13, DL, MVT::i32),
                   DAG.getTargetConstant(0, DL, MVT::i32),
                   DAG.getTargetConstant(2, DL, MVT::i32)};
  SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
                                   DAG.getVTList(MVT::i32, MVT::Other), Ops);
 
  SDValue TEB = CurrentTEB.getValue(0);
  Chain = CurrentTEB.getValue(1);
 
  // Load the ThreadLocalStoragePointer from the TEB
  // A pointer to the TLS array is located at offset 0x2c from the TEB.
  SDValue TLSArray =
      DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL));
  TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
 
  // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
  // offset into the TLSArray.
 
  // Load the TLS index from the C runtime
  SDValue TLSIndex =
      DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG);
  TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex);
  TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo());
 
  SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
                              DAG.getConstant(2, DL, MVT::i32));
  SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
                            DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
                            MachinePointerInfo());
 
  // Get the offset of the start of the .tls section (section base)
  const auto *GA = cast<GlobalAddressSDNode>(Op);
  auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL);
  SDValue Offset = DAG.getLoad(
      PtrVT, DL, Chain,
      DAG.getNode(ARMISD::Wrapper, DL, MVT::i32,
                  DAG.getTargetConstantPool(CPV, PtrVT, Align(4))),
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
 
  return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset);
}
 
// Lower ISD::GlobalTLSAddress using the "general dynamic" model
SDValue
ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
                                                 SelectionDAG &DAG) const {
  SDLoc dl(GA);
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
  unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
  ARMConstantPoolValue *CPV =
    ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
                                    ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
  SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
  Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
  Argument = DAG.getLoad(
      PtrVT, dl, DAG.getEntryNode(), Argument,
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
  SDValue Chain = Argument.getValue(1);
 
  SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
  Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
 
  // call __tls_get_addr.
  ArgListTy Args;
  Args.emplace_back(Argument, Type::getInt32Ty(*DAG.getContext()));
 
  // FIXME: is there useful debug info available here?
  TargetLowering::CallLoweringInfo CLI(DAG);
  CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
      CallingConv::C, Type::getInt32Ty(*DAG.getContext()),
      DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args));
 
  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
  return CallResult.first;
}
 
// Lower ISD::GlobalTLSAddress using the "initial exec" or
// "local exec" model.
SDValue
ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
                                        SelectionDAG &DAG,
                                        TLSModel::Model model) const {
  const GlobalValue *GV = GA->getGlobal();
  SDLoc dl(GA);
  SDValue Offset;
  SDValue Chain = DAG.getEntryNode();
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  // Get the Thread Pointer
  SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
 
  if (model == TLSModel::InitialExec) {
    MachineFunction &MF = DAG.getMachineFunction();
    ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
    // Initial exec model.
    unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
    ARMConstantPoolValue *CPV =
      ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
                                      ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
                                      true);
    Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
    Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
    Offset = DAG.getLoad(
        PtrVT, dl, Chain, Offset,
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
    Chain = Offset.getValue(1);
 
    SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
    Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
 
    Offset = DAG.getLoad(
        PtrVT, dl, Chain, Offset,
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
  } else {
    // local exec model
    assert(model == TLSModel::LocalExec);
    ARMConstantPoolValue *CPV =
      ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF);
    Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
    Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
    Offset = DAG.getLoad(
        PtrVT, dl, Chain, Offset,
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
  }
 
  // The address of the thread local variable is the add of the thread
  // pointer with the offset of the variable.
  return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
}
 
SDValue
ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
  GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
  if (DAG.getTarget().useEmulatedTLS())
    return LowerToTLSEmulatedModel(GA, DAG);
 
  const Triple &TT = getTargetMachine().getTargetTriple();
  if (TT.isOSDarwin())
    return LowerGlobalTLSAddressDarwin(Op, DAG);
 
  if (TT.isOSWindows())
    return LowerGlobalTLSAddressWindows(Op, DAG);
 
  // TODO: implement the "local dynamic" model
  assert(TT.isOSBinFormatELF() && "Only ELF implemented here");
  TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal());
 
  switch (model) {
    case TLSModel::GeneralDynamic:
    case TLSModel::LocalDynamic:
      return LowerToTLSGeneralDynamicModel(GA, DAG);
    case TLSModel::InitialExec:
    case TLSModel::LocalExec:
      return LowerToTLSExecModels(GA, DAG, model);
  }
  llvm_unreachable("bogus TLS model");
}
 
/// Return true if all users of V are within function F, looking through
/// ConstantExprs.
static bool allUsersAreInFunction(const Value *V, const Function *F) {
  SmallVector<const User*,4> Worklist(V->users());
  while (!Worklist.empty()) {
    auto *U = Worklist.pop_back_val();
    if (isa<ConstantExpr>(U)) {
      append_range(Worklist, U->users());
      continue;
    }
 
    auto *I = dyn_cast<Instruction>(U);
    if (!I || I->getParent()->getParent() != F)
      return false;
  }
  return true;
}
 
static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
                                     const GlobalValue *GV, SelectionDAG &DAG,
                                     EVT PtrVT, const SDLoc &dl) {
  // If we're creating a pool entry for a constant global with unnamed address,
  // and the global is small enough, we can emit it inline into the constant pool
  // to save ourselves an indirection.
  //
  // This is a win if the constant is only used in one function (so it doesn't
  // need to be duplicated) or duplicating the constant wouldn't increase code
  // size (implying the constant is no larger than 4 bytes).
  const Function &F = DAG.getMachineFunction().getFunction();
 
  // We rely on this decision to inline being idempotent and unrelated to the
  // use-site. We know that if we inline a variable at one use site, we'll
  // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
  // doesn't know about this optimization, so bail out if it's enabled else
  // we could decide to inline here (and thus never emit the GV) but require
  // the GV from fast-isel generated code.
  if (!EnableConstpoolPromotion ||
      DAG.getMachineFunction().getTarget().Options.EnableFastISel)
      return SDValue();
 
  auto *GVar = dyn_cast<GlobalVariable>(GV);
  if (!GVar || !GVar->hasInitializer() ||
      !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
      !GVar->hasLocalLinkage())
    return SDValue();
 
  // If we inline a value that contains relocations, we move the relocations
  // from .data to .text. This is not allowed in position-independent code.
  auto *Init = GVar->getInitializer();
  if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) &&
      Init->needsDynamicRelocation())
    return SDValue();
 
  // The constant islands pass can only really deal with alignment requests
  // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
  // any type wanting greater alignment requirements than 4 bytes. We also
  // can only promote constants that are multiples of 4 bytes in size or
  // are paddable to a multiple of 4. Currently we only try and pad constants
  // that are strings for simplicity.
  auto *CDAInit = dyn_cast<ConstantDataArray>(Init);
  unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
  Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GVar);
  unsigned RequiredPadding = 4 - (Size % 4);
  bool PaddingPossible =
    RequiredPadding == 4 || (CDAInit && CDAInit->isString());
  if (!PaddingPossible || PrefAlign > 4 || Size > ConstpoolPromotionMaxSize ||
      Size == 0)
    return SDValue();
 
  unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
 
  // We can't bloat the constant pool too much, else the ConstantIslands pass
  // may fail to converge. If we haven't promoted this global yet (it may have
  // multiple uses), and promoting it would increase the constant pool size (Sz
  // > 4), ensure we have space to do so up to MaxTotal.
  if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4)
    if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
        ConstpoolPromotionMaxTotal)
      return SDValue();
 
  // This is only valid if all users are in a single function; we can't clone
  // the constant in general. The LLVM IR unnamed_addr allows merging
  // constants, but not cloning them.
  //
  // We could potentially allow cloning if we could prove all uses of the
  // constant in the current function don't care about the address, like
  // printf format strings. But that isn't implemented for now.
  if (!allUsersAreInFunction(GVar, &F))
    return SDValue();
 
  // We're going to inline this global. Pad it out if needed.
  if (RequiredPadding != 4) {
    StringRef S = CDAInit->getAsString();
 
    SmallVector<uint8_t,16> V(S.size());
    std::copy(S.bytes_begin(), S.bytes_end(), V.begin());
    while (RequiredPadding--)
      V.push_back(0);
    Init = ConstantDataArray::get(*DAG.getContext(), V);
  }
 
  auto CPVal = ARMConstantPoolConstant::Create(GVar, Init);
  SDValue CPAddr = DAG.getTargetConstantPool(CPVal, PtrVT, Align(4));
  if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) {
    AFI->markGlobalAsPromotedToConstantPool(GVar);
    AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() +
                                      PaddedSize - 4);
  }
  ++NumConstpoolPromoted;
  return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
}
 
bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const {
  if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
    if (!(GV = GA->getAliaseeObject()))
      return false;
  if (const auto *V = dyn_cast<GlobalVariable>(GV))
    return V->isConstant();
  return isa<Function>(GV);
}
 
SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
                                              SelectionDAG &DAG) const {
  switch (Subtarget->getTargetTriple().getObjectFormat()) {
  default: llvm_unreachable("unknown object format");
  case Triple::COFF:
    return LowerGlobalAddressWindows(Op, DAG);
  case Triple::ELF:
    return LowerGlobalAddressELF(Op, DAG);
  case Triple::MachO:
    return LowerGlobalAddressDarwin(Op, DAG);
  }
}
 
SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
                                                 SelectionDAG &DAG) const {
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  SDLoc dl(Op);
  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
  bool IsRO = isReadOnly(GV);
 
  // promoteToConstantPool only if not generating XO text section
  if (GV->isDSOLocal() && !Subtarget->genExecuteOnly())
    if (SDValue V = promoteToConstantPool(this, GV, DAG, PtrVT, dl))
      return V;
 
  if (isPositionIndependent()) {
    SDValue G = DAG.getTargetGlobalAddress(
        GV, dl, PtrVT, 0, GV->isDSOLocal() ? 0 : ARMII::MO_GOT);
    SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
    if (!GV->isDSOLocal())
      Result =
          DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
                      MachinePointerInfo::getGOT(DAG.getMachineFunction()));
    return Result;
  } else if (Subtarget->isROPI() && IsRO) {
    // PC-relative.
    SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
    SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
    return Result;
  } else if (Subtarget->isRWPI() && !IsRO) {
    // SB-relative.
    SDValue RelAddr;
    if (Subtarget->useMovt()) {
      ++NumMovwMovt;
      SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
      RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G);
    } else { // use literal pool for address constant
      ARMConstantPoolValue *CPV =
        ARMConstantPoolConstant::Create(GV, ARMCP::SBREL);
      SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
      CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
      RelAddr = DAG.getLoad(
          PtrVT, dl, DAG.getEntryNode(), CPAddr,
          MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
    }
    SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
    SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr);
    return Result;
  }
 
  // If we have T2 ops, we can materialize the address directly via movt/movw
  // pair. This is always cheaper. If need to generate Execute Only code, and we
  // only have Thumb1 available, we can't use a constant pool and are forced to
  // use immediate relocations.
  if (Subtarget->useMovt() || Subtarget->genExecuteOnly()) {
    if (Subtarget->useMovt())
      ++NumMovwMovt;
    // FIXME: Once remat is capable of dealing with instructions with register
    // operands, expand this into two nodes.
    return DAG.getNode(ARMISD::Wrapper, dl, PtrVT,
                       DAG.getTargetGlobalAddress(GV, dl, PtrVT));
  } else {
    SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, Align(4));
    CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
    return DAG.getLoad(
        PtrVT, dl, DAG.getEntryNode(), CPAddr,
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
  }
}
 
SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
                                                    SelectionDAG &DAG) const {
  assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
         "ROPI/RWPI not currently supported for Darwin");
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  SDLoc dl(Op);
  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
 
  if (Subtarget->useMovt())
    ++NumMovwMovt;
 
  // FIXME: Once remat is capable of dealing with instructions with register
  // operands, expand this into multiple nodes
  unsigned Wrapper =
      isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
 
  SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
  SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
 
  if (Subtarget->isGVIndirectSymbol(GV))
    Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
                         MachinePointerInfo::getGOT(DAG.getMachineFunction()));
  return Result;
}
 
SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
                                                     SelectionDAG &DAG) const {
  assert(getTargetMachine().getTargetTriple().isOSWindows() &&
         "non-Windows COFF is not supported");
  assert(Subtarget->useMovt() &&
         "Windows on ARM expects to use movw/movt");
  assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
         "ROPI/RWPI not currently supported for Windows");
 
  const TargetMachine &TM = getTargetMachine();
  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
  ARMII::TOF TargetFlags = ARMII::MO_NO_FLAG;
  if (GV->hasDLLImportStorageClass())
    TargetFlags = ARMII::MO_DLLIMPORT;
  else if (!TM.shouldAssumeDSOLocal(GV))
    TargetFlags = ARMII::MO_COFFSTUB;
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  SDValue Result;
  SDLoc DL(Op);
 
  ++NumMovwMovt;
 
  // FIXME: Once remat is capable of dealing with instructions with register
  // operands, expand this into two nodes.
  Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT,
                       DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*offset=*/0,
                                                  TargetFlags));
  if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
    Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
                         MachinePointerInfo::getGOT(DAG.getMachineFunction()));
  return Result;
}
 
SDValue
ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
  SDLoc dl(Op);
  SDValue Val = DAG.getConstant(0, dl, MVT::i32);
  return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
                     DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
                     Op.getOperand(1), Val);
}
 
SDValue
ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
  SDLoc dl(Op);
  return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
                     Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
}
 
SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
                                                      SelectionDAG &DAG) const {
  SDLoc dl(Op);
  return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
                     Op.getOperand(0));
}
 
SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
    SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
  unsigned IntNo =
      Op.getConstantOperandVal(Op.getOperand(0).getValueType() == MVT::Other);
  switch (IntNo) {
    default:
      return SDValue();  // Don't custom lower most intrinsics.
    case Intrinsic::arm_gnu_eabi_mcount: {
      MachineFunction &MF = DAG.getMachineFunction();
      EVT PtrVT = getPointerTy(DAG.getDataLayout());
      SDLoc dl(Op);
      SDValue Chain = Op.getOperand(0);
      // call "\01__gnu_mcount_nc"
      const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
      const uint32_t *Mask =
          ARI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
      assert(Mask && "Missing call preserved mask for calling convention");
      // Mark LR an implicit live-in.
      Register Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
      SDValue ReturnAddress =
          DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, PtrVT);
      constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
      SDValue Callee =
          DAG.getTargetExternalSymbol("\01__gnu_mcount_nc", PtrVT, 0);
      SDValue RegisterMask = DAG.getRegisterMask(Mask);
      if (Subtarget->isThumb())
        return SDValue(
            DAG.getMachineNode(
                ARM::tBL_PUSHLR, dl, ResultTys,
                {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
                 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
            0);
      return SDValue(
          DAG.getMachineNode(ARM::BL_PUSHLR, dl, ResultTys,
                             {ReturnAddress, Callee, RegisterMask, Chain}),
          0);
    }
  }
}
 
SDValue
ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
                                          const ARMSubtarget *Subtarget) const {
  unsigned IntNo = Op.getConstantOperandVal(0);
  SDLoc dl(Op);
  switch (IntNo) {
  default: return SDValue();    // Don't custom lower most intrinsics.
  case Intrinsic::localaddress: {
    const MachineFunction &MF = DAG.getMachineFunction();
    const auto *RegInfo = Subtarget->getRegisterInfo();
    unsigned Reg = RegInfo->getLocalAddressRegister(MF);
    return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
                              Op.getSimpleValueType());
  }
  case Intrinsic::eh_recoverfp: {
    SDValue FnOp = Op.getOperand(1);
    GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
    auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
    if (!Fn)
      report_fatal_error(
          "llvm.eh.recoverfp must take a function as the first argument");
    const auto *RegInfo = Subtarget->getRegisterInfo();
    Register BaseReg = RegInfo->getBaseRegister();
    MachineFunction &MF = DAG.getMachineFunction();
    MachineBasicBlock &MBB = *MF.begin();
    if (!MBB.isLiveIn(BaseReg))
      MBB.addLiveIn(BaseReg);
    EVT PtrVT = getPointerTy(DAG.getDataLayout());
    return DAG.getCopyFromReg(DAG.getEntryNode(), dl, BaseReg, PtrVT);
  }
  case Intrinsic::thread_pointer: {
    EVT PtrVT = getPointerTy(DAG.getDataLayout());
    return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
  }
  case Intrinsic::arm_cls: {
    // Note: arm_cls and arm_cls64 intrinsics are expanded directly here
    // in LowerINTRINSIC_WO_CHAIN since there's no native scalar CLS
    // instruction.
    const SDValue &Operand = Op.getOperand(1);
    const EVT VTy = Op.getValueType();
    return DAG.getNode(ISD::CTLS, dl, VTy, Operand);
  }
  case Intrinsic::arm_cls64: {
    // arm_cls64 returns i32 but takes i64 input.
    // Use ISD::CTLS for i64 and truncate the result.
    SDValue CTLS64 = DAG.getNode(ISD::CTLS, dl, MVT::i64, Op.getOperand(1));
    return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, CTLS64);
  }
  case Intrinsic::arm_neon_vcls:
  case Intrinsic::arm_mve_vcls: {
    // Lower vector CLS intrinsics to ISD::CTLS.
    // Vector CTLS is Legal when NEON/MVE is available (set elsewhere).
    const EVT VTy = Op.getValueType();
    return DAG.getNode(ISD::CTLS, dl, VTy, Op.getOperand(1));
  }
  case Intrinsic::eh_sjlj_lsda: {
    MachineFunction &MF = DAG.getMachineFunction();
    ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
    unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
    EVT PtrVT = getPointerTy(DAG.getDataLayout());
    SDValue CPAddr;
    bool IsPositionIndependent = isPositionIndependent();
    unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
    ARMConstantPoolValue *CPV =
      ARMConstantPoolConstant::Create(&MF.getFunction(), ARMPCLabelIndex,
                                      ARMCP::CPLSDA, PCAdj);
    CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
    CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
    SDValue Result = DAG.getLoad(
        PtrVT, dl, DAG.getEntryNode(), CPAddr,
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
 
    if (IsPositionIndependent) {
      SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
      Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
    }
    return Result;
  }
  case Intrinsic::arm_neon_vabs:
    return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::arm_neon_vabds:
    if (Op.getValueType().isInteger())
      return DAG.getNode(ISD::ABDS, SDLoc(Op), Op.getValueType(),
                         Op.getOperand(1), Op.getOperand(2));
    return SDValue();
  case Intrinsic::arm_neon_vabdu:
    return DAG.getNode(ISD::ABDU, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::arm_neon_vmulls:
  case Intrinsic::arm_neon_vmullu: {
    unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
      ? ARMISD::VMULLs : ARMISD::VMULLu;
    return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::arm_neon_vminnm:
  case Intrinsic::arm_neon_vmaxnm: {
    unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
      ? ISD::FMINNUM : ISD::FMAXNUM;
    return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::arm_neon_vminu:
  case Intrinsic::arm_neon_vmaxu: {
    if (Op.getValueType().isFloatingPoint())
      return SDValue();
    unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
      ? ISD::UMIN : ISD::UMAX;
    return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
                         Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::arm_neon_vmins:
  case Intrinsic::arm_neon_vmaxs: {
    // v{min,max}s is overloaded between signed integers and floats.
    if (!Op.getValueType().isFloatingPoint()) {
      unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
        ? ISD::SMIN : ISD::SMAX;
      return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
                         Op.getOperand(1), Op.getOperand(2));
    }
    unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
      ? ISD::FMINIMUM : ISD::FMAXIMUM;
    return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  }
  case Intrinsic::arm_neon_vtbl1:
    return DAG.getNode(ARMISD::VTBL1, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2));
  case Intrinsic::arm_neon_vtbl2:
    return DAG.getNode(ARMISD::VTBL2, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::arm_mve_pred_i2v:
  case Intrinsic::arm_mve_pred_v2i:
    return DAG.getNode(ARMISD::PREDICATE_CAST, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::arm_mve_vreinterpretq:
    return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(Op), Op.getValueType(),
                       Op.getOperand(1));
  case Intrinsic::arm_mve_lsll:
    return DAG.getNode(ARMISD::LSLL, SDLoc(Op), Op->getVTList(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::arm_mve_asrl:
    return DAG.getNode(ARMISD::ASRL, SDLoc(Op), Op->getVTList(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::arm_mve_vsli:
    return DAG.getNode(ARMISD::VSLIIMM, SDLoc(Op), Op->getVTList(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  case Intrinsic::arm_mve_vsri:
    return DAG.getNode(ARMISD::VSRIIMM, SDLoc(Op), Op->getVTList(),
                       Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
  }
}
 
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
                                 const ARMSubtarget *Subtarget) {
  SDLoc dl(Op);
  auto SSID = static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
  if (SSID == SyncScope::SingleThread)
    return Op;
 
  if (!Subtarget->hasDataBarrier()) {
    // Some ARMv6 cpus can support data barriers with an mcr instruction.
    // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
    // here.
    assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
           "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
    return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
                       DAG.getConstant(0, dl, MVT::i32));
  }
 
  AtomicOrdering Ord =
      static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
  ARM_MB::MemBOpt Domain = ARM_MB::ISH;
  if (Subtarget->isMClass()) {
    // Only a full system barrier exists in the M-class architectures.
    Domain = ARM_MB::SY;
  } else if (Subtarget->preferISHSTBarriers() &&
             Ord == AtomicOrdering::Release) {
    // Swift happens to implement ISHST barriers in a way that's compatible with
    // Release semantics but weaker than ISH so we'd be fools not to use
    // it. Beware: other processors probably don't!
    Domain = ARM_MB::ISHST;
  }
 
  return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
                     DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
                     DAG.getConstant(Domain, dl, MVT::i32));
}
 
static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
                             const ARMSubtarget *Subtarget) {
  // ARM pre v5TE and Thumb1 does not have preload instructions.
  if (!(Subtarget->isThumb2() ||
        (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
    // Just preserve the chain.
    return Op.getOperand(0);
 
  SDLoc dl(Op);
  unsigned isRead = ~Op.getConstantOperandVal(2) & 1;
  if (!isRead &&
      (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
    // ARMv7 with MP extension has PLDW.
    return Op.getOperand(0);
 
  unsigned isData = Op.getConstantOperandVal(4);
  if (Subtarget->isThumb()) {
    // Invert the bits.
    isRead = ~isRead & 1;
    isData = ~isData & 1;
  }
 
  return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
                     Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
                     DAG.getConstant(isData, dl, MVT::i32));
}
 
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
 
  // vastart just stores the address of the VarArgsFrameIndex slot into the
  // memory location argument.
  SDLoc dl(Op);
  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
  SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
  return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
                      MachinePointerInfo(SV));
}
 
SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
                                                CCValAssign &NextVA,
                                                SDValue &Root,
                                                SelectionDAG &DAG,
                                                const SDLoc &dl) const {
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
 
  const TargetRegisterClass *RC;
  if (AFI->isThumb1OnlyFunction())
    RC = &ARM::tGPRRegClass;
  else
    RC = &ARM::GPRRegClass;
 
  // Transform the arguments stored in physical registers into virtual ones.
  Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
  SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
 
  SDValue ArgValue2;
  if (NextVA.isMemLoc()) {
    MachineFrameInfo &MFI = MF.getFrameInfo();
    int FI = MFI.CreateFixedObject(4, NextVA.getLocMemOffset(), true);
 
    // Create load node to retrieve arguments from the stack.
    SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
    ArgValue2 = DAG.getLoad(
        MVT::i32, dl, Root, FIN,
        MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
  } else {
    Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
    ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
  }
  if (!Subtarget->isLittle())
    std::swap (ArgValue, ArgValue2);
  return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
}
 
// The remaining GPRs hold either the beginning of variable-argument
// data, or the beginning of an aggregate passed by value (usually
// byval).  Either way, we allocate stack slots adjacent to the data
// provided by our caller, and store the unallocated registers there.
// If this is a variadic function, the va_list pointer will begin with
// these values; otherwise, this reassembles a (byval) structure that
// was split between registers and memory.
// Return: The frame index registers were stored into.
int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
                                      const SDLoc &dl, SDValue &Chain,
                                      const Value *OrigArg,
                                      unsigned InRegsParamRecordIdx,
                                      int ArgOffset, unsigned ArgSize) const {
  // Currently, two use-cases possible:
  // Case #1. Non-var-args function, and we meet first byval parameter.
  //          Setup first unallocated register as first byval register;
  //          eat all remained registers
  //          (these two actions are performed by HandleByVal method).
  //          Then, here, we initialize stack frame with
  //          "store-reg" instructions.
  // Case #2. Var-args function, that doesn't contain byval parameters.
  //          The same: eat all remained unallocated registers,
  //          initialize stack frame.
 
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
  unsigned RBegin, REnd;
  if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
    CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd);
  } else {
    unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
    RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
    REnd = ARM::R4;
  }
 
  if (REnd != RBegin)
    ArgOffset = -4 * (ARM::R4 - RBegin);
 
  auto PtrVT = getPointerTy(DAG.getDataLayout());
  int FrameIndex = MFI.CreateFixedObject(ArgSize, ArgOffset, false);
  SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
 
  SmallVector<SDValue, 4> MemOps;
  const TargetRegisterClass *RC =
      AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
 
  for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
    Register VReg = MF.addLiveIn(Reg, RC);
    SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
    SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
                                 MachinePointerInfo(OrigArg, 4 * i));
    MemOps.push_back(Store);
    FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
  }
 
  if (!MemOps.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
  return FrameIndex;
}
 
// Setup stack frame, the va_list pointer will start from.
void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
                                             const SDLoc &dl, SDValue &Chain,
                                             unsigned ArgOffset,
                                             unsigned TotalArgRegsSaveSize,
                                             bool ForceMutable) const {
  MachineFunction &MF = DAG.getMachineFunction();
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
 
  // Try to store any remaining integer argument regs
  // to their spots on the stack so that they may be loaded by dereferencing
  // the result of va_next.
  // If there is no regs to be stored, just point address after last
  // argument passed via stack.
  int FrameIndex = StoreByValRegs(
      CCInfo, DAG, dl, Chain, nullptr, CCInfo.getInRegsParamsCount(),
      CCInfo.getStackSize(), std::max(4U, TotalArgRegsSaveSize));
  AFI->setVarArgsFrameIndex(FrameIndex);
}
 
bool ARMTargetLowering::splitValueIntoRegisterParts(
    SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
    unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
  EVT ValueVT = Val.getValueType();
  if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
    unsigned ValueBits = ValueVT.getSizeInBits();
    unsigned PartBits = PartVT.getSizeInBits();
    Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val);
    Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val);
    Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
    Parts[0] = Val;
    return true;
  }
  return false;
}
 
SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
    SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
    MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
  if ((ValueVT == MVT::f16 || ValueVT == MVT::bf16) && PartVT == MVT::f32) {
    unsigned ValueBits = ValueVT.getSizeInBits();
    unsigned PartBits = PartVT.getSizeInBits();
    SDValue Val = Parts[0];
 
    Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val);
    Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val);
    Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
    return Val;
  }
  return SDValue();
}
 
SDValue ARMTargetLowering::LowerFormalArguments(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
 
  // Assign locations to all of the incoming arguments.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                 *DAG.getContext());
  CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg));
 
  Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
  unsigned CurArgIdx = 0;
 
  // Initially ArgRegsSaveSize is zero.
  // Then we increase this value each time we meet byval parameter.
  // We also increase this value in case of varargs function.
  AFI->setArgRegsSaveSize(0);
 
  // Calculate the amount of stack space that we need to allocate to store
  // byval and variadic arguments that are passed in registers.
  // We need to know this before we allocate the first byval or variadic
  // argument, as they will be allocated a stack slot below the CFA (Canonical
  // Frame Address, the stack pointer at entry to the function).
  unsigned ArgRegBegin = ARM::R4;
  for (const CCValAssign &VA : ArgLocs) {
    if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
      break;
 
    unsigned Index = VA.getValNo();
    ISD::ArgFlagsTy Flags = Ins[Index].Flags;
    if (!Flags.isByVal())
      continue;
 
    assert(VA.isMemLoc() && "unexpected byval pointer in reg");
    unsigned RBegin, REnd;
    CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd);
    ArgRegBegin = std::min(ArgRegBegin, RBegin);
 
    CCInfo.nextInRegsParam();
  }
  CCInfo.rewindByValRegsInfo();
 
  int lastInsIndex = -1;
  if (isVarArg && MFI.hasVAStart()) {
    unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
    if (RegIdx != std::size(GPRArgRegs))
      ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]);
  }
 
  unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
  AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
  auto PtrVT = getPointerTy(DAG.getDataLayout());
 
  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
    CCValAssign &VA = ArgLocs[i];
    if (Ins[VA.getValNo()].isOrigArg()) {
      std::advance(CurOrigArg,
                   Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
      CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
    }
    // Arguments stored in registers.
    if (VA.isRegLoc()) {
      EVT RegVT = VA.getLocVT();
      SDValue ArgValue;
 
      if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
        // f64 and vector types are split up into multiple registers or
        // combinations of registers and stack slots.
        SDValue ArgValue1 =
            GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
        VA = ArgLocs[++i]; // skip ahead to next loc
        SDValue ArgValue2;
        if (VA.isMemLoc()) {
          int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), true);
          SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
          ArgValue2 = DAG.getLoad(
              MVT::f64, dl, Chain, FIN,
              MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
        } else {
          ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
        }
        ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
        ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
                               ArgValue1, DAG.getIntPtrConstant(0, dl));
        ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
                               ArgValue2, DAG.getIntPtrConstant(1, dl));
      } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
        ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
      } else {
        const TargetRegisterClass *RC;
 
        if (RegVT == MVT::f16 || RegVT == MVT::bf16)
          RC = &ARM::HPRRegClass;
        else if (RegVT == MVT::f32)
          RC = &ARM::SPRRegClass;
        else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
                 RegVT == MVT::v4bf16)
          RC = &ARM::DPRRegClass;
        else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
                 RegVT == MVT::v8bf16)
          RC = &ARM::QPRRegClass;
        else if (RegVT == MVT::i32)
          RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
                                           : &ARM::GPRRegClass;
        else
          llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
 
        // Transform the arguments in physical registers into virtual ones.
        Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
        ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
 
        // If this value is passed in r0 and has the returned attribute (e.g.
        // C++ 'structors), record this fact for later use.
        if (VA.getLocReg() == ARM::R0 && Ins[VA.getValNo()].Flags.isReturned()) {
          AFI->setPreservesR0();
        }
      }
 
      // If this is an 8 or 16-bit value, it is really passed promoted
      // to 32 bits.  Insert an assert[sz]ext to capture this, then
      // truncate to the right size.
      switch (VA.getLocInfo()) {
      default: llvm_unreachable("Unknown loc info!");
      case CCValAssign::Full: break;
      case CCValAssign::BCvt:
        ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
        break;
      }
 
      // f16 arguments have their size extended to 4 bytes and passed as if they
      // had been copied to the LSBs of a 32-bit register.
      // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
      if (VA.needsCustom() &&
          (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
        ArgValue = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), ArgValue);
 
      // On CMSE Entry Functions, formal integer arguments whose bitwidth is
      // less than 32 bits must be sign- or zero-extended in the callee for
      // security reasons. Although the ABI mandates an extension done by the
      // caller, the latter cannot be trusted to follow the rules of the ABI.
      const ISD::InputArg &Arg = Ins[VA.getValNo()];
      if (AFI->isCmseNSEntryFunction() && Arg.ArgVT.isScalarInteger() &&
          RegVT.isScalarInteger() && Arg.ArgVT.bitsLT(MVT::i32))
        ArgValue = handleCMSEValue(ArgValue, Arg, DAG, dl);
 
      InVals.push_back(ArgValue);
    } else { // VA.isRegLoc()
      // Only arguments passed on the stack should make it here.
      assert(VA.isMemLoc());
      assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
 
      int index = VA.getValNo();
 
      // Some Ins[] entries become multiple ArgLoc[] entries.
      // Process them only once.
      if (index != lastInsIndex)
        {
          ISD::ArgFlagsTy Flags = Ins[index].Flags;
          // FIXME: For now, all byval parameter objects are marked mutable.
          // This can be changed with more analysis.
          // In case of tail call optimization mark all arguments mutable.
          // Since they could be overwritten by lowering of arguments in case of
          // a tail call.
          if (Flags.isByVal()) {
            assert(Ins[index].isOrigArg() &&
                   "Byval arguments cannot be implicit");
            unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
 
            int FrameIndex = StoreByValRegs(
                CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
                VA.getLocMemOffset(), Flags.getByValSize());
            InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
            CCInfo.nextInRegsParam();
          } else if (VA.needsCustom() && (VA.getValVT() == MVT::f16 ||
                                          VA.getValVT() == MVT::bf16)) {
            // f16 and bf16 values are passed in the least-significant half of
            // a 4 byte stack slot. This is done as-if the extension was done
            // in a 32-bit register, so the actual bytes used for the value
            // differ between little and big endian.
            assert(VA.getLocVT().getSizeInBits() == 32);
            unsigned FIOffset = VA.getLocMemOffset();
            int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits() / 8,
                                           FIOffset, true);
 
            SDValue Addr = DAG.getFrameIndex(FI, PtrVT);
            if (DAG.getDataLayout().isBigEndian())
              Addr = DAG.getObjectPtrOffset(dl, Addr, TypeSize::getFixed(2));
 
            InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, Addr,
                                         MachinePointerInfo::getFixedStack(
                                             DAG.getMachineFunction(), FI)));
 
          } else {
            unsigned FIOffset = VA.getLocMemOffset();
            int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits()/8,
                                           FIOffset, true);
 
            // Create load nodes to retrieve arguments from the stack.
            SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
            InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
                                         MachinePointerInfo::getFixedStack(
                                             DAG.getMachineFunction(), FI)));
          }
          lastInsIndex = index;
        }
    }
  }
 
  // varargs
  if (isVarArg && MFI.hasVAStart()) {
    VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.getStackSize(),
                         TotalArgRegsSaveSize);
    if (AFI->isCmseNSEntryFunction()) {
      DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
          DAG.getMachineFunction().getFunction(),
          "secure entry function must not be variadic", dl.getDebugLoc()));
    }
  }
 
  unsigned StackArgSize = CCInfo.getStackSize();
  bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
  if (canGuaranteeTCO(CallConv, TailCallOpt)) {
    // The only way to guarantee a tail call is if the callee restores its
    // argument area, but it must also keep the stack aligned when doing so.
    MaybeAlign StackAlign = DAG.getDataLayout().getStackAlignment();
    assert(StackAlign && "data layout string is missing stack alignment");
    StackArgSize = alignTo(StackArgSize, *StackAlign);
 
    AFI->setArgumentStackToRestore(StackArgSize);
  }
  AFI->setArgumentStackSize(StackArgSize);
 
  if (CCInfo.getStackSize() > 0 && AFI->isCmseNSEntryFunction()) {
    DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
        DAG.getMachineFunction().getFunction(),
        "secure entry function requires arguments on stack", dl.getDebugLoc()));
  }
 
  return Chain;
}
 
/// isFloatingPointZero - Return true if this is +0.0.
static bool isFloatingPointZero(SDValue Op) {
  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
    return CFP->getValueAPF().isPosZero();
  else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
    // Maybe this has already been legalized into the constant pool?
    if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
      SDValue WrapperOp = Op.getOperand(1).getOperand(0);
      if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp))
        if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
          return CFP->getValueAPF().isPosZero();
    }
  } else if (Op->getOpcode() == ISD::BITCAST &&
             Op->getValueType(0) == MVT::f64) {
    // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
    // created by LowerConstantFP().
    SDValue BitcastOp = Op->getOperand(0);
    if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
        isNullConstant(BitcastOp->getOperand(0)))
      return true;
  }
  return false;
}
 
static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG) {
  // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
  if (Op->getFlags().hasNoSignedWrap())
    return true;
 
  // We can still figure out if the second operand is safe to use
  // in a CMN instruction by checking if it is known to be not the minimum
  // signed value. If it is not, then we can safely use CMN.
  // Note: We can eventually remove this check and simply rely on
  // Op->getFlags().hasNoSignedWrap() once SelectionDAG/ISelLowering
  // consistently sets them appropriately when making said nodes.
 
  KnownBits KnownSrc = DAG.computeKnownBits(Op.getOperand(1));
  return !KnownSrc.getSignedMinValue().isMinSignedValue();
}
 
static bool isCMN(SDValue Op, ISD::CondCode CC, SelectionDAG &DAG) {
  return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
         (isIntEqualitySetCC(CC) ||
          (isUnsignedIntSetCC(CC) && DAG.isKnownNeverZero(Op.getOperand(1))) ||
          (isSignedIntSetCC(CC) && isSafeSignedCMN(Op, DAG)));
}
 
/// Returns how profitable it is to fold a comparison's operand's shift and/or
/// extension operations into the comparison instruction's second operand
/// (so_reg_imm / so_reg_reg for ARM, t2_so_reg for Thumb-2).
static unsigned getCmpOperandFoldingProfit(SDValue Op, const ARMSubtarget &ST) {
  // Thumb-1 CMP does not support shifted second operands.
  if (ST.isThumb1Only() || !Op.hasOneUse())
    return 0;
 
  unsigned Opc = Op.getOpcode();
  if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) {
    if (auto *ShiftAmt = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
      return ShiftAmt->getZExtValue() <= 31 ? 1 : 0;
    // Register-controlled shift: only ARM-mode CMP/CMN (so_reg_reg) supports
    // this; Thumb-2 t2_so_reg requires an immediate shift amount.
    return ST.isThumb() ? 0 : 1;
  }
 
  if (Opc == ISD::ROTR) {
    // Rotr constants will be normalized via mod 32, or & 31,
    // so we do not have to bounds check.
    if (isa<ConstantSDNode>(Op.getOperand(1)))
      return 1;
    return ST.isThumb() ? 0 : 1;
  }
 
  return 0;
}
 
/// Returns appropriate ARM CMP (cmp) and corresponding condition code for
/// the given operands.
SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                                     SDValue &ARMcc, SelectionDAG &DAG,
                                     const SDLoc &dl) const {
  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
    unsigned C = RHSC->getZExtValue();
    if (!isLegalICmpImmediate((int32_t)C)) {
      // Constant does not fit, try adjusting it by one.
      switch (CC) {
      default: break;
      case ISD::SETLT:
      case ISD::SETGE:
        if (C != 0x80000000 && isLegalICmpImmediate(C-1)) {
          CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
          RHS = DAG.getConstant(C - 1, dl, MVT::i32);
        }
        break;
      case ISD::SETULT:
      case ISD::SETUGE:
        if (C != 0 && isLegalICmpImmediate(C-1)) {
          CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
          RHS = DAG.getConstant(C - 1, dl, MVT::i32);
        }
        break;
      case ISD::SETLE:
      case ISD::SETGT:
        if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) {
          CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
          RHS = DAG.getConstant(C + 1, dl, MVT::i32);
        }
        break;
      case ISD::SETULE:
      case ISD::SETUGT:
        if (C != 0xffffffff && isLegalICmpImmediate(C+1)) {
          CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
          RHS = DAG.getConstant(C + 1, dl, MVT::i32);
        }
        break;
      }
    }
  }
 
  // Thumb1 has very limited immediate modes, so turning an "and" into a
  // shift can save multiple instructions.
  //
  // If we have (x & C1), and C1 is an appropriate mask, we can transform it
  // into "((x << n) >> n)".  But that isn't necessarily profitable on its
  // own. If it's the operand to an unsigned comparison with an immediate,
  // we can eliminate one of the shifts: we transform
  // "((x << n) >> n) == C2" to "(x << n) == (C2 << n)".
  //
  // We avoid transforming cases which aren't profitable due to encoding
  // details:
  //
  // 1. C2 fits into the immediate field of a cmp, and the transformed version
  // would not; in that case, we're essentially trading one immediate load for
  // another.
  // 2. C1 is 255 or 65535, so we can use uxtb or uxth.
  // 3. C2 is zero; we have other code for this special case.
  //
  // FIXME: Figure out profitability for Thumb2; we usually can't save an
  // instruction, since the AND is always one instruction anyway, but we could
  // use narrow instructions in some cases.
  if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND &&
      LHS->hasOneUse() && isa<ConstantSDNode>(LHS.getOperand(1)) &&
      LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
      !isSignedIntSetCC(CC)) {
    unsigned Mask = LHS.getConstantOperandVal(1);
    auto *RHSC = cast<ConstantSDNode>(RHS.getNode());
    uint64_t RHSV = RHSC->getZExtValue();
    if (isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
      unsigned ShiftBits = llvm::countl_zero(Mask);
      if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
        SDValue ShiftAmt = DAG.getConstant(ShiftBits, dl, MVT::i32);
        LHS = DAG.getNode(ISD::SHL, dl, MVT::i32, LHS.getOperand(0), ShiftAmt);
        RHS = DAG.getConstant(RHSV << ShiftBits, dl, MVT::i32);
      }
    }
  }
 
  // The specific comparison "(x<<c) > 0x80000000U" can be optimized to a
  // single "lsls x, c+1".  The shift sets the "C" and "Z" flags the same
  // way a cmp would.
  // FIXME: Add support for ARM/Thumb2; this would need isel patterns, and
  // some tweaks to the heuristics for the previous and->shift transform.
  // FIXME: Optimize cases where the LHS isn't a shift.
  if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL &&
      isa<ConstantSDNode>(RHS) && RHS->getAsZExtVal() == 0x80000000U &&
      CC == ISD::SETUGT && isa<ConstantSDNode>(LHS.getOperand(1)) &&
      LHS.getConstantOperandVal(1) < 31) {
    unsigned ShiftAmt = LHS.getConstantOperandVal(1) + 1;
    SDValue Shift =
        DAG.getNode(ARMISD::LSLS, dl, DAG.getVTList(MVT::i32, FlagsVT),
                    LHS.getOperand(0), DAG.getConstant(ShiftAmt, dl, MVT::i32));
    ARMcc = DAG.getConstant(ARMCC::HI, dl, MVT::i32);
    return Shift.getValue(1);
  }
 
  ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
 
  unsigned CompareType;
  switch (CondCode) {
  default:
    CompareType = ARMISD::CMP;
    break;
  case ARMCC::EQ:
  case ARMCC::NE:
    // Uses only Z Flag
    CompareType = ARMISD::CMPZ;
    break;
  }
 
  // TODO: Remove CMPZ check once we generalize and remove the CMPZ enum from
  // the codebase.
 
  // TODO: When we have a solution to the vselect predicate not allowing pl/mi
  // all the time, allow those cases to be cmn too no matter what.
  if (CompareType != ARMISD::CMPZ && isCMN(RHS, CC, DAG)) {
    CompareType = ARMISD::CMN;
    RHS = RHS.getOperand(1);
  } else if (CompareType != ARMISD::CMPZ && isCMN(LHS, CC, DAG)) {
    CompareType = ARMISD::CMN;
    LHS = LHS.getOperand(1);
    CondCode = IntCCToARMCC(ISD::getSetCCSwappedOperands(CC));
  }
 
  // Prefer folding shifts / CMN into the cmp/cmn second operand (so_reg /
  // t2_so_reg). When both sides compete, pick the higher
  // getCmpOperandFoldingProfit. Only when RHS is not a legal icmp
  // immediate: otherwise keep the canonical (reg, imm) form.
  ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS.getNode());
  if (!C || !isLegalICmpImmediate(C->getSExtValue())) {
    if (getCmpOperandFoldingProfit(LHS, *Subtarget) >
        getCmpOperandFoldingProfit(RHS, *Subtarget)) {
      std::swap(LHS, RHS);
      if (CompareType == ARMISD::CMP)
        CondCode = ARMCC::getSwappedCondition(CondCode);
    }
  }
 
  // If the RHS is a constant zero then the V (overflow) flag will never be
  // set. This can allow us to simplify GE to PL or LT to MI, which can be
  // simpler for other passes (like the peephole optimiser) to deal with.
  if (isNullConstant(RHS)) {
    switch (CondCode) {
    default:
      break;
    case ARMCC::GE:
      CondCode = ARMCC::PL;
      break;
    case ARMCC::LT:
      CondCode = ARMCC::MI;
      break;
    }
  }
 
  ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
  return DAG.getNode(CompareType, dl, FlagsVT, LHS, RHS);
}
 
/// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
                                     SelectionDAG &DAG, const SDLoc &dl,
                                     bool Signaling) const {
  assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64);
  SDValue Flags;
  if (!isFloatingPointZero(RHS))
    Flags = DAG.getNode(Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP, dl, FlagsVT,
                        LHS, RHS);
  else
    Flags = DAG.getNode(Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0, dl,
                        FlagsVT, LHS);
  return DAG.getNode(ARMISD::FMSTAT, dl, FlagsVT, Flags);
}
 
// This function returns three things: the arithmetic computation itself
// (Value), a comparison (OverflowCmp), and a condition code (ARMcc).  The
// comparison and the condition code define the case in which the arithmetic
// computation *does not* overflow.
std::pair<SDValue, SDValue>
ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
                                 SDValue &ARMcc) const {
  assert(Op.getValueType() == MVT::i32 &&  "Unsupported value type");
 
  SDValue Value, OverflowCmp;
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
  SDLoc dl(Op);
 
  // FIXME: We are currently always generating CMPs because we don't support
  // generating CMN through the backend. This is not as good as the natural
  // CMP case because it causes a register dependency and cannot be folded
  // later.
 
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Unknown overflow instruction!");
  case ISD::SADDO:
    ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
    Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
    break;
  case ISD::UADDO:
    ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
    // We use ADDC here to correspond to its use in LowerALUO.
    // We do not use it in the USUBO case as Value may not be used.
    Value = DAG.getNode(ARMISD::ADDC, dl,
                        DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS)
                .getValue(0);
    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, Value, LHS);
    break;
  case ISD::SSUBO:
    ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
    Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
    break;
  case ISD::USUBO:
    ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
    Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
    OverflowCmp = DAG.getNode(ARMISD::CMP, dl, FlagsVT, LHS, RHS);
    break;
  case ISD::UMULO:
    // We generate a UMUL_LOHI and then check if the high word is 0.
    ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
    Value = DAG.getNode(ISD::UMUL_LOHI, dl,
                        DAG.getVTList(Op.getValueType(), Op.getValueType()),
                        LHS, RHS);
    OverflowCmp = DAG.getNode(ARMISD::CMPZ, dl, FlagsVT, Value.getValue(1),
                              DAG.getConstant(0, dl, MVT::i32));
    Value = Value.getValue(0); // We only want the low 32 bits for the result.
    break;
  case ISD::SMULO:
    // We generate a SMUL_LOHI and then check if all the bits of the high word
    // are the same as the sign bit of the low word.
    ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
    Value = DAG.getNode(ISD::SMUL_LOHI, dl,
                        DAG.getVTList(Op.getValueType(), Op.getValueType()),
                        LHS, RHS);
    OverflowCmp = DAG.getNode(ARMISD::CMPZ, dl, FlagsVT, Value.getValue(1),
                              DAG.getNode(ISD::SRA, dl, Op.getValueType(),
                                          Value.getValue(0),
                                          DAG.getConstant(31, dl, MVT::i32)));
    Value = Value.getValue(0); // We only want the low 32 bits for the result.
    break;
  } // switch (...)
 
  return std::make_pair(Value, OverflowCmp);
}
 
static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG, bool Invert) {
  SDLoc DL(Value);
  EVT VT = Value.getValueType();
 
  if (Invert)
    Value = DAG.getNode(ISD::SUB, DL, MVT::i32,
                        DAG.getConstant(1, DL, MVT::i32), Value);
 
  SDValue Cmp = DAG.getNode(ARMISD::SUBC, DL, DAG.getVTList(VT, MVT::i32),
                            Value, DAG.getConstant(1, DL, VT));
  return Cmp.getValue(1);
}
 
static SDValue carryFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG,
                                bool Invert) {
  SDLoc DL(Flags);
 
  if (Invert) {
    // Convert flags to boolean with ADDE 0,0,Carry then compute 1 - bool.
    SDValue BoolCarry = DAG.getNode(
        ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
        DAG.getConstant(0, DL, VT), DAG.getConstant(0, DL, VT), Flags);
    return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(1, DL, VT), BoolCarry);
  }
 
  // Now convert the carry flag into a boolean carry. We do this
  // using ARMISD::ADDE 0, 0, Carry
  return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
                     DAG.getConstant(0, DL, VT), DAG.getConstant(0, DL, VT),
                     Flags);
}
 
// Value is 1 if 'V' bit is 1, else 0
static SDValue overflowFlagToValue(SDValue Flags, EVT VT, SelectionDAG &DAG) {
  SDLoc DL(Flags);
  SDValue Zero = DAG.getConstant(0, DL, VT);
  SDValue One = DAG.getConstant(1, DL, VT);
  SDValue ARMcc = DAG.getConstant(ARMCC::VS, DL, MVT::i32);
  return DAG.getNode(ARMISD::CMOV, DL, VT, Zero, One, ARMcc, Flags);
}
 
SDValue ARMTargetLowering::LowerALUO(SDValue Op, SelectionDAG &DAG) const {
  // Let legalize expand this if it isn't a legal type yet.
  if (!isTypeLegal(Op.getValueType()))
    return SDValue();
 
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
  SDLoc dl(Op);
 
  EVT VT = Op.getValueType();
  SDVTList VTs = DAG.getVTList(VT, MVT::i32);
  SDValue Value;
  SDValue Overflow;
  switch (Op.getOpcode()) {
  case ISD::UADDO:
    Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS);
    // Convert the carry flag into a boolean value.
    Overflow = carryFlagToValue(Value.getValue(1), VT, DAG, false);
    break;
  case ISD::USUBO:
    Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS);
    // Convert the carry flag into a boolean value.
    Overflow = carryFlagToValue(Value.getValue(1), VT, DAG, true);
    break;
  default: {
    // Handle other operations with getARMXALUOOp
    SDValue OverflowCmp, ARMcc;
    std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
    // We use 0 and 1 as false and true values.
    // ARMcc represents the "no overflow" condition (e.g., VC for signed ops).
    // CMOV operand order is (FalseVal, TrueVal), so we put 1 in FalseVal
    // position to get Overflow=1 when the "no overflow" condition is false.
    Overflow =
        DAG.getNode(ARMISD::CMOV, dl, MVT::i32,
                    DAG.getConstant(1, dl, MVT::i32), // FalseVal: overflow
                    DAG.getConstant(0, dl, MVT::i32), // TrueVal: no overflow
                    ARMcc, OverflowCmp);
    break;
  }
  }
 
  return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
}
 
static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG,
                              const ARMSubtarget *Subtarget) {
  EVT VT = Op.getValueType();
  if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP() || Subtarget->isThumb1Only())
    return SDValue();
  if (!VT.isSimple())
    return SDValue();
 
  unsigned NewOpcode;
  switch (VT.getSimpleVT().SimpleTy) {
  default:
    return SDValue();
  case MVT::i8:
    switch (Op->getOpcode()) {
    case ISD::UADDSAT:
      NewOpcode = ARMISD::UQADD8b;
      break;
    case ISD::SADDSAT:
      NewOpcode = ARMISD::QADD8b;
      break;
    case ISD::USUBSAT:
      NewOpcode = ARMISD::UQSUB8b;
      break;
    case ISD::SSUBSAT:
      NewOpcode = ARMISD::QSUB8b;
      break;
    }
    break;
  case MVT::i16:
    switch (Op->getOpcode()) {
    case ISD::UADDSAT:
      NewOpcode = ARMISD::UQADD16b;
      break;
    case ISD::SADDSAT:
      NewOpcode = ARMISD::QADD16b;
      break;
    case ISD::USUBSAT:
      NewOpcode = ARMISD::UQSUB16b;
      break;
    case ISD::SSUBSAT:
      NewOpcode = ARMISD::QSUB16b;
      break;
    }
    break;
  }
 
  SDLoc dl(Op);
  SDValue Add =
      DAG.getNode(NewOpcode, dl, MVT::i32,
                  DAG.getSExtOrTrunc(Op->getOperand(0), dl, MVT::i32),
                  DAG.getSExtOrTrunc(Op->getOperand(1), dl, MVT::i32));
  return DAG.getNode(ISD::TRUNCATE, dl, VT, Add);
}
 
SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
  SDValue Cond = Op.getOperand(0);
  SDValue SelectTrue = Op.getOperand(1);
  SDValue SelectFalse = Op.getOperand(2);
  SDLoc dl(Op);
  unsigned Opc = Cond.getOpcode();
 
  if (Cond.getResNo() == 1 &&
      (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
       Opc == ISD::USUBO)) {
    if (!isTypeLegal(Cond->getValueType(0)))
      return SDValue();
 
    SDValue Value, OverflowCmp;
    SDValue ARMcc;
    std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
    EVT VT = Op.getValueType();
 
    return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, OverflowCmp, DAG);
  }
 
  // Convert:
  //
  //   (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
  //   (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
  //
  if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
    const ConstantSDNode *CMOVTrue =
      dyn_cast<ConstantSDNode>(Cond.getOperand(0));
    const ConstantSDNode *CMOVFalse =
      dyn_cast<ConstantSDNode>(Cond.getOperand(1));
 
    if (CMOVTrue && CMOVFalse) {
      unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
      unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
 
      SDValue True;
      SDValue False;
      if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
        True = SelectTrue;
        False = SelectFalse;
      } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
        True = SelectFalse;
        False = SelectTrue;
      }
 
      if (True.getNode() && False.getNode())
        return getCMOV(dl, Op.getValueType(), True, False, Cond.getOperand(2),
                       Cond.getOperand(3), DAG);
    }
  }
 
  return DAG.getSelectCC(dl, Cond,
                         DAG.getConstant(0, dl, Cond.getValueType()),
                         SelectTrue, SelectFalse, ISD::SETNE);
}
 
static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
                                 bool &swpCmpOps, bool &swpVselOps) {
  // Start by selecting the GE condition code for opcodes that return true for
  // 'equality'
  if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
      CC == ISD::SETULE || CC == ISD::SETGE  || CC == ISD::SETLE)
    CondCode = ARMCC::GE;
 
  // and GT for opcodes that return false for 'equality'.
  else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
           CC == ISD::SETULT || CC == ISD::SETGT  || CC == ISD::SETLT)
    CondCode = ARMCC::GT;
 
  // Since we are constrained to GE/GT, if the opcode contains 'less', we need
  // to swap the compare operands.
  if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
      CC == ISD::SETULT || CC == ISD::SETLE  || CC == ISD::SETLT)
    swpCmpOps = true;
 
  // Both GT and GE are ordered comparisons, and return false for 'unordered'.
  // If we have an unordered opcode, we need to swap the operands to the VSEL
  // instruction (effectively negating the condition).
  //
  // This also has the effect of swapping which one of 'less' or 'greater'
  // returns true, so we also swap the compare operands. It also switches
  // whether we return true for 'equality', so we compensate by picking the
  // opposite condition code to our original choice.
  if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
      CC == ISD::SETUGT) {
    swpCmpOps = !swpCmpOps;
    swpVselOps = !swpVselOps;
    CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
  }
 
  // 'ordered' is 'anything but unordered', so use the VS condition code and
  // swap the VSEL operands.
  if (CC == ISD::SETO) {
    CondCode = ARMCC::VS;
    swpVselOps = true;
  }
 
  // 'unordered or not equal' is 'anything but equal', so use the EQ condition
  // code and swap the VSEL operands. Also do this if we don't care about the
  // unordered case.
  if (CC == ISD::SETUNE || CC == ISD::SETNE) {
    CondCode = ARMCC::EQ;
    swpVselOps = true;
  }
}
 
SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
                                   SDValue TrueVal, SDValue ARMcc,
                                   SDValue Flags, SelectionDAG &DAG) const {
  if (!Subtarget->hasFP64() && VT == MVT::f64) {
    FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
                           DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
    TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl,
                          DAG.getVTList(MVT::i32, MVT::i32), TrueVal);
 
    SDValue TrueLow = TrueVal.getValue(0);
    SDValue TrueHigh = TrueVal.getValue(1);
    SDValue FalseLow = FalseVal.getValue(0);
    SDValue FalseHigh = FalseVal.getValue(1);
 
    SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
                              ARMcc, Flags);
    SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
                               ARMcc, Flags);
 
    return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
  }
  return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, Flags);
}
 
static bool isGTorGE(ISD::CondCode CC) {
  return CC == ISD::SETGT || CC == ISD::SETGE;
}
 
static bool isLTorLE(ISD::CondCode CC) {
  return CC == ISD::SETLT || CC == ISD::SETLE;
}
 
// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
// All of these conditions (and their <= and >= counterparts) will do:
//          x < k ? k : x
//          x > k ? x : k
//          k < x ? x : k
//          k > x ? k : x
static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
                            const SDValue TrueVal, const SDValue FalseVal,
                            const ISD::CondCode CC, const SDValue K) {
  return (isGTorGE(CC) &&
          ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
         (isLTorLE(CC) &&
          ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
}
 
// Check if two chained conditionals could be converted into SSAT or USAT.
//
// SSAT can replace a set of two conditional selectors that bound a number to an
// interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
//
//     x < -k ? -k : (x > k ? k : x)
//     x < -k ? -k : (x < k ? x : k)
//     x > -k ? (x > k ? k : x) : -k
//     x < k ? (x < -k ? -k : x) : k
//     etc.
//
// LLVM canonicalizes these to either a min(max()) or a max(min())
// pattern. This function tries to match one of these and will return a SSAT
// node if successful.
//
// USAT works similarly to SSAT but bounds on the interval [0, k] where k + 1
// is a power of 2.
static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
  EVT VT = Op.getValueType();
  SDValue V1 = Op.getOperand(0);
  SDValue K1 = Op.getOperand(1);
  SDValue TrueVal1 = Op.getOperand(2);
  SDValue FalseVal1 = Op.getOperand(3);
  ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get();
 
  const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
  if (Op2.getOpcode() != ISD::SELECT_CC)
    return SDValue();
 
  SDValue V2 = Op2.getOperand(0);
  SDValue K2 = Op2.getOperand(1);
  SDValue TrueVal2 = Op2.getOperand(2);
  SDValue FalseVal2 = Op2.getOperand(3);
  ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get();
 
  SDValue V1Tmp = V1;
  SDValue V2Tmp = V2;
 
  // Check that the registers and the constants match a max(min()) or min(max())
  // pattern
  if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
      K2 != FalseVal2 ||
      !((isGTorGE(CC1) && isLTorLE(CC2)) || (isLTorLE(CC1) && isGTorGE(CC2))))
    return SDValue();
 
  // Check that the constant in the lower-bound check is
  // the opposite of the constant in the upper-bound check
  // in 1's complement.
  if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
    return SDValue();
 
  int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
  int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
  int64_t PosVal = std::max(Val1, Val2);
  int64_t NegVal = std::min(Val1, Val2);
 
  if (!((Val1 > Val2 && isLTorLE(CC1)) || (Val1 < Val2 && isLTorLE(CC2))) ||
      !isPowerOf2_64(PosVal + 1))
    return SDValue();
 
  // Handle the difference between USAT (unsigned) and SSAT (signed)
  // saturation
  // At this point, PosVal is guaranteed to be positive
  uint64_t K = PosVal;
  SDLoc dl(Op);
  if (Val1 == ~Val2)
    return DAG.getNode(ARMISD::SSAT, dl, VT, V2Tmp,
                       DAG.getConstant(llvm::countr_one(K), dl, VT));
  if (NegVal == 0)
    return DAG.getNode(ARMISD::USAT, dl, VT, V2Tmp,
                       DAG.getConstant(llvm::countr_one(K), dl, VT));
 
  return SDValue();
}
 
// Check if a condition of the type x < k ? k : x can be converted into a
// bit operation instead of conditional moves.
// Currently this is allowed given:
// - The conditions and values match up
// - k is 0 or -1 (all ones)
// This function will not check the last condition, thats up to the caller
// It returns true if the transformation can be made, and in such case
// returns x in V, and k in SatK.
static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
                                         SDValue &SatK)
{
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
  SDValue TrueVal = Op.getOperand(2);
  SDValue FalseVal = Op.getOperand(3);
 
  SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS : isa<ConstantSDNode>(RHS)
                                               ? &RHS
                                               : nullptr;
 
  // No constant operation in comparison, early out
  if (!K)
    return false;
 
  SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
  V = (KTmp == TrueVal) ? FalseVal : TrueVal;
  SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
 
  // If the constant on left and right side, or variable on left and right,
  // does not match, early out
  if (*K != KTmp || V != VTmp)
    return false;
 
  if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, *K)) {
    SatK = *K;
    return true;
  }
 
  return false;
}
 
bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const {
  if (VT == MVT::f32)
    return !Subtarget->hasVFP2Base();
  if (VT == MVT::f64)
    return !Subtarget->hasFP64();
  if (VT == MVT::f16)
    return !Subtarget->hasFullFP16();
  return false;
}
 
static SDValue matchCSET(unsigned &Opcode, bool &InvertCond, SDValue TrueVal,
                         SDValue FalseVal, const ARMSubtarget *Subtarget) {
  ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
  ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
  if (!CFVal || !CTVal || !Subtarget->hasV8_1MMainlineOps())
    return SDValue();
 
  unsigned TVal = CTVal->getZExtValue();
  unsigned FVal = CFVal->getZExtValue();
 
  Opcode = 0;
  InvertCond = false;
  if (TVal == ~FVal) {
    Opcode = ARMISD::CSINV;
  } else if (TVal == ~FVal + 1) {
    Opcode = ARMISD::CSNEG;
  } else if (TVal + 1 == FVal) {
    Opcode = ARMISD::CSINC;
  } else if (TVal == FVal + 1) {
    Opcode = ARMISD::CSINC;
    std::swap(TrueVal, FalseVal);
    std::swap(TVal, FVal);
    InvertCond = !InvertCond;
  } else {
    return SDValue();
  }
 
  // If one of the constants is cheaper than another, materialise the
  // cheaper one and let the csel generate the other.
  if (Opcode != ARMISD::CSINC &&
      HasLowerConstantMaterializationCost(FVal, TVal, Subtarget)) {
    std::swap(TrueVal, FalseVal);
    std::swap(TVal, FVal);
    InvertCond = !InvertCond;
  }
 
  // Attempt to use ZR checking TVal is 0, possibly inverting the condition
  // to get there. CSINC not is invertable like the other two (~(~a) == a,
  // -(-a) == a, but (a+1)+1 != a).
  if (FVal == 0 && Opcode != ARMISD::CSINC) {
    std::swap(TrueVal, FalseVal);
    std::swap(TVal, FVal);
    InvertCond = !InvertCond;
  }
 
  return TrueVal;
}
 
SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  SDLoc dl(Op);
 
  // Try to convert two saturating conditional selects into a single SSAT
  if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
    if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
      return SatValue;
 
  // Try to convert expressions of the form x < k ? k : x (and similar forms)
  // into more efficient bit operations, which is possible when k is 0 or -1
  // On ARM and Thumb-2 which have flexible operand 2 this will result in
  // single instructions. On Thumb the shift and the bit operation will be two
  // instructions.
  // Only allow this transformation on full-width (32-bit) operations
  SDValue LowerSatConstant;
  SDValue SatValue;
  if (VT == MVT::i32 &&
      isLowerSaturatingConditional(Op, SatValue, LowerSatConstant)) {
    SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue,
                                 DAG.getConstant(31, dl, VT));
    if (isNullConstant(LowerSatConstant)) {
      SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV,
                                      DAG.getAllOnesConstant(dl, VT));
      return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV);
    } else if (isAllOnesConstant(LowerSatConstant))
      return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV);
  }
 
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
  SDValue TrueVal = Op.getOperand(2);
  SDValue FalseVal = Op.getOperand(3);
  ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
  ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
  if (Op.getValueType().isInteger()) {
 
    // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns.
    // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1))
    // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1))
    // Both require less instructions than compare and conditional select.
    if ((CC == ISD::SETGT || CC == ISD::SETLT) && LHS == TrueVal && RHSC &&
        RHSC->isZero() && CFVal && CFVal->isZero() &&
        LHS.getValueType() == RHS.getValueType()) {
      EVT VT = LHS.getValueType();
      SDValue Shift =
          DAG.getNode(ISD::SRA, dl, VT, LHS,
                      DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
 
      if (CC == ISD::SETGT)
        Shift = DAG.getNOT(dl, Shift, VT);
 
      return DAG.getNode(ISD::AND, dl, VT, LHS, Shift);
    }
 
    // (SELECT_CC setlt, x, 0, 1, 0) -> SRL(x, bw-1)
    if (CC == ISD::SETLT && isNullConstant(RHS) && isOneConstant(TrueVal) &&
        isNullConstant(FalseVal) && LHS.getValueType() == VT)
      return DAG.getNode(ISD::SRL, dl, VT, LHS,
                         DAG.getConstant(VT.getSizeInBits() - 1, dl, VT));
  }
 
  if (LHS.getValueType() == MVT::i32) {
    unsigned Opcode;
    bool InvertCond;
    if (SDValue Op =
            matchCSET(Opcode, InvertCond, TrueVal, FalseVal, Subtarget)) {
      if (InvertCond)
        CC = ISD::getSetCCInverse(CC, LHS.getValueType());
 
      SDValue ARMcc;
      SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
      EVT VT = Op.getValueType();
      return DAG.getNode(Opcode, dl, VT, Op, Op, ARMcc, Cmp);
    }
  }
 
  if (isUnsupportedFloatingType(LHS.getValueType())) {
    softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
 
    // If softenSetCCOperands only returned one value, we should compare it to
    // zero.
    if (!RHS.getNode()) {
      RHS = DAG.getConstant(0, dl, LHS.getValueType());
      CC = ISD::SETNE;
    }
  }
 
  if (LHS.getValueType() == MVT::i32) {
    // Try to generate VSEL on ARMv8.
    // The VSEL instruction can't use all the usual ARM condition
    // codes: it only has two bits to select the condition code, so it's
    // constrained to use only GE, GT, VS and EQ.
    //
    // To implement all the various ISD::SETXXX opcodes, we sometimes need to
    // swap the operands of the previous compare instruction (effectively
    // inverting the compare condition, swapping 'less' and 'greater') and
    // sometimes need to swap the operands to the VSEL (which inverts the
    // condition in the sense of firing whenever the previous condition didn't)
    if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 ||
                                        TrueVal.getValueType() == MVT::f32 ||
                                        TrueVal.getValueType() == MVT::f64)) {
      ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
      if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
          CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
        CC = ISD::getSetCCInverse(CC, LHS.getValueType());
        std::swap(TrueVal, FalseVal);
      }
    }
 
    SDValue ARMcc;
    SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
    // Choose GE over PL, which vsel does now support
    if (ARMcc->getAsZExtVal() == ARMCC::PL)
      ARMcc = DAG.getConstant(ARMCC::GE, dl, MVT::i32);
    return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
  }
 
  ARMCC::CondCodes CondCode, CondCode2;
  FPCCToARMCC(CC, CondCode, CondCode2);
 
  // Normalize the fp compare. If RHS is zero we prefer to keep it there so we
  // match CMPFPw0 instead of CMPFP, though we don't do this for f16 because we
  // must use VSEL (limited condition codes), due to not having conditional f16
  // moves.
  if (Subtarget->hasFPARMv8Base() &&
      !(isFloatingPointZero(RHS) && TrueVal.getValueType() != MVT::f16) &&
      (TrueVal.getValueType() == MVT::f16 ||
       TrueVal.getValueType() == MVT::f32 ||
       TrueVal.getValueType() == MVT::f64)) {
    bool swpCmpOps = false;
    bool swpVselOps = false;
    checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
 
    if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
        CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
      if (swpCmpOps)
        std::swap(LHS, RHS);
      if (swpVselOps)
        std::swap(TrueVal, FalseVal);
    }
  }
 
  SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
  SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
  SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, Cmp, DAG);
  if (CondCode2 != ARMCC::AL) {
    SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
    Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, Cmp, DAG);
  }
  return Result;
}
 
/// canChangeToInt - Given the fp compare operand, return true if it is suitable
/// to morph to an integer compare sequence.
static bool canChangeToInt(SDValue Op, bool &SeenZero,
                           const ARMSubtarget *Subtarget) {
  SDNode *N = Op.getNode();
  if (!N->hasOneUse())
    // Otherwise it requires moving the value from fp to integer registers.
    return false;
  if (!N->getNumValues())
    return false;
  EVT VT = Op.getValueType();
  if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
    // f32 case is generally profitable. f64 case only makes sense when vcmpe +
    // vmrs are very slow, e.g. cortex-a8.
    return false;
 
  if (isFloatingPointZero(Op)) {
    SeenZero = true;
    return true;
  }
  return ISD::isNormalLoad(N);
}
 
static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
  if (isFloatingPointZero(Op))
    return DAG.getConstant(0, SDLoc(Op), MVT::i32);
 
  if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op))
    return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(),
                       Ld->getPointerInfo(), Ld->getAlign(),
                       Ld->getMemOperand()->getFlags());
 
  llvm_unreachable("Unknown VFP cmp argument!");
}
 
static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
                           SDValue &RetVal1, SDValue &RetVal2) {
  SDLoc dl(Op);
 
  if (isFloatingPointZero(Op)) {
    RetVal1 = DAG.getConstant(0, dl, MVT::i32);
    RetVal2 = DAG.getConstant(0, dl, MVT::i32);
    return;
  }
 
  if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) {
    SDValue Ptr = Ld->getBasePtr();
    RetVal1 =
        DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
                    Ld->getAlign(), Ld->getMemOperand()->getFlags());
 
    EVT PtrType = Ptr.getValueType();
    SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
                                 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
    RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
                          Ld->getPointerInfo().getWithOffset(4),
                          commonAlignment(Ld->getAlign(), 4),
                          Ld->getMemOperand()->getFlags());
    return;
  }
 
  llvm_unreachable("Unknown VFP cmp argument!");
}
 
/// OptimizeVFPBrcond - With nnan and without daz, it's legal to optimize some
/// f32 and even f64 comparisons to integer ones.
SDValue
ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
  SDValue Chain = Op.getOperand(0);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
  SDValue LHS = Op.getOperand(2);
  SDValue RHS = Op.getOperand(3);
  SDValue Dest = Op.getOperand(4);
  SDLoc dl(Op);
 
  bool LHSSeenZero = false;
  bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget);
  bool RHSSeenZero = false;
  bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget);
  if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
    // If unsafe fp math optimization is enabled and there are no other uses of
    // the CMP operands, and the condition code is EQ or NE, we can optimize it
    // to an integer comparison.
    if (CC == ISD::SETOEQ)
      CC = ISD::SETEQ;
    else if (CC == ISD::SETUNE)
      CC = ISD::SETNE;
 
    SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
    SDValue ARMcc;
    if (LHS.getValueType() == MVT::f32) {
      LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
                        bitcastf32Toi32(LHS, DAG), Mask);
      RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
                        bitcastf32Toi32(RHS, DAG), Mask);
      SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
      return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
                         Cmp);
    }
 
    SDValue LHS1, LHS2;
    SDValue RHS1, RHS2;
    expandf64Toi32(LHS, DAG, LHS1, LHS2);
    expandf64Toi32(RHS, DAG, RHS1, RHS2);
    LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
    RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
    ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
    ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
    SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
    return DAG.getNode(ARMISD::BCC_i64, dl, MVT::Other, Ops);
  }
 
  return SDValue();
}
 
// Generate CMP + CMOV for integer abs.
SDValue ARMTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const {
  SDLoc DL(Op);
 
  SDValue Neg = DAG.getNegative(Op.getOperand(0), DL, MVT::i32);
 
  // Generate CMP & CMOV.
  SDValue Cmp = DAG.getNode(ARMISD::CMP, DL, FlagsVT, Op.getOperand(0),
                            DAG.getConstant(0, DL, MVT::i32));
  return DAG.getNode(ARMISD::CMOV, DL, MVT::i32, Op.getOperand(0), Neg,
                     DAG.getConstant(ARMCC::MI, DL, MVT::i32), Cmp);
}
 
static SDValue getInvertedARMCondCode(SDValue ARMcc, SelectionDAG &DAG) {
  ARMCC::CondCodes CondCode =
      (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
  CondCode = ARMCC::getOppositeCondition(CondCode);
  return DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
}
 
SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
  SDValue Chain = Op.getOperand(0);
  SDValue Cond = Op.getOperand(1);
  SDValue Dest = Op.getOperand(2);
  SDLoc dl(Op);
 
  // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
  // instruction.
  unsigned Opc = Cond.getOpcode();
  bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
                      !Subtarget->isThumb1Only();
  if (Cond.getResNo() == 1 &&
      (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
       Opc == ISD::USUBO || OptimizeMul)) {
    // Only lower legal XALUO ops.
    if (!isTypeLegal(Cond->getValueType(0)))
      return SDValue();
 
    // The actual operation with overflow check.
    SDValue Value, OverflowCmp;
    SDValue ARMcc;
    std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
 
    // Reverse the condition code.
    ARMcc = getInvertedARMCondCode(ARMcc, DAG);
 
    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
                       OverflowCmp);
  }
 
  return SDValue();
}
 
SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
  SDValue Chain = Op.getOperand(0);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
  SDValue LHS = Op.getOperand(2);
  SDValue RHS = Op.getOperand(3);
  SDValue Dest = Op.getOperand(4);
  SDLoc dl(Op);
 
  if (isUnsupportedFloatingType(LHS.getValueType())) {
    softenSetCCOperands(DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
 
    // If softenSetCCOperands only returned one value, we should compare it to
    // zero.
    if (!RHS.getNode()) {
      RHS = DAG.getConstant(0, dl, LHS.getValueType());
      CC = ISD::SETNE;
    }
  }
 
  // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
  // instruction.
  unsigned Opc = LHS.getOpcode();
  bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
                      !Subtarget->isThumb1Only();
  if (LHS.getResNo() == 1 && (isOneConstant(RHS) || isNullConstant(RHS)) &&
      (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
       Opc == ISD::USUBO || OptimizeMul) &&
      (CC == ISD::SETEQ || CC == ISD::SETNE)) {
    // Only lower legal XALUO ops.
    if (!isTypeLegal(LHS->getValueType(0)))
      return SDValue();
 
    // The actual operation with overflow check.
    SDValue Value, OverflowCmp;
    SDValue ARMcc;
    std::tie(Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc);
 
    if ((CC == ISD::SETNE) != isOneConstant(RHS)) {
      // Reverse the condition code.
      ARMcc = getInvertedARMCondCode(ARMcc, DAG);
    }
 
    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc,
                       OverflowCmp);
  }
 
  if (LHS.getValueType() == MVT::i32) {
    SDValue ARMcc;
    SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
    return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, Cmp);
  }
 
  SDNodeFlags Flags = Op->getFlags();
  if (Flags.hasNoNaNs() &&
      DAG.getDenormalMode(MVT::f32) == DenormalMode::getIEEE() &&
      DAG.getDenormalMode(MVT::f64) == DenormalMode::getIEEE() &&
      (CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETNE ||
       CC == ISD::SETUNE)) {
    if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
      return Result;
  }
 
  ARMCC::CondCodes CondCode, CondCode2;
  FPCCToARMCC(CC, CondCode, CondCode2);
 
  SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
  SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
  SDValue Ops[] = {Chain, Dest, ARMcc, Cmp};
  SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
  if (CondCode2 != ARMCC::AL) {
    ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
    SDValue Ops[] = {Res, Dest, ARMcc, Cmp};
    Res = DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Ops);
  }
  return Res;
}
 
SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
  SDValue Chain = Op.getOperand(0);
  SDValue Table = Op.getOperand(1);
  SDValue Index = Op.getOperand(2);
  SDLoc dl(Op);
 
  EVT PTy = getPointerTy(DAG.getDataLayout());
  JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
  SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
  Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
  Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
  SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index);
  if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
    // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table
    // which does another jump to the destination. This also makes it easier
    // to translate it to TBB / TBH later (Thumb2 only).
    // FIXME: This might not work if the function is extremely large.
    return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
                       Addr, Op.getOperand(2), JTI);
  }
  if (isPositionIndependent() || Subtarget->isROPI()) {
    Addr =
        DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
                    MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
    Chain = Addr.getValue(1);
    Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr);
    return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
  } else {
    Addr =
        DAG.getLoad(PTy, dl, Chain, Addr,
                    MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
    Chain = Addr.getValue(1);
    return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
  }
}
 
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
  EVT VT = Op.getValueType();
  SDLoc dl(Op);
 
  if (Op.getValueType().getVectorElementType() == MVT::i32) {
    if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
      return Op;
    return DAG.UnrollVectorOp(Op.getNode());
  }
 
  const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
 
  EVT NewTy;
  const EVT OpTy = Op.getOperand(0).getValueType();
  if (OpTy == MVT::v4f32)
    NewTy = MVT::v4i32;
  else if (OpTy == MVT::v4f16 && HasFullFP16)
    NewTy = MVT::v4i16;
  else if (OpTy == MVT::v8f16 && HasFullFP16)
    NewTy = MVT::v8i16;
  else
    llvm_unreachable("Invalid type for custom lowering!");
 
  if (VT != MVT::v4i16 && VT != MVT::v8i16)
    return DAG.UnrollVectorOp(Op.getNode());
 
  Op = DAG.getNode(Op.getOpcode(), dl, NewTy, Op.getOperand(0));
  return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
}
 
SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  if (VT.isVector())
    return LowerVectorFP_TO_INT(Op, DAG);
 
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
 
  if (isUnsupportedFloatingType(SrcVal.getValueType())) {
    RTLIB::Libcall LC;
    if (Op.getOpcode() == ISD::FP_TO_SINT ||
        Op.getOpcode() == ISD::STRICT_FP_TO_SINT)
      LC = RTLIB::getFPTOSINT(SrcVal.getValueType(),
                              Op.getValueType());
    else
      LC = RTLIB::getFPTOUINT(SrcVal.getValueType(),
                              Op.getValueType());
    SDLoc Loc(Op);
    MakeLibCallOptions CallOptions;
    SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
    SDValue Result;
    std::tie(Result, Chain) = makeLibCall(DAG, LC, Op.getValueType(), SrcVal,
                                          CallOptions, Loc, Chain);
    return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
  }
 
  return Op;
}
 
static SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
                                  const ARMSubtarget *Subtarget) {
  EVT VT = Op.getValueType();
  EVT ToVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
  EVT FromVT = Op.getOperand(0).getValueType();
 
  if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f32)
    return Op;
  if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f64 &&
      Subtarget->hasFP64())
    return Op;
  if (VT == MVT::i32 && ToVT == MVT::i32 && FromVT == MVT::f16 &&
      Subtarget->hasFullFP16())
    return Op;
  if (VT == MVT::v4i32 && ToVT == MVT::i32 && FromVT == MVT::v4f32 &&
      Subtarget->hasMVEFloatOps())
    return Op;
  if (VT == MVT::v8i16 && ToVT == MVT::i16 && FromVT == MVT::v8f16 &&
      Subtarget->hasMVEFloatOps())
    return Op;
 
  if (FromVT != MVT::v4f32 && FromVT != MVT::v8f16)
    return SDValue();
 
  SDLoc DL(Op);
  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
  unsigned BW = ToVT.getScalarSizeInBits() - IsSigned;
  SDValue CVT = DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
                            DAG.getValueType(VT.getScalarType()));
  SDValue Max = DAG.getNode(IsSigned ? ISD::SMIN : ISD::UMIN, DL, VT, CVT,
                            DAG.getConstant((1 << BW) - 1, DL, VT));
  if (IsSigned)
    Max = DAG.getNode(ISD::SMAX, DL, VT, Max,
                      DAG.getSignedConstant(-(1 << BW), DL, VT));
  return Max;
}
 
static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
  EVT VT = Op.getValueType();
  SDLoc dl(Op);
 
  if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
    if (VT.getVectorElementType() == MVT::f32)
      return Op;
    return DAG.UnrollVectorOp(Op.getNode());
  }
 
  assert((Op.getOperand(0).getValueType() == MVT::v4i16 ||
          Op.getOperand(0).getValueType() == MVT::v8i16) &&
         "Invalid type for custom lowering!");
 
  const bool HasFullFP16 = DAG.getSubtarget<ARMSubtarget>().hasFullFP16();
 
  EVT DestVecType;
  if (VT == MVT::v4f32)
    DestVecType = MVT::v4i32;
  else if (VT == MVT::v4f16 && HasFullFP16)
    DestVecType = MVT::v4i16;
  else if (VT == MVT::v8f16 && HasFullFP16)
    DestVecType = MVT::v8i16;
  else
    return DAG.UnrollVectorOp(Op.getNode());
 
  unsigned CastOpc;
  unsigned Opc;
  switch (Op.getOpcode()) {
  default: llvm_unreachable("Invalid opcode!");
  case ISD::SINT_TO_FP:
    CastOpc = ISD::SIGN_EXTEND;
    Opc = ISD::SINT_TO_FP;
    break;
  case ISD::UINT_TO_FP:
    CastOpc = ISD::ZERO_EXTEND;
    Opc = ISD::UINT_TO_FP;
    break;
  }
 
  Op = DAG.getNode(CastOpc, dl, DestVecType, Op.getOperand(0));
  return DAG.getNode(Opc, dl, VT, Op);
}
 
SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
  EVT VT = Op.getValueType();
  if (VT.isVector())
    return LowerVectorINT_TO_FP(Op, DAG);
 
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
 
  if (isUnsupportedFloatingType(VT)) {
    RTLIB::Libcall LC;
    if (Op.getOpcode() == ISD::SINT_TO_FP ||
        Op.getOpcode() == ISD::STRICT_SINT_TO_FP)
      LC = RTLIB::getSINTTOFP(SrcVal.getValueType(), Op.getValueType());
    else
      LC = RTLIB::getUINTTOFP(SrcVal.getValueType(), Op.getValueType());
    SDLoc Loc(Op);
    MakeLibCallOptions CallOptions;
    SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
    SDValue Result;
    std::tie(Result, Chain) = makeLibCall(DAG, LC, Op.getValueType(), SrcVal,
                                          CallOptions, Loc, Chain);
    return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
  }
 
  return Op;
}
 
SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
  // Implement fcopysign with a fabs and a conditional fneg.
  SDValue Tmp0 = Op.getOperand(0);
  SDValue Tmp1 = Op.getOperand(1);
  SDLoc dl(Op);
  EVT VT = Op.getValueType();
  EVT SrcVT = Tmp1.getValueType();
  bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
    Tmp0.getOpcode() == ARMISD::VMOVDRR;
  bool UseNEON = !InGPR && Subtarget->hasNEON();
 
  if (UseNEON) {
    // Use VBSL to copy the sign bit.
    unsigned EncodedVal = ARM_AM::createVMOVModImm(0x6, 0x80);
    SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
                               DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
    EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
    if (VT == MVT::f64)
      Mask = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
                         DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
                         DAG.getConstant(32, dl, MVT::i32));
    else /*if (VT == MVT::f32)*/
      Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
    if (SrcVT == MVT::f32) {
      Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1);
      if (VT == MVT::f64)
        Tmp1 = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
                           DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
                           DAG.getConstant(32, dl, MVT::i32));
    } else if (VT == MVT::f32)
      Tmp1 = DAG.getNode(ARMISD::VSHRuIMM, dl, MVT::v1i64,
                         DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
                         DAG.getConstant(32, dl, MVT::i32));
    Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
    Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
 
    SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff),
                                            dl, MVT::i32);
    AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
    SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
                                  DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
 
    SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
                              DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask),
                              DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot));
    if (VT == MVT::f32) {
      Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
      Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
                        DAG.getConstant(0, dl, MVT::i32));
    } else {
      Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
    }
 
    return Res;
  }
 
  // Bitcast operand 1 to i32.
  if (SrcVT == MVT::f64)
    Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
                       Tmp1).getValue(1);
  Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
 
  // Or in the signbit with integer operations.
  SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
  SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
  Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
  if (VT == MVT::f32) {
    Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
                       DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
    return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
                       DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1));
  }
 
  // f64: Or the high part with signbit and then combine two parts.
  Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
                     Tmp0);
  SDValue Lo = Tmp0.getValue(0);
  SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
  Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1);
  return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
}
 
SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  MFI.setReturnAddressIsTaken(true);
 
  EVT VT = Op.getValueType();
  SDLoc dl(Op);
  unsigned Depth = Op.getConstantOperandVal(0);
  if (Depth) {
    SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
    SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
    return DAG.getLoad(VT, dl, DAG.getEntryNode(),
                       DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
                       MachinePointerInfo());
  }
 
  // Return LR, which contains the return address. Mark it an implicit live-in.
  Register Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
  return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
}
 
SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
  const ARMBaseRegisterInfo &ARI =
    *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  MFI.setFrameAddressIsTaken(true);
 
  EVT VT = Op.getValueType();
  SDLoc dl(Op);  // FIXME probably not meaningful
  unsigned Depth = Op.getConstantOperandVal(0);
  Register FrameReg = ARI.getFrameRegister(MF);
  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
  while (Depth--)
    FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
                            MachinePointerInfo());
  return FrameAddr;
}
 
// FIXME? Maybe this could be a TableGen attribute on some registers and
// this table could be generated automatically from RegInfo.
Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT,
                                              const MachineFunction &MF) const {
  return StringSwitch<Register>(RegName)
      .Case("sp", ARM::SP)
      .Default(Register());
}
 
// Result is 64 bit value so split into two 32 bit values and return as a
// pair of values.
static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
                                SelectionDAG &DAG) {
  SDLoc DL(N);
 
  // This function is only supposed to be called for i64 type destination.
  assert(N->getValueType(0) == MVT::i64
          && "ExpandREAD_REGISTER called for non-i64 type result.");
 
  SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
                             DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
                             N->getOperand(0),
                             N->getOperand(1));
 
  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
                    Read.getValue(1)));
  Results.push_back(Read.getValue(2)); // Chain
}
 
/// \p BC is a bitcast that is about to be turned into a VMOVDRR.
/// When \p DstVT, the destination type of \p BC, is on the vector
/// register bank and the source of bitcast, \p Op, operates on the same bank,
/// it might be possible to combine them, such that everything stays on the
/// vector register bank.
/// \p return The node that would replace \p BT, if the combine
/// is possible.
static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
                                                SelectionDAG &DAG) {
  SDValue Op = BC->getOperand(0);
  EVT DstVT = BC->getValueType(0);
 
  // The only vector instruction that can produce a scalar (remember,
  // since the bitcast was about to be turned into VMOVDRR, the source
  // type is i64) from a vector is EXTRACT_VECTOR_ELT.
  // Moreover, we can do this combine only if there is one use.
  // Finally, if the destination type is not a vector, there is not
  // much point on forcing everything on the vector bank.
  if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
      !Op.hasOneUse())
    return SDValue();
 
  // If the index is not constant, we will introduce an additional
  // multiply that will stick.
  // Give up in that case.
  ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1));
  if (!Index)
    return SDValue();
  unsigned DstNumElt = DstVT.getVectorNumElements();
 
  // Compute the new index.
  const APInt &APIntIndex = Index->getAPIntValue();
  APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
  NewIndex *= APIntIndex;
  // Check if the new constant index fits into i32.
  if (NewIndex.getBitWidth() > 32)
    return SDValue();
 
  // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
  // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
  SDLoc dl(Op);
  SDValue ExtractSrc = Op.getOperand(0);
  EVT VecVT = EVT::getVectorVT(
      *DAG.getContext(), DstVT.getScalarType(),
      ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
  SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast,
                     DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32));
}
 
/// ExpandBITCAST - If the target supports VFP, this function is called to
/// expand a bit convert where either the source or destination type is i64 to
/// use a VMOVDRR or VMOVRRD node.  This should not be done when the non-i64
/// operand type is illegal (e.g., v2f32 for a target that doesn't support
/// vectors), since the legalizer won't know what to do with that.
SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
                                         const ARMSubtarget *Subtarget) const {
  SDLoc dl(N);
  SDValue Op = N->getOperand(0);
 
  // This function is only supposed to be called for i16 and i64 types, either
  // as the source or destination of the bit convert.
  EVT SrcVT = Op.getValueType();
  EVT DstVT = N->getValueType(0);
 
  if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
      (DstVT == MVT::f16 || DstVT == MVT::bf16))
    return MoveToHPR(SDLoc(N), DAG, MVT::i32, DstVT.getSimpleVT(),
                     DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op));
 
  if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
      (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
    if (Subtarget->hasFullFP16() && !Subtarget->hasBF16())
      Op = DAG.getBitcast(MVT::f16, Op);
    return DAG.getNode(
        ISD::TRUNCATE, SDLoc(N), DstVT,
        MoveFromHPR(SDLoc(N), DAG, MVT::i32, SrcVT.getSimpleVT(), Op));
  }
 
  if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
    return SDValue();
 
  // Turn i64->f64 into VMOVDRR.
  if (SrcVT == MVT::i64 && isTypeLegal(DstVT)) {
    // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
    // if we can combine the bitcast with its source.
    if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
      return Val;
    SDValue Lo, Hi;
    std::tie(Lo, Hi) = DAG.SplitScalar(Op, dl, MVT::i32, MVT::i32);
    return DAG.getNode(ISD::BITCAST, dl, DstVT,
                       DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
  }
 
  // Turn f64->i64 into VMOVRRD.
  if (DstVT == MVT::i64 && isTypeLegal(SrcVT)) {
    SDValue Cvt;
    if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
        SrcVT.getVectorNumElements() > 1)
      Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
                        DAG.getVTList(MVT::i32, MVT::i32),
                        DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op));
    else
      Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
                        DAG.getVTList(MVT::i32, MVT::i32), Op);
    // Merge the pieces into a single i64 value.
    return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
  }
 
  return SDValue();
}
 
/// getZeroVector - Returns a vector of specified type with all zero elements.
/// Zero vectors are used to represent vector negation and in those cases
/// will be implemented with the NEON VNEG instruction.  However, VNEG does
/// not support i64 elements, so sometimes the zero vectors will need to be
/// explicitly constructed.  Regardless, use a canonical VMOV to create the
/// zero vector.
static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
  assert(VT.isVector() && "Expected a vector type");
  // The canonical modified immediate encoding of a zero vector is....0!
  SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
  EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
  SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
  return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
}
 
/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
/// i32 values and take a 2 x i32 value to shift plus a shift amount.
SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
                                                SelectionDAG &DAG) const {
  assert(Op.getNumOperands() == 3 && "Not a double-shift!");
  EVT VT = Op.getValueType();
  unsigned VTBits = VT.getSizeInBits();
  SDLoc dl(Op);
  SDValue ShOpLo = Op.getOperand(0);
  SDValue ShOpHi = Op.getOperand(1);
  SDValue ShAmt  = Op.getOperand(2);
  SDValue ARMcc;
  unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
 
  assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
 
  SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
                                 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
  SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
  SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
                                   DAG.getConstant(VTBits, dl, MVT::i32));
  SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
  SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
  SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
  SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                            ISD::SETGE, ARMcc, DAG, dl);
  SDValue Lo =
      DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift, ARMcc, CmpLo);
 
  SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
  SDValue HiBigShift = Opc == ISD::SRA
                           ? DAG.getNode(Opc, dl, VT, ShOpHi,
                                         DAG.getConstant(VTBits - 1, dl, VT))
                           : DAG.getConstant(0, dl, VT);
  SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                            ISD::SETGE, ARMcc, DAG, dl);
  SDValue Hi =
      DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
 
  SDValue Ops[2] = { Lo, Hi };
  return DAG.getMergeValues(Ops, dl);
}
 
/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
/// i32 values and take a 2 x i32 value to shift plus a shift amount.
SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
                                               SelectionDAG &DAG) const {
  assert(Op.getNumOperands() == 3 && "Not a double-shift!");
  EVT VT = Op.getValueType();
  unsigned VTBits = VT.getSizeInBits();
  SDLoc dl(Op);
  SDValue ShOpLo = Op.getOperand(0);
  SDValue ShOpHi = Op.getOperand(1);
  SDValue ShAmt  = Op.getOperand(2);
  SDValue ARMcc;
 
  assert(Op.getOpcode() == ISD::SHL_PARTS);
  SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
                                 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
  SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
  SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
  SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
 
  SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
                                   DAG.getConstant(VTBits, dl, MVT::i32));
  SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
  SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                            ISD::SETGE, ARMcc, DAG, dl);
  SDValue Hi =
      DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift, ARMcc, CmpHi);
 
  SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
                          ISD::SETGE, ARMcc, DAG, dl);
  SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
  SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
                           DAG.getConstant(0, dl, VT), ARMcc, CmpLo);
 
  SDValue Ops[2] = { Lo, Hi };
  return DAG.getMergeValues(Ops, dl);
}
 
SDValue ARMTargetLowering::LowerGET_ROUNDING(SDValue Op,
                                             SelectionDAG &DAG) const {
  // The rounding mode is in bits 23:22 of the FPSCR.
  // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
  // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
  // so that the shift + and get folded into a bitfield extract.
  SDLoc dl(Op);
  SDValue Chain = Op.getOperand(0);
  SDValue Ops[] = {Chain,
                   DAG.getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
 
  SDValue FPSCR =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, {MVT::i32, MVT::Other}, Ops);
  Chain = FPSCR.getValue(1);
  SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
                                  DAG.getConstant(1U << 22, dl, MVT::i32));
  SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
                              DAG.getConstant(22, dl, MVT::i32));
  SDValue And = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
                            DAG.getConstant(3, dl, MVT::i32));
  return DAG.getMergeValues({And, Chain}, dl);
}
 
SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op,
                                             SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue RMValue = Op->getOperand(1);
 
  // The rounding mode is in bits 23:22 of the FPSCR.
  // The llvm.set.rounding argument value to ARM rounding mode value mapping
  // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
  // ((arg - 1) & 3) << 22).
  //
  // It is expected that the argument of llvm.set.rounding is within the
  // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is
  // responsibility of the code generated llvm.set.rounding to ensure this
  // condition.
 
  // Calculate new value of FPSCR[23:22].
  RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
                        DAG.getConstant(1, DL, MVT::i32));
  RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
                        DAG.getConstant(0x3, DL, MVT::i32));
  RMValue = DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
                        DAG.getConstant(ARM::RoundingBitsPos, DL, MVT::i32));
 
  // Get current value of FPSCR.
  SDValue Ops[] = {Chain,
                   DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
  SDValue FPSCR =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
  Chain = FPSCR.getValue(1);
  FPSCR = FPSCR.getValue(0);
 
  // Put new rounding mode into FPSCR[23:22].
  const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos);
  FPSCR = DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
                      DAG.getConstant(RMMask, DL, MVT::i32));
  FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCR, RMValue);
  SDValue Ops2[] = {
      Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
  return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
}
 
SDValue ARMTargetLowering::LowerSET_FPMODE(SDValue Op,
                                           SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
  SDValue Mode = Op->getOperand(1);
 
  // Generate nodes to build:
  // FPSCR = (FPSCR & FPStatusBits) | (Mode & ~FPStatusBits)
  SDValue Ops[] = {Chain,
                   DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
  SDValue FPSCR =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
  Chain = FPSCR.getValue(1);
  FPSCR = FPSCR.getValue(0);
 
  SDValue FPSCRMasked =
      DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
                  DAG.getConstant(ARM::FPStatusBits, DL, MVT::i32));
  SDValue InputMasked =
      DAG.getNode(ISD::AND, DL, MVT::i32, Mode,
                  DAG.getConstant(~ARM::FPStatusBits, DL, MVT::i32));
  FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCRMasked, InputMasked);
 
  SDValue Ops2[] = {
      Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
  return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
}
 
SDValue ARMTargetLowering::LowerRESET_FPMODE(SDValue Op,
                                             SelectionDAG &DAG) const {
  SDLoc DL(Op);
  SDValue Chain = Op->getOperand(0);
 
  // To get the default FP mode all control bits are cleared:
  // FPSCR = FPSCR & (FPStatusBits | FPReservedBits)
  SDValue Ops[] = {Chain,
                   DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
  SDValue FPSCR =
      DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
  Chain = FPSCR.getValue(1);
  FPSCR = FPSCR.getValue(0);
 
  SDValue FPSCRMasked = DAG.getNode(
      ISD::AND, DL, MVT::i32, FPSCR,
      DAG.getConstant(ARM::FPStatusBits | ARM::FPReservedBits, DL, MVT::i32));
  SDValue Ops2[] = {Chain,
                    DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32),
                    FPSCRMasked};
  return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
}
 
static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
                         const ARMSubtarget *ST) {
  SDLoc dl(N);
  EVT VT = N->getValueType(0);
  if (VT.isVector() && ST->hasNEON()) {
 
    // Compute the least significant set bit: LSB = X & -X
    SDValue X = N->getOperand(0);
    SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
    SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
 
    EVT ElemTy = VT.getVectorElementType();
 
    if (ElemTy == MVT::i8) {
      // Compute with: cttz(x) = ctpop(lsb - 1)
      SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
                                DAG.getTargetConstant(1, dl, ElemTy));
      SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
      return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
    }
 
    if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
        (N->getOpcode() == ISD::CTTZ_ZERO_POISON)) {
      // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
      unsigned NumBits = ElemTy.getSizeInBits();
      SDValue WidthMinus1 =
          DAG.getNode(ARMISD::VMOVIMM, dl, VT,
                      DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
      SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
      return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
    }
 
    // Compute with: cttz(x) = ctpop(lsb - 1)
 
    // Compute LSB - 1.
    SDValue Bits;
    if (ElemTy == MVT::i64) {
      // Load constant 0xffff'ffff'ffff'ffff to register.
      SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
                               DAG.getTargetConstant(0x1eff, dl, MVT::i32));
      Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
    } else {
      SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
                                DAG.getTargetConstant(1, dl, ElemTy));
      Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
    }
    return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
  }
 
  if (!ST->hasV6T2Ops())
    return SDValue();
 
  SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
  return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
}
 
static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
                          const ARMSubtarget *ST) {
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
  assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
          VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
         "Unexpected type for custom ctpop lowering");
 
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
  SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0));
  Res = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Res);
 
  // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
  unsigned EltSize = 8;
  unsigned NumElts = VT.is64BitVector() ? 8 : 16;
  while (EltSize != VT.getScalarSizeInBits()) {
    SmallVector<SDValue, 8> Ops;
    Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddlu, DL,
                                  TLI.getPointerTy(DAG.getDataLayout())));
    Ops.push_back(Res);
 
    EltSize *= 2;
    NumElts /= 2;
    MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
    Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WidenVT, Ops);
  }
 
  return Res;
}
 
/// Getvshiftimm - Check if this is a valid build_vector for the immediate
/// operand of a vector shift operation, where all the elements of the
/// build_vector must have the same constant integer value.
static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
  // Ignore bit_converts.
  while (Op.getOpcode() == ISD::BITCAST)
    Op = Op.getOperand(0);
  BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
  APInt SplatBits, SplatUndef;
  unsigned SplatBitSize;
  bool HasAnyUndefs;
  if (!BVN ||
      !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
                            ElementBits) ||
      SplatBitSize > ElementBits)
    return false;
  Cnt = SplatBits.getSExtValue();
  return true;
}
 
/// isVShiftLImm - Check if this is a valid build_vector for the immediate
/// operand of a vector shift left operation.  That value must be in the range:
///   0 <= Value < ElementBits for a left shift; or
///   0 <= Value <= ElementBits for a long left shift.
static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
  assert(VT.isVector() && "vector shift count is not a vector type");
  int64_t ElementBits = VT.getScalarSizeInBits();
  if (!getVShiftImm(Op, ElementBits, Cnt))
    return false;
  return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
}
 
/// isVShiftRImm - Check if this is a valid build_vector for the immediate
/// operand of a vector shift right operation.  For a shift opcode, the value
/// is positive, but for an intrinsic the value count must be negative. The
/// absolute value must be in the range:
///   1 <= |Value| <= ElementBits for a right shift; or
///   1 <= |Value| <= ElementBits/2 for a narrow right shift.
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
                         int64_t &Cnt) {
  assert(VT.isVector() && "vector shift count is not a vector type");
  int64_t ElementBits = VT.getScalarSizeInBits();
  if (!getVShiftImm(Op, ElementBits, Cnt))
    return false;
  if (!isIntrinsic)
    return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
  if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
    Cnt = -Cnt;
    return true;
  }
  return false;
}
 
static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
                          const ARMSubtarget *ST) {
  EVT VT = N->getValueType(0);
  SDLoc dl(N);
  int64_t Cnt;
 
  if (!VT.isVector())
    return SDValue();
 
  // We essentially have two forms here. Shift by an immediate and shift by a
  // vector register (there are also shift by a gpr, but that is just handled
  // with a tablegen pattern). We cannot easily match shift by an immediate in
  // tablegen so we do that here and generate a VSHLIMM/VSHRsIMM/VSHRuIMM.
  // For shifting by a vector, we don't have VSHR, only VSHL (which can be
  // signed or unsigned, and a negative shift indicates a shift right).
  if (N->getOpcode() == ISD::SHL) {
    if (isVShiftLImm(N->getOperand(1), VT, false, Cnt))
      return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
                         DAG.getConstant(Cnt, dl, MVT::i32));
    return DAG.getNode(ARMISD::VSHLu, dl, VT, N->getOperand(0),
                       N->getOperand(1));
  }
 
  assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
         "unexpected vector shift opcode");
 
  if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
    unsigned VShiftOpc =
        (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
    return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
                       DAG.getConstant(Cnt, dl, MVT::i32));
  }
 
  // Other right shifts we don't have operations for (we use a shift left by a
  // negative number).
  EVT ShiftVT = N->getOperand(1).getValueType();
  SDValue NegatedCount = DAG.getNode(
      ISD::SUB, dl, ShiftVT, getZeroVector(ShiftVT, DAG, dl), N->getOperand(1));
  unsigned VShiftOpc =
      (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
  return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), NegatedCount);
}
 
static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
                                const ARMSubtarget *ST) {
  EVT VT = N->getValueType(0);
  SDLoc dl(N);
 
  // We can get here for a node like i32 = ISD::SHL i32, i64
  if (VT != MVT::i64)
    return SDValue();
 
  assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||
          N->getOpcode() == ISD::SHL) &&
         "Unknown shift to lower!");
 
  unsigned ShOpc = N->getOpcode();
  if (ST->hasMVEIntegerOps()) {
    SDValue ShAmt = N->getOperand(1);
    unsigned ShPartsOpc = ARMISD::LSLL;
    ConstantSDNode *Con = dyn_cast<ConstantSDNode>(ShAmt);
 
    // If the shift amount is greater than 32 or has a greater bitwidth than 64
    // then do the default optimisation
    if ((!Con && ShAmt->getValueType(0).getSizeInBits() > 64) ||
        (Con && (Con->getAPIntValue() == 0 || Con->getAPIntValue().uge(32))))
      return SDValue();
 
    // Extract the lower 32 bits of the shift amount if it's not an i32
    if (ShAmt->getValueType(0) != MVT::i32)
      ShAmt = DAG.getZExtOrTrunc(ShAmt, dl, MVT::i32);
 
    if (ShOpc == ISD::SRL) {
      if (!Con)
        // There is no t2LSRLr instruction so negate and perform an lsll if the
        // shift amount is in a register, emulating a right shift.
        ShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
                            DAG.getConstant(0, dl, MVT::i32), ShAmt);
      else
        // Else generate an lsrl on the immediate shift amount
        ShPartsOpc = ARMISD::LSRL;
    } else if (ShOpc == ISD::SRA)
      ShPartsOpc = ARMISD::ASRL;
 
    // Split Lower/Upper 32 bits of the destination/source
    SDValue Lo, Hi;
    std::tie(Lo, Hi) =
        DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32);
    // Generate the shift operation as computed above
    Lo = DAG.getNode(ShPartsOpc, dl, DAG.getVTList(MVT::i32, MVT::i32), Lo, Hi,
                     ShAmt);
    // The upper 32 bits come from the second return value of lsll
    Hi = SDValue(Lo.getNode(), 1);
    return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
  }
 
  // We only lower SRA, SRL of 1 here, all others use generic lowering.
  if (!isOneConstant(N->getOperand(1)) || N->getOpcode() == ISD::SHL)
    return SDValue();
 
  // If we are in thumb mode, we don't have RRX.
  if (ST->isThumb1Only())
    return SDValue();
 
  // Okay, we have a 64-bit SRA or SRL of 1.  Lower this to an RRX expr.
  SDValue Lo, Hi;
  std::tie(Lo, Hi) = DAG.SplitScalar(N->getOperand(0), dl, MVT::i32, MVT::i32);
 
  // First, build a LSRS1/ASRS1 op, which shifts the top part by one and
  // captures the shifted out bit into a carry flag.
  unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::LSRS1 : ARMISD::ASRS1;
  Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, FlagsVT), Hi);
 
  // The low part is an ARMISD::RRX operand, which shifts the carry in.
  Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
 
  // Merge the pieces into a single i64 value.
 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
}
 
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
                           const ARMSubtarget *ST) {
  bool Invert = false;
  bool Swap = false;
  unsigned Opc = ARMCC::AL;
 
  SDValue Op0 = Op.getOperand(0);
  SDValue Op1 = Op.getOperand(1);
  SDValue CC = Op.getOperand(2);
  EVT VT = Op.getValueType();
  ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
  SDLoc dl(Op);
 
  EVT CmpVT;
  if (ST->hasNEON())
    CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
  else {
    assert(ST->hasMVEIntegerOps() &&
           "No hardware support for integer vector comparison!");
 
    if (Op.getValueType().getVectorElementType() != MVT::i1)
      return SDValue();
 
    // Make sure we expand floating point setcc to scalar if we do not have
    // mve.fp, so that we can handle them from there.
    if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps())
      return SDValue();
 
    CmpVT = VT;
  }
 
  if (Op0.getValueType().getVectorElementType() == MVT::i64 &&
      (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) {
    // Special-case integer 64-bit equality comparisons. They aren't legal,
    // but they can be lowered with a few vector instructions.
    unsigned CmpElements = CmpVT.getVectorNumElements() * 2;
    EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, CmpElements);
    SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0);
    SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1);
    SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1,
                              DAG.getCondCode(ISD::SETEQ));
    SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
    SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed);
    Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged);
    if (SetCCOpcode == ISD::SETNE)
      Merged = DAG.getNOT(dl, Merged, CmpVT);
    Merged = DAG.getSExtOrTrunc(Merged, dl, VT);
    return Merged;
  }
 
  if (CmpVT.getVectorElementType() == MVT::i64)
    // 64-bit comparisons are not legal in general.
    return SDValue();
 
  if (Op1.getValueType().isFloatingPoint()) {
    switch (SetCCOpcode) {
    default: llvm_unreachable("Illegal FP comparison");
    case ISD::SETUNE:
    case ISD::SETNE:
      if (ST->hasMVEFloatOps()) {
        Opc = ARMCC::NE; break;
      } else {
        Invert = true; [[fallthrough]];
      }
    case ISD::SETOEQ:
    case ISD::SETEQ:  Opc = ARMCC::EQ; break;
    case ISD::SETOLT:
    case ISD::SETLT: Swap = true; [[fallthrough]];
    case ISD::SETOGT:
    case ISD::SETGT:  Opc = ARMCC::GT; break;
    case ISD::SETOLE:
    case ISD::SETLE:  Swap = true; [[fallthrough]];
    case ISD::SETOGE:
    case ISD::SETGE: Opc = ARMCC::GE; break;
    case ISD::SETUGE: Swap = true; [[fallthrough]];
    case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break;
    case ISD::SETUGT: Swap = true; [[fallthrough]];
    case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break;
    case ISD::SETUEQ: Invert = true; [[fallthrough]];
    case ISD::SETONE: {
      // Expand this to (OLT | OGT).
      SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
                                   DAG.getConstant(ARMCC::GT, dl, MVT::i32));
      SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
                                   DAG.getConstant(ARMCC::GT, dl, MVT::i32));
      SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
      if (Invert)
        Result = DAG.getNOT(dl, Result, VT);
      return Result;
    }
    case ISD::SETUO: Invert = true; [[fallthrough]];
    case ISD::SETO: {
      // Expand this to (OLT | OGE).
      SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
                                   DAG.getConstant(ARMCC::GT, dl, MVT::i32));
      SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
                                   DAG.getConstant(ARMCC::GE, dl, MVT::i32));
      SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
      if (Invert)
        Result = DAG.getNOT(dl, Result, VT);
      return Result;
    }
    }
  } else {
    // Integer comparisons.
    switch (SetCCOpcode) {
    default: llvm_unreachable("Illegal integer comparison");
    case ISD::SETNE:
      if (ST->hasMVEIntegerOps()) {
        Opc = ARMCC::NE; break;
      } else {
        Invert = true; [[fallthrough]];
      }
    case ISD::SETEQ:  Opc = ARMCC::EQ; break;
    case ISD::SETLT:  Swap = true; [[fallthrough]];
    case ISD::SETGT:  Opc = ARMCC::GT; break;
    case ISD::SETLE:  Swap = true; [[fallthrough]];
    case ISD::SETGE:  Opc = ARMCC::GE; break;
    case ISD::SETULT: Swap = true; [[fallthrough]];
    case ISD::SETUGT: Opc = ARMCC::HI; break;
    case ISD::SETULE: Swap = true; [[fallthrough]];
    case ISD::SETUGE: Opc = ARMCC::HS; break;
    }
 
    // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
    if (ST->hasNEON() && Opc == ARMCC::EQ) {
      SDValue AndOp;
      if (ISD::isBuildVectorAllZeros(Op1.getNode()))
        AndOp = Op0;
      else if (ISD::isBuildVectorAllZeros(Op0.getNode()))
        AndOp = Op1;
 
      // Ignore bitconvert.
      if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
        AndOp = AndOp.getOperand(0);
 
      if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
        Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
        Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
        SDValue Result = DAG.getNode(ARMISD::VTST, dl, CmpVT, Op0, Op1);
        if (!Invert)
          Result = DAG.getNOT(dl, Result, VT);
        return Result;
      }
    }
  }
 
  if (Swap)
    std::swap(Op0, Op1);
 
  // If one of the operands is a constant vector zero, attempt to fold the
  // comparison to a specialized compare-against-zero form.
  if (ISD::isBuildVectorAllZeros(Op0.getNode()) &&
      (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::EQ ||
       Opc == ARMCC::NE)) {
    if (Opc == ARMCC::GE)
      Opc = ARMCC::LE;
    else if (Opc == ARMCC::GT)
      Opc = ARMCC::LT;
    std::swap(Op0, Op1);
  }
 
  SDValue Result;
  if (ISD::isBuildVectorAllZeros(Op1.getNode()) &&
      (Opc == ARMCC::GE || Opc == ARMCC::GT || Opc == ARMCC::LE ||
       Opc == ARMCC::LT || Opc == ARMCC::NE || Opc == ARMCC::EQ))
    Result = DAG.getNode(ARMISD::VCMPZ, dl, CmpVT, Op0,
                         DAG.getConstant(Opc, dl, MVT::i32));
  else
    Result = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
                         DAG.getConstant(Opc, dl, MVT::i32));
 
  Result = DAG.getSExtOrTrunc(Result, dl, VT);
 
  if (Invert)
    Result = DAG.getNOT(dl, Result, VT);
 
  return Result;
}
 
static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
  SDValue LHS = Op.getOperand(0);
  SDValue RHS = Op.getOperand(1);
 
  assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.");
 
  SDValue Carry = Op.getOperand(2);
  SDValue Cond = Op.getOperand(3);
  SDLoc DL(Op);
 
  // ARMISD::SUBE expects a carry not a borrow like ISD::USUBO_CARRY so we
  // have to invert the carry first.
  SDValue InvCarry = valueToCarryFlag(Carry, DAG, true);
 
  SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
  SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, InvCarry);
 
  SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
  SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
  SDValue ARMcc = DAG.getConstant(
      IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32);
  return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
                     Cmp.getValue(1));
}
 
/// isVMOVModifiedImm - Check if the specified splat value corresponds to a
/// valid vector constant for a NEON or MVE instruction with a "modified
/// immediate" operand (e.g., VMOV).  If so, return the encoded value.
static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
                                 unsigned SplatBitSize, SelectionDAG &DAG,
                                 const SDLoc &dl, EVT &VT, EVT VectorVT,
                                 VMOVModImmType type) {
  unsigned OpCmode, Imm;
  bool is128Bits = VectorVT.is128BitVector();
 
  // SplatBitSize is set to the smallest size that splats the vector, so a
  // zero vector will always have SplatBitSize == 8.  However, NEON modified
  // immediate instructions others than VMOV do not support the 8-bit encoding
  // of a zero vector, and the default encoding of zero is supposed to be the
  // 32-bit version.
  if (SplatBits == 0)
    SplatBitSize = 32;
 
  switch (SplatBitSize) {
  case 8:
    if (type != VMOVModImm)
      return SDValue();
    // Any 1-byte value is OK.  Op=0, Cmode=1110.
    assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
    OpCmode = 0xe;
    Imm = SplatBits;
    VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
    break;
 
  case 16:
    // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
    VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
    if ((SplatBits & ~0xff) == 0) {
      // Value = 0x00nn: Op=x, Cmode=100x.
      OpCmode = 0x8;
      Imm = SplatBits;
      break;
    }
    if ((SplatBits & ~0xff00) == 0) {
      // Value = 0xnn00: Op=x, Cmode=101x.
      OpCmode = 0xa;
      Imm = SplatBits >> 8;
      break;
    }
    return SDValue();
 
  case 32:
    // NEON's 32-bit VMOV supports splat values where:
    // * only one byte is nonzero, or
    // * the least significant byte is 0xff and the second byte is nonzero, or
    // * the least significant 2 bytes are 0xff and the third is nonzero.
    VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
    if ((SplatBits & ~0xff) == 0) {
      // Value = 0x000000nn: Op=x, Cmode=000x.
      OpCmode = 0;
      Imm = SplatBits;
      break;
    }
    if ((SplatBits & ~0xff00) == 0) {
      // Value = 0x0000nn00: Op=x, Cmode=001x.
      OpCmode = 0x2;
      Imm = SplatBits >> 8;
      break;
    }
    if ((SplatBits & ~0xff0000) == 0) {
      // Value = 0x00nn0000: Op=x, Cmode=010x.
      OpCmode = 0x4;
      Imm = SplatBits >> 16;
      break;
    }
    if ((SplatBits & ~0xff000000) == 0) {
      // Value = 0xnn000000: Op=x, Cmode=011x.
      OpCmode = 0x6;
      Imm = SplatBits >> 24;
      break;
    }
 
    // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
    if (type == OtherModImm) return SDValue();
 
    if ((SplatBits & ~0xffff) == 0 &&
        ((SplatBits | SplatUndef) & 0xff) == 0xff) {
      // Value = 0x0000nnff: Op=x, Cmode=1100.
      OpCmode = 0xc;
      Imm = SplatBits >> 8;
      break;
    }
 
    // cmode == 0b1101 is not supported for MVE VMVN
    if (type == MVEVMVNModImm)
      return SDValue();
 
    if ((SplatBits & ~0xffffff) == 0 &&
        ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
      // Value = 0x00nnffff: Op=x, Cmode=1101.
      OpCmode = 0xd;
      Imm = SplatBits >> 16;
      break;
    }
 
    // Note: there are a few 32-bit splat values (specifically: 00ffff00,
    // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
    // VMOV.I32.  A (very) minor optimization would be to replicate the value
    // and fall through here to test for a valid 64-bit splat.  But, then the
    // caller would also need to check and handle the change in size.
    return SDValue();
 
  case 64: {
    if (type != VMOVModImm)
      return SDValue();
    // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
    uint64_t BitMask = 0xff;
    unsigned ImmMask = 1;
    Imm = 0;
    for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
      if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
        Imm |= ImmMask;
      } else if ((SplatBits & BitMask) != 0) {
        return SDValue();
      }
      BitMask <<= 8;
      ImmMask <<= 1;
    }
 
    // Op=1, Cmode=1110.
    OpCmode = 0x1e;
    VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
    break;
  }
 
  default:
    llvm_unreachable("unexpected size for isVMOVModifiedImm");
  }
 
  unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode, Imm);
  return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
}
 
SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
                                           const ARMSubtarget *ST) const {
  EVT VT = Op.getValueType();
  bool IsDouble = (VT == MVT::f64);
  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op);
  const APFloat &FPVal = CFP->getValueAPF();
 
  // Prevent floating-point constants from using literal loads
  // when execute-only is enabled.
  if (ST->genExecuteOnly()) {
    // We shouldn't trigger this for v6m execute-only
    assert((!ST->isThumb1Only() || ST->hasV8MBaselineOps()) &&
           "Unexpected architecture");
 
    // If we can represent the constant as an immediate, don't lower it
    if (isFPImmLegal(FPVal, VT))
      return Op;
    // Otherwise, construct as integer, and move to float register
    APInt INTVal = FPVal.bitcastToAPInt();
    SDLoc DL(CFP);
    switch (VT.getSimpleVT().SimpleTy) {
      default:
        llvm_unreachable("Unknown floating point type!");
        break;
      case MVT::f64: {
        SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
        SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
        return DAG.getNode(ARMISD::VMOVDRR, DL, MVT::f64, Lo, Hi);
      }
      case MVT::f32:
          return DAG.getNode(ARMISD::VMOVSR, DL, VT,
              DAG.getConstant(INTVal, DL, MVT::i32));
    }
  }
 
  if (!ST->hasVFP3Base())
    return SDValue();
 
  // Use the default (constant pool) lowering for double constants when we have
  // an SP-only FPU
  if (IsDouble && !Subtarget->hasFP64())
    return SDValue();
 
  // Try splatting with a VMOV.f32...
  int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal);
 
  if (ImmVal != -1) {
    if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
      // We have code in place to select a valid ConstantFP already, no need to
      // do any mangling.
      return Op;
    }
 
    // It's a float and we are trying to use NEON operations where
    // possible. Lower it to a splat followed by an extract.
    SDLoc DL(Op);
    SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
    SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
                                      NewVal);
    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
                       DAG.getConstant(0, DL, MVT::i32));
  }
 
  // The rest of our options are NEON only, make sure that's allowed before
  // proceeding..
  if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
    return SDValue();
 
  EVT VMovVT;
  uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
 
  // It wouldn't really be worth bothering for doubles except for one very
  // important value, which does happen to match: 0.0. So make sure we don't do
  // anything stupid.
  if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
    return SDValue();
 
  // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
  SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
                                     VMovVT, VT, VMOVModImm);
  if (NewVal != SDValue()) {
    SDLoc DL(Op);
    SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
                                      NewVal);
    if (IsDouble)
      return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
 
    // It's a float: cast and extract a vector element.
    SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
                                       VecConstant);
    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
                       DAG.getConstant(0, DL, MVT::i32));
  }
 
  // Finally, try a VMVN.i32
  NewVal = isVMOVModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
                             VT, VMVNModImm);
  if (NewVal != SDValue()) {
    SDLoc DL(Op);
    SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
 
    if (IsDouble)
      return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
 
    // It's a float: cast and extract a vector element.
    SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
                                       VecConstant);
    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
                       DAG.getConstant(0, DL, MVT::i32));
  }
 
  return SDValue();
}
 
// check if an VEXT instruction can handle the shuffle mask when the
// vector sources of the shuffle are the same.
static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
  unsigned NumElts = VT.getVectorNumElements();
 
  // Assume that the first shuffle index is not UNDEF.  Fail if it is.
  if (M[0] < 0)
    return false;
 
  Imm = M[0];
 
  // If this is a VEXT shuffle, the immediate value is the index of the first
  // element.  The other shuffle indices must be the successive elements after
  // the first one.
  unsigned ExpectedElt = Imm;
  for (unsigned i = 1; i < NumElts; ++i) {
    // Increment the expected index.  If it wraps around, just follow it
    // back to index zero and keep going.
    ++ExpectedElt;
    if (ExpectedElt == NumElts)
      ExpectedElt = 0;
 
    if (M[i] < 0) continue; // ignore UNDEF indices
    if (ExpectedElt != static_cast<unsigned>(M[i]))
      return false;
  }
 
  return true;
}
 
static bool isVEXTMask(ArrayRef<int> M, EVT VT,
                       bool &ReverseVEXT, unsigned &Imm) {
  unsigned NumElts = VT.getVectorNumElements();
  ReverseVEXT = false;
 
  // Assume that the first shuffle index is not UNDEF.  Fail if it is.
  if (M[0] < 0)
    return false;
 
  Imm = M[0];
 
  // If this is a VEXT shuffle, the immediate value is the index of the first
  // element.  The other shuffle indices must be the successive elements after
  // the first one.
  unsigned ExpectedElt = Imm;
  for (unsigned i = 1; i < NumElts; ++i) {
    // Increment the expected index.  If it wraps around, it may still be
    // a VEXT but the source vectors must be swapped.
    ExpectedElt += 1;
    if (ExpectedElt == NumElts * 2) {
      ExpectedElt = 0;
      ReverseVEXT = true;
    }
 
    if (M[i] < 0) continue; // ignore UNDEF indices
    if (ExpectedElt != static_cast<unsigned>(M[i]))
      return false;
  }
 
  // Adjust the index value if the source operands will be swapped.
  if (ReverseVEXT)
    Imm -= NumElts;
 
  return true;
}
 
static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
  // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
  // range, then 0 is placed into the resulting vector. So pretty much any mask
  // of 8 elements can work here.
  return VT == MVT::v8i8 && M.size() == 8;
}
 
static unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask,
                               unsigned Index) {
  if (Mask.size() == Elements * 2)
    return Index / Elements;
  return Mask[Index] == 0 ? 0 : 1;
}
 
// Checks whether the shuffle mask represents a vector transpose (VTRN) by
// checking that pairs of elements in the shuffle mask represent the same index
// in each vector, incrementing the expected index by 2 at each step.
// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
//  v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
//  v2={e,f,g,h}
// WhichResult gives the offset for each element in the mask based on which
// of the two results it belongs to.
//
// The transpose can be represented either as:
// result1 = shufflevector v1, v2, result1_shuffle_mask
// result2 = shufflevector v1, v2, result2_shuffle_mask
// where v1/v2 and the shuffle masks have the same number of elements
// (here WhichResult (see below) indicates which result is being checked)
//
// or as:
// results = shufflevector v1, v2, shuffle_mask
// where both results are returned in one vector and the shuffle mask has twice
// as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
// want to check the low half and high half of the shuffle mask as if it were
// the other case
static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
  unsigned EltSz = VT.getScalarSizeInBits();
  if (EltSz == 64)
    return false;
 
  unsigned NumElts = VT.getVectorNumElements();
  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
    return false;
 
  // If the mask is twice as long as the input vector then we need to check the
  // upper and lower parts of the mask with a matching value for WhichResult
  // FIXME: A mask with only even values will be rejected in case the first
  // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
  // M[0] is used to determine WhichResult
  for (unsigned i = 0; i < M.size(); i += NumElts) {
    WhichResult = SelectPairHalf(NumElts, M, i);
    for (unsigned j = 0; j < NumElts; j += 2) {
      if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
        return false;
    }
  }
 
  if (M.size() == NumElts*2)
    WhichResult = 0;
 
  return true;
}
 
/// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
  unsigned EltSz = VT.getScalarSizeInBits();
  if (EltSz == 64)
    return false;
 
  unsigned NumElts = VT.getVectorNumElements();
  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
    return false;
 
  for (unsigned i = 0; i < M.size(); i += NumElts) {
    WhichResult = SelectPairHalf(NumElts, M, i);
    for (unsigned j = 0; j < NumElts; j += 2) {
      if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
        return false;
    }
  }
 
  if (M.size() == NumElts*2)
    WhichResult = 0;
 
  return true;
}
 
// Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
// that the mask elements are either all even and in steps of size 2 or all odd
// and in steps of size 2.
// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
//  v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
//  v2={e,f,g,h}
// Requires similar checks to that of isVTRNMask with
// respect the how results are returned.
static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
  unsigned EltSz = VT.getScalarSizeInBits();
  if (EltSz == 64)
    return false;
 
  unsigned NumElts = VT.getVectorNumElements();
  if (M.size() != NumElts && M.size() != NumElts*2)
    return false;
 
  for (unsigned i = 0; i < M.size(); i += NumElts) {
    WhichResult = SelectPairHalf(NumElts, M, i);
    for (unsigned j = 0; j < NumElts; ++j) {
      if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
        return false;
    }
  }
 
  if (M.size() == NumElts*2)
    WhichResult = 0;
 
  // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
  if (VT.is64BitVector() && EltSz == 32)
    return false;
 
  return true;
}
 
/// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
  unsigned EltSz = VT.getScalarSizeInBits();
  if (EltSz == 64)
    return false;
 
  unsigned NumElts = VT.getVectorNumElements();
  if (M.size() != NumElts && M.size() != NumElts*2)
    return false;
 
  unsigned Half = NumElts / 2;
  for (unsigned i = 0; i < M.size(); i += NumElts) {
    WhichResult = SelectPairHalf(NumElts, M, i);
    for (unsigned j = 0; j < NumElts; j += Half) {
      unsigned Idx = WhichResult;
      for (unsigned k = 0; k < Half; ++k) {
        int MIdx = M[i + j + k];
        if (MIdx >= 0 && (unsigned) MIdx != Idx)
          return false;
        Idx += 2;
      }
    }
  }
 
  if (M.size() == NumElts*2)
    WhichResult = 0;
 
  // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
  if (VT.is64BitVector() && EltSz == 32)
    return false;
 
  return true;
}
 
// Checks whether the shuffle mask represents a vector zip (VZIP) by checking
// that pairs of elements of the shufflemask represent the same index in each
// vector incrementing sequentially through the vectors.
// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
//  v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
//  v2={e,f,g,h}
// Requires similar checks to that of isVTRNMask with respect the how results
// are returned.
static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
  unsigned EltSz = VT.getScalarSizeInBits();
  if (EltSz == 64)
    return false;
 
  unsigned NumElts = VT.getVectorNumElements();
  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
    return false;
 
  for (unsigned i = 0; i < M.size(); i += NumElts) {
    WhichResult = SelectPairHalf(NumElts, M, i);
    unsigned Idx = WhichResult * NumElts / 2;
    for (unsigned j = 0; j < NumElts; j += 2) {
      if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
        return false;
      Idx += 1;
    }
  }
 
  if (M.size() == NumElts*2)
    WhichResult = 0;
 
  // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
  if (VT.is64BitVector() && EltSz == 32)
    return false;
 
  return true;
}
 
/// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
  unsigned EltSz = VT.getScalarSizeInBits();
  if (EltSz == 64)
    return false;
 
  unsigned NumElts = VT.getVectorNumElements();
  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
    return false;
 
  for (unsigned i = 0; i < M.size(); i += NumElts) {
    WhichResult = SelectPairHalf(NumElts, M, i);
    unsigned Idx = WhichResult * NumElts / 2;
    for (unsigned j = 0; j < NumElts; j += 2) {
      if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
          (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
        return false;
      Idx += 1;
    }
  }
 
  if (M.size() == NumElts*2)
    WhichResult = 0;
 
  // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
  if (VT.is64BitVector() && EltSz == 32)
    return false;
 
  return true;
}
 
/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
                                           unsigned &WhichResult,
                                           bool &isV_UNDEF) {
  isV_UNDEF = false;
  if (isVTRNMask(ShuffleMask, VT, WhichResult))
    return ARMISD::VTRN;
  if (isVUZPMask(ShuffleMask, VT, WhichResult))
    return ARMISD::VUZP;
  if (isVZIPMask(ShuffleMask, VT, WhichResult))
    return ARMISD::VZIP;
 
  isV_UNDEF = true;
  if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
    return ARMISD::VTRN;
  if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
    return ARMISD::VUZP;
  if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
    return ARMISD::VZIP;
 
  return 0;
}
 
/// \return true if this is a reverse operation on an vector.
static bool isReverseMask(ArrayRef<int> M, EVT VT) {
  unsigned NumElts = VT.getVectorNumElements();
  // Make sure the mask has the right size.
  if (NumElts != M.size())
      return false;
 
  // Look for <15, ..., 3, -1, 1, 0>.
  for (unsigned i = 0; i != NumElts; ++i)
    if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
      return false;
 
  return true;
}
 
static bool isTruncMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
  unsigned NumElts = VT.getVectorNumElements();
  // Make sure the mask has the right size.
  if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
    return false;
 
  // Half-width truncation patterns (e.g. v4i32 -> v8i16):
  // !Top &&  SingleSource: <0, 2, 4, 6, 0, 2, 4, 6>
  // !Top && !SingleSource: <0, 2, 4, 6, 8, 10, 12, 14>
  //  Top &&  SingleSource: <1, 3, 5, 7, 1, 3, 5, 7>
  //  Top && !SingleSource: <1, 3, 5, 7, 9, 11, 13, 15>
  int Ofs = Top ? 1 : 0;
  int Upper = SingleSource ? 0 : NumElts;
  for (int i = 0, e = NumElts / 2; i != e; ++i) {
    if (M[i] >= 0 && M[i] != (i * 2) + Ofs)
      return false;
    if (M[i + e] >= 0 && M[i + e] != (i * 2) + Ofs + Upper)
      return false;
  }
  return true;
}
 
static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
  unsigned NumElts = VT.getVectorNumElements();
  // Make sure the mask has the right size.
  if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
    return false;
 
  // If Top
  //   Look for <0, N, 2, N+2, 4, N+4, ..>.
  //   This inserts Input2 into Input1
  // else if not Top
  //   Look for <0, N+1, 2, N+3, 4, N+5, ..>
  //   This inserts Input1 into Input2
  unsigned Offset = Top ? 0 : 1;
  unsigned N = SingleSource ? 0 : NumElts;
  for (unsigned i = 0; i < NumElts; i += 2) {
    if (M[i] >= 0 && M[i] != (int)i)
      return false;
    if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
      return false;
  }
 
  return true;
}
 
static bool isVMOVNTruncMask(ArrayRef<int> M, EVT ToVT, bool rev) {
  unsigned NumElts = ToVT.getVectorNumElements();
  if (NumElts != M.size())
    return false;
 
  // Test if the Trunc can be convertible to a VMOVN with this shuffle. We are
  // looking for patterns of:
  // !rev: 0 N/2 1 N/2+1 2 N/2+2 ...
  //  rev: N/2 0 N/2+1 1 N/2+2 2 ...
 
  unsigned Off0 = rev ? NumElts / 2 : 0;
  unsigned Off1 = rev ? 0 : NumElts / 2;
  for (unsigned i = 0; i < NumElts; i += 2) {
    if (M[i] >= 0 && M[i] != (int)(Off0 + i / 2))
      return false;
    if (M[i + 1] >= 0 && M[i + 1] != (int)(Off1 + i / 2))
      return false;
  }
 
  return true;
}
 
// Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
// from a pair of inputs. For example:
// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
//             FP_ROUND(EXTRACT_ELT(Y, 0),
//             FP_ROUND(EXTRACT_ELT(X, 1),
//             FP_ROUND(EXTRACT_ELT(Y, 1), ...)
static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
                                         const ARMSubtarget *ST) {
  assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
  if (!ST->hasMVEFloatOps())
    return SDValue();
 
  SDLoc dl(BV);
  EVT VT = BV.getValueType();
  if (VT != MVT::v8f16)
    return SDValue();
 
  // We are looking for a buildvector of fptrunc elements, where all the
  // elements are interleavingly extracted from two sources. Check the first two
  // items are valid enough and extract some info from them (they are checked
  // properly in the loop below).
  if (BV.getOperand(0).getOpcode() != ISD::FP_ROUND ||
      BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
      BV.getOperand(0).getOperand(0).getConstantOperandVal(1) != 0)
    return SDValue();
  if (BV.getOperand(1).getOpcode() != ISD::FP_ROUND ||
      BV.getOperand(1).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
      BV.getOperand(1).getOperand(0).getConstantOperandVal(1) != 0)
    return SDValue();
  SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
  SDValue Op1 = BV.getOperand(1).getOperand(0).getOperand(0);
  if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
    return SDValue();
 
  // Check all the values in the BuildVector line up with our expectations.
  for (unsigned i = 1; i < 4; i++) {
    auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
      return Trunc.getOpcode() == ISD::FP_ROUND &&
             Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
             Trunc.getOperand(0).getOperand(0) == Op &&
             Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
    };
    if (!Check(BV.getOperand(i * 2 + 0), Op0, i))
      return SDValue();
    if (!Check(BV.getOperand(i * 2 + 1), Op1, i))
      return SDValue();
  }
 
  SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0,
                           DAG.getConstant(0, dl, MVT::i32));
  return DAG.getNode(ARMISD::VCVTN, dl, VT, N1, Op1,
                     DAG.getConstant(1, dl, MVT::i32));
}
 
// Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
// from a single input on alternating lanes. For example:
// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
//             FP_ROUND(EXTRACT_ELT(X, 2),
//             FP_ROUND(EXTRACT_ELT(X, 4), ...)
static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
                                       const ARMSubtarget *ST) {
  assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
  if (!ST->hasMVEFloatOps())
    return SDValue();
 
  SDLoc dl(BV);
  EVT VT = BV.getValueType();
  if (VT != MVT::v4f32)
    return SDValue();
 
  // We are looking for a buildvector of fptext elements, where all the
  // elements are alternating lanes from a single source. For example <0,2,4,6>
  // or <1,3,5,7>. Check the first two items are valid enough and extract some
  // info from them (they are checked properly in the loop below).
  if (BV.getOperand(0).getOpcode() != ISD::FP_EXTEND ||
      BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return SDValue();
  SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
  int Offset = BV.getOperand(0).getOperand(0).getConstantOperandVal(1);
  if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
    return SDValue();
 
  // Check all the values in the BuildVector line up with our expectations.
  for (unsigned i = 1; i < 4; i++) {
    auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
      return Trunc.getOpcode() == ISD::FP_EXTEND &&
             Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
             Trunc.getOperand(0).getOperand(0) == Op &&
             Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
    };
    if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
      return SDValue();
  }
 
  return DAG.getNode(ARMISD::VCVTL, dl, VT, Op0,
                     DAG.getConstant(Offset, dl, MVT::i32));
}
 
// If N is an integer constant that can be moved into a register in one
// instruction, return an SDValue of such a constant (will become a MOV
// instruction).  Otherwise return null.
static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
                                     const ARMSubtarget *ST, const SDLoc &dl) {
  uint64_t Val;
  if (!isa<ConstantSDNode>(N))
    return SDValue();
  Val = N->getAsZExtVal();
 
  if (ST->isThumb1Only()) {
    if (Val <= 255 || ~Val <= 255)
      return DAG.getConstant(Val, dl, MVT::i32);
  } else {
    if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1)
      return DAG.getConstant(Val, dl, MVT::i32);
  }
  return SDValue();
}
 
static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
                                    const ARMSubtarget *ST) {
  SDLoc dl(Op);
  EVT VT = Op.getValueType();
 
  assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!");
 
  unsigned NumElts = VT.getVectorNumElements();
  unsigned BoolMask;
  unsigned BitsPerBool;
  if (NumElts == 2) {
    BitsPerBool = 8;
    BoolMask = 0xff;
  } else if (NumElts == 4) {
    BitsPerBool = 4;
    BoolMask = 0xf;
  } else if (NumElts == 8) {
    BitsPerBool = 2;
    BoolMask = 0x3;
  } else if (NumElts == 16) {
    BitsPerBool = 1;
    BoolMask = 0x1;
  } else
    return SDValue();
 
  // If this is a single value copied into all lanes (a splat), we can just sign
  // extend that single value
  SDValue FirstOp = Op.getOperand(0);
  if (!isa<ConstantSDNode>(FirstOp) &&
      llvm::all_of(llvm::drop_begin(Op->ops()), [&FirstOp](const SDUse &U) {
        return U.get().isUndef() || U.get() == FirstOp;
      })) {
    SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
                              DAG.getValueType(MVT::i1));
    return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), Ext);
  }
 
  // First create base with bits set where known
  unsigned Bits32 = 0;
  for (unsigned i = 0; i < NumElts; ++i) {
    SDValue V = Op.getOperand(i);
    if (!isa<ConstantSDNode>(V) && !V.isUndef())
      continue;
    bool BitSet = V.isUndef() ? false : V->getAsZExtVal();
    if (BitSet)
      Bits32 |= BoolMask << (i * BitsPerBool);
  }
 
  // Add in unknown nodes
  SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
                             DAG.getConstant(Bits32, dl, MVT::i32));
  for (unsigned i = 0; i < NumElts; ++i) {
    SDValue V = Op.getOperand(i);
    if (isa<ConstantSDNode>(V) || V.isUndef())
      continue;
    Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V,
                       DAG.getConstant(i, dl, MVT::i32));
  }
 
  return Base;
}
 
static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
                                        const ARMSubtarget *ST) {
  if (!ST->hasMVEIntegerOps())
    return SDValue();
 
  // We are looking for a buildvector where each element is Op[0] + i*N
  EVT VT = Op.getValueType();
  SDValue Op0 = Op.getOperand(0);
  unsigned NumElts = VT.getVectorNumElements();
 
  // Get the increment value from operand 1
  SDValue Op1 = Op.getOperand(1);
  if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(0) != Op0 ||
      !isa<ConstantSDNode>(Op1.getOperand(1)))
    return SDValue();
  unsigned N = Op1.getConstantOperandVal(1);
  if (N != 1 && N != 2 && N != 4 && N != 8)
    return SDValue();
 
  // Check that each other operand matches
  for (unsigned I = 2; I < NumElts; I++) {
    SDValue OpI = Op.getOperand(I);
    if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(0) != Op0 ||
        !isa<ConstantSDNode>(OpI.getOperand(1)) ||
        OpI.getConstantOperandVal(1) != I * N)
      return SDValue();
  }
 
  SDLoc DL(Op);
  return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0,
                     DAG.getConstant(N, DL, MVT::i32));
}
 
// Returns true if the operation N can be treated as qr instruction variant at
// operand Op.
static bool IsQRMVEInstruction(const SDNode *N, const SDNode *Op) {
  switch (N->getOpcode()) {
  case ISD::ADD:
  case ISD::MUL:
  case ISD::SADDSAT:
  case ISD::UADDSAT:
  case ISD::AVGFLOORS:
  case ISD::AVGFLOORU:
    return true;
  case ISD::SUB:
  case ISD::SSUBSAT:
  case ISD::USUBSAT:
    return N->getOperand(1).getNode() == Op;
  case ISD::INTRINSIC_WO_CHAIN:
    switch (N->getConstantOperandVal(0)) {
    case Intrinsic::arm_mve_add_predicated:
    case Intrinsic::arm_mve_mul_predicated:
    case Intrinsic::arm_mve_qadd_predicated:
    case Intrinsic::arm_mve_vhadd:
    case Intrinsic::arm_mve_hadd_predicated:
    case Intrinsic::arm_mve_vqdmulh:
    case Intrinsic::arm_mve_qdmulh_predicated:
    case Intrinsic::arm_mve_vqrdmulh:
    case Intrinsic::arm_mve_qrdmulh_predicated:
    case Intrinsic::arm_mve_vqdmull:
    case Intrinsic::arm_mve_vqdmull_predicated:
      return true;
    case Intrinsic::arm_mve_sub_predicated:
    case Intrinsic::arm_mve_qsub_predicated:
    case Intrinsic::arm_mve_vhsub:
    case Intrinsic::arm_mve_hsub_predicated:
      return N->getOperand(2).getNode() == Op;
    default:
      return false;
    }
  default:
    return false;
  }
}
 
// If this is a case we can't handle, return null and let the default
// expansion code take care of it.
SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
                                             const ARMSubtarget *ST) const {
  BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
  SDLoc dl(Op);
  EVT VT = Op.getValueType();
 
  if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
    return LowerBUILD_VECTOR_i1(Op, DAG, ST);
 
  if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
    return R;
 
  APInt SplatBits, SplatUndef;
  unsigned SplatBitSize;
  bool HasAnyUndefs;
  if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
    if (SplatUndef.isAllOnes())
      return DAG.getUNDEF(VT);
 
    // If all the users of this constant splat are qr instruction variants,
    // generate a vdup of the constant.
    if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == SplatBitSize &&
        (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32) &&
        all_of(BVN->users(),
               [BVN](const SDNode *U) { return IsQRMVEInstruction(U, BVN); })) {
      EVT DupVT = SplatBitSize == 32   ? MVT::v4i32
                  : SplatBitSize == 16 ? MVT::v8i16
                                       : MVT::v16i8;
      SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
      SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
      return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
    }
 
    if ((ST->hasNEON() && SplatBitSize <= 64) ||
        (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
      // Check if an immediate VMOV works.
      EVT VmovVT;
      SDValue Val =
          isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
                            SplatBitSize, DAG, dl, VmovVT, VT, VMOVModImm);
 
      if (Val.getNode()) {
        SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
        return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
      }
 
      // Try an immediate VMVN.
      uint64_t NegatedImm = (~SplatBits).getZExtValue();
      Val = isVMOVModifiedImm(
          NegatedImm, SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
          VT, ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
      if (Val.getNode()) {
        SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
        return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Vmov);
      }
 
      // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
      if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
        int ImmVal = ARM_AM::getFP32Imm(SplatBits);
        if (ImmVal != -1) {
          SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
          return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
        }
      }
 
      // If we are under MVE, generate a VDUP(constant), bitcast to the original
      // type.
      if (ST->hasMVEIntegerOps() &&
          (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32)) {
        EVT DupVT = SplatBitSize == 32   ? MVT::v4i32
                    : SplatBitSize == 16 ? MVT::v8i16
                                         : MVT::v16i8;
        SDValue Const = DAG.getConstant(SplatBits.getZExtValue(), dl, MVT::i32);
        SDValue VDup = DAG.getNode(ARMISD::VDUP, dl, DupVT, Const);
        return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, VDup);
      }
    }
  }
 
  // Scan through the operands to see if only one value is used.
  //
  // As an optimisation, even if more than one value is used it may be more
  // profitable to splat with one value then change some lanes.
  //
  // Heuristically we decide to do this if the vector has a "dominant" value,
  // defined as splatted to more than half of the lanes.
  unsigned NumElts = VT.getVectorNumElements();
  bool isOnlyLowElement = true;
  bool usesOnlyOneValue = true;
  bool hasDominantValue = false;
  bool isConstant = true;
 
  // Map of the number of times a particular SDValue appears in the
  // element list.
  DenseMap<SDValue, unsigned> ValueCounts;
  SDValue Value;
  for (unsigned i = 0; i < NumElts; ++i) {
    SDValue V = Op.getOperand(i);
    if (V.isUndef())
      continue;
    if (i > 0)
      isOnlyLowElement = false;
    if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
      isConstant = false;
 
    unsigned &Count = ValueCounts[V];
 
    // Is this value dominant? (takes up more than half of the lanes)
    if (++Count > (NumElts / 2)) {
      hasDominantValue = true;
      Value = V;
    }
  }
  if (ValueCounts.size() != 1)
    usesOnlyOneValue = false;
  if (!Value.getNode() && !ValueCounts.empty())
    Value = ValueCounts.begin()->first;
 
  if (ValueCounts.empty())
    return DAG.getUNDEF(VT);
 
  // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
  // Keep going if we are hitting this case.
  if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode()) &&
      (VT != MVT::v8f16 || ST->hasFullFP16()))
    return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
 
  unsigned EltSize = VT.getScalarSizeInBits();
 
  // Use VDUP for non-constant splats.  For f32 constant splats, reduce to
  // i32 and try again.
  if (hasDominantValue && EltSize <= 32) {
    if (!isConstant) {
      SDValue N;
 
      // If we are VDUPing a value that comes directly from a vector, that will
      // cause an unnecessary move to and from a GPR, where instead we could
      // just use VDUPLANE. We can only do this if the lane being extracted
      // is at a constant index, as the VDUP from lane instructions only have
      // constant-index forms.
      ConstantSDNode *constIndex;
      if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
          (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) {
        // We need to create a new undef vector to use for the VDUPLANE if the
        // size of the vector from which we get the value is different than the
        // size of the vector that we need to create. We will insert the element
        // such that the register coalescer will remove unnecessary copies.
        if (VT != Value->getOperand(0).getValueType()) {
          unsigned index = constIndex->getAPIntValue().getLimitedValue() %
                             VT.getVectorNumElements();
          N =  DAG.getNode(ARMISD::VDUPLANE, dl, VT,
                 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
                        Value, DAG.getConstant(index, dl, MVT::i32)),
                           DAG.getConstant(index, dl, MVT::i32));
        } else
          N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
                        Value->getOperand(0), Value->getOperand(1));
      } else
        N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
 
      if (!usesOnlyOneValue) {
        // The dominant value was splatted as 'N', but we now have to insert
        // all differing elements.
        for (unsigned I = 0; I < NumElts; ++I) {
          if (Op.getOperand(I) == Value)
            continue;
          SmallVector<SDValue, 3> Ops;
          Ops.push_back(N);
          Ops.push_back(Op.getOperand(I));
          Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
          N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
        }
      }
      return N;
    }
    if (VT.getVectorElementType().isFloatingPoint()) {
      SmallVector<SDValue, 8> Ops;
      MVT FVT = VT.getVectorElementType().getSimpleVT();
      assert(FVT == MVT::f32 || FVT == MVT::f16);
      MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
      for (unsigned i = 0; i < NumElts; ++i)
        Ops.push_back(DAG.getNode(ISD::BITCAST, dl, IVT,
                                  Op.getOperand(i)));
      EVT VecVT = EVT::getVectorVT(*DAG.getContext(), IVT, NumElts);
      SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
      Val = LowerBUILD_VECTOR(Val, DAG, ST);
      if (Val.getNode())
        return DAG.getNode(ISD::BITCAST, dl, VT, Val);
    }
    if (usesOnlyOneValue) {
      SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
      if (isConstant && Val.getNode())
        return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
    }
  }
 
  // If all elements are constants and the case above didn't get hit, fall back
  // to the default expansion, which will generate a load from the constant
  // pool.
  if (isConstant)
    return SDValue();
 
  // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
  // vmovn). Empirical tests suggest this is rarely worth it for vectors of
  // length <= 2.
  if (NumElts >= 4)
    if (SDValue shuffle = ReconstructShuffle(Op, DAG))
      return shuffle;
 
  // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
  // VCVT's
  if (SDValue VCVT = LowerBuildVectorOfFPTrunc(Op, DAG, Subtarget))
    return VCVT;
  if (SDValue VCVT = LowerBuildVectorOfFPExt(Op, DAG, Subtarget))
    return VCVT;
 
  if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
    // If we haven't found an efficient lowering, try splitting a 128-bit vector
    // into two 64-bit vectors; we might discover a better way to lower it.
    SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
    EVT ExtVT = VT.getVectorElementType();
    EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElts / 2);
    SDValue Lower = DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[0], NumElts / 2));
    if (Lower.getOpcode() == ISD::BUILD_VECTOR)
      Lower = LowerBUILD_VECTOR(Lower, DAG, ST);
    SDValue Upper =
        DAG.getBuildVector(HVT, dl, ArrayRef(&Ops[NumElts / 2], NumElts / 2));
    if (Upper.getOpcode() == ISD::BUILD_VECTOR)
      Upper = LowerBUILD_VECTOR(Upper, DAG, ST);
    if (Lower && Upper)
      return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper);
  }
 
  // Vectors with 32- or 64-bit elements can be built by directly assigning
  // the subregisters.  Lower it to an ARMISD::BUILD_VECTOR so the operands
  // will be legalized.
  if (EltSize >= 32) {
    // Do the expansion with floating-point types, since that is what the VFP
    // registers are defined to use, and since i64 is not legal.
    EVT EltVT = EVT::getFloatingPointVT(EltSize);
    EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
    SmallVector<SDValue, 8> Ops;
    for (unsigned i = 0; i < NumElts; ++i)
      Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i)));
    SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
    return DAG.getNode(ISD::BITCAST, dl, VT, Val);
  }
 
  // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
  // know the default expansion would otherwise fall back on something even
  // worse. For a vector with one or two non-undef values, that's
  // scalar_to_vector for the elements followed by a shuffle (provided the
  // shuffle is valid for the target) and materialization element by element
  // on the stack followed by a load for everything else.
  if ((!isConstant && !usesOnlyOneValue) ||
      (VT == MVT::v8f16 && !ST->hasFullFP16())) {
    SDValue Vec = DAG.getUNDEF(VT);
    for (unsigned i = 0 ; i < NumElts; ++i) {
      SDValue V = Op.getOperand(i);
      if (V.isUndef())
        continue;
      SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
      Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
    }
    return Vec;
  }
 
  return SDValue();
}
 
// Gather data to see if the operation can be modelled as a
// shuffle in combination with VEXTs.
SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
                                              SelectionDAG &DAG) const {
  assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
  SDLoc dl(Op);
  EVT VT = Op.getValueType();
  unsigned NumElts = VT.getVectorNumElements();
 
  struct ShuffleSourceInfo {
    SDValue Vec;
    unsigned MinElt = std::numeric_limits<unsigned>::max();
    unsigned MaxElt = 0;
 
    // We may insert some combination of BITCASTs and VEXT nodes to force Vec to