RISCVISelLowering.cpp

Go to the documentation of this file.

//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation  -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the interfaces that RISC-V uses to lower LLVM code into a
// selection DAG.
//
//===----------------------------------------------------------------------===//
 
#include "RISCVISelLowering.h"
#include "MCTargetDesc/RISCVMatInt.h"
#include "RISCV.h"
#include "RISCVConstantPoolValue.h"
#include "RISCVMachineFunctionInfo.h"
#include "RISCVRegisterInfo.h"
#include "RISCVSelectionDAGInfo.h"
#include "RISCVSubtarget.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
 
using namespace llvm;
 
#define DEBUG_TYPE "riscv-lower"
 
STATISTIC(NumTailCalls, "Number of tail calls");
 
static cl::opt<unsigned> ExtensionMaxWebSize(
    DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
    cl::desc("Give the maximum size (in number of nodes) of the web of "
             "instructions that we will consider for VW expansion"),
    cl::init(18));
 
static cl::opt<bool>
    AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
                     cl::desc("Allow the formation of VW_W operations (e.g., "
                              "VWADD_W) with splat constants"),
                     cl::init(false));
 
static cl::opt<unsigned> NumRepeatedDivisors(
    DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
    cl::desc("Set the minimum number of repetitions of a divisor to allow "
             "transformation to multiplications by the reciprocal"),
    cl::init(2));
 
static cl::opt<int>
    FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
              cl::desc("Give the maximum number of instructions that we will "
                       "use for creating a floating-point immediate value"),
              cl::init(3));
 
static cl::opt<bool>
    ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
                      cl::desc("Swap add and addi in cases where the add may "
                               "be combined with a shift"),
                      cl::init(true));
 
static cl::opt<int> BrMergingBaseCostThresh(
    "riscv-br-merging-base-cost", cl::init(2),
    cl::desc(
        "Sets the cost threshold for when multiple conditionals will be merged "
        "into one branch versus be split in multiple branches. Merging "
        "conditionals saves branches at the cost of additional instructions. "
        "This value sets the instruction cost limit, below which conditionals "
        "will be merged, and above which conditionals will be split. Set to -1 "
        "to never merge branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingLikelyBias(
    "riscv-br-merging-likely-bias", cl::init(0),
    cl::desc(
        "Increases 'riscv-br-merging-base-cost' in cases that it is "
        "likely that all conditionals will be executed. For example for "
        "merging the conditionals (a == b && c > d), if its known that "
        "a == b is likely, then it is likely that if the conditionals are "
        "split both sides will be executed, so it may be desirable to "
        "increase the instruction cost threshold. Set to -1 to never merge "
        "likely branches."),
    cl::Hidden);
 
static cl::opt<int> BrMergingUnlikelyBias(
    "riscv-br-merging-unlikely-bias", cl::init(-1),
    cl::desc(
        "Decreases 'riscv-br-merging-base-cost' in cases that it is unlikely "
        "that all conditionals will be executed. For example for merging "
        "the conditionals (a == b && c > d), if its known that a == b is "
        "unlikely, then it is unlikely that if the conditionals are split "
        "both sides will be executed, so it may be desirable to decrease "
        "the instruction cost threshold. Set to -1 to never merge unlikely "
        "branches."),
    cl::Hidden);
 
// TODO: Support more ops
static const unsigned ZvfbfaOps[] = {
    ISD::FNEG,        ISD::FABS,        ISD::FCOPYSIGN,   ISD::FADD,
    ISD::FSUB,        ISD::FMUL,        ISD::FMINNUM,     ISD::FMAXNUM,
    ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUM,    ISD::FMAXIMUM,
    ISD::FMA,         ISD::IS_FPCLASS,  ISD::STRICT_FADD, ISD::STRICT_FSUB,
    ISD::STRICT_FMUL, ISD::STRICT_FMA,  ISD::SETCC};
 
RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                                         const RISCVSubtarget &STI)
    : TargetLowering(TM, STI), Subtarget(STI) {
 
  RISCVABI::ABI ABI = Subtarget.getTargetABI();
  assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
 
  if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
      !Subtarget.hasStdExtF()) {
    errs() << "Hard-float 'f' ABI can't be used for a target that "
                "doesn't support the F instruction set extension (ignoring "
                          "target-abi)\n";
    ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
  } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
             !Subtarget.hasStdExtD()) {
    errs() << "Hard-float 'd' ABI can't be used for a target that "
              "doesn't support the D instruction set extension (ignoring "
              "target-abi)\n";
    ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
  }
 
  switch (ABI) {
  default:
    reportFatalUsageError("Don't know how to lower this ABI");
  case RISCVABI::ABI_ILP32:
  case RISCVABI::ABI_ILP32E:
  case RISCVABI::ABI_LP64E:
  case RISCVABI::ABI_ILP32F:
  case RISCVABI::ABI_ILP32D:
  case RISCVABI::ABI_LP64:
  case RISCVABI::ABI_LP64F:
  case RISCVABI::ABI_LP64D:
    break;
  }
 
  MVT XLenVT = Subtarget.getXLenVT();
 
  // Set up the register classes.
  addRegisterClass(XLenVT, &RISCV::GPRRegClass);
 
  if (Subtarget.hasStdExtZfhmin())
    addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
  if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
    addRegisterClass(MVT::bf16, &RISCV::FPR16RegClass);
  if (Subtarget.hasStdExtF())
    addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
  if (Subtarget.hasStdExtD())
    addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
  if (Subtarget.hasStdExtZhinxmin())
    addRegisterClass(MVT::f16, &RISCV::GPRF16RegClass);
  if (Subtarget.hasStdExtZfinx())
    addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);
  if (Subtarget.hasStdExtZdinx()) {
    if (Subtarget.is64Bit())
      addRegisterClass(MVT::f64, &RISCV::GPRRegClass);
    else
      addRegisterClass(MVT::f64, &RISCV::GPRPairRegClass);
  }
 
  static const MVT::SimpleValueType BoolVecVTs[] = {
      MVT::nxv1i1,  MVT::nxv2i1,  MVT::nxv4i1, MVT::nxv8i1,
      MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
  static const MVT::SimpleValueType IntVecVTs[] = {
      MVT::nxv1i8,  MVT::nxv2i8,   MVT::nxv4i8,   MVT::nxv8i8,  MVT::nxv16i8,
      MVT::nxv32i8, MVT::nxv64i8,  MVT::nxv1i16,  MVT::nxv2i16, MVT::nxv4i16,
      MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
      MVT::nxv4i32, MVT::nxv8i32,  MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
      MVT::nxv4i64, MVT::nxv8i64};
  static const MVT::SimpleValueType F16VecVTs[] = {
      MVT::nxv1f16, MVT::nxv2f16,  MVT::nxv4f16,
      MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
  static const MVT::SimpleValueType BF16VecVTs[] = {
      MVT::nxv1bf16, MVT::nxv2bf16,  MVT::nxv4bf16,
      MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
  static const MVT::SimpleValueType F32VecVTs[] = {
      MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
  static const MVT::SimpleValueType F64VecVTs[] = {
      MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
  static const MVT::SimpleValueType VecTupleVTs[] = {
      MVT::riscv_nxv1i8x2,  MVT::riscv_nxv1i8x3,  MVT::riscv_nxv1i8x4,
      MVT::riscv_nxv1i8x5,  MVT::riscv_nxv1i8x6,  MVT::riscv_nxv1i8x7,
      MVT::riscv_nxv1i8x8,  MVT::riscv_nxv2i8x2,  MVT::riscv_nxv2i8x3,
      MVT::riscv_nxv2i8x4,  MVT::riscv_nxv2i8x5,  MVT::riscv_nxv2i8x6,
      MVT::riscv_nxv2i8x7,  MVT::riscv_nxv2i8x8,  MVT::riscv_nxv4i8x2,
      MVT::riscv_nxv4i8x3,  MVT::riscv_nxv4i8x4,  MVT::riscv_nxv4i8x5,
      MVT::riscv_nxv4i8x6,  MVT::riscv_nxv4i8x7,  MVT::riscv_nxv4i8x8,
      MVT::riscv_nxv8i8x2,  MVT::riscv_nxv8i8x3,  MVT::riscv_nxv8i8x4,
      MVT::riscv_nxv8i8x5,  MVT::riscv_nxv8i8x6,  MVT::riscv_nxv8i8x7,
      MVT::riscv_nxv8i8x8,  MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
      MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
 
  if (Subtarget.hasVInstructions()) {
    auto addRegClassForRVV = [this](MVT VT) {
      // Disable the smallest fractional LMUL types if ELEN is less than
      // RVVBitsPerBlock.
      unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
      if (VT.getVectorMinNumElements() < MinElts)
        return;
 
      unsigned Size = VT.getSizeInBits().getKnownMinValue();
      const TargetRegisterClass *RC;
      if (Size <= RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRRegClass;
      else if (Size == 2 * RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRM2RegClass;
      else if (Size == 4 * RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRM4RegClass;
      else if (Size == 8 * RISCV::RVVBitsPerBlock)
        RC = &RISCV::VRM8RegClass;
      else
        llvm_unreachable("Unexpected size");
 
      addRegisterClass(VT, RC);
    };
 
    for (MVT VT : BoolVecVTs)
      addRegClassForRVV(VT);
    for (MVT VT : IntVecVTs) {
      if (VT.getVectorElementType() == MVT::i64 &&
          !Subtarget.hasVInstructionsI64())
        continue;
      addRegClassForRVV(VT);
    }
 
    if (Subtarget.hasVInstructionsF16Minimal() ||
        Subtarget.hasVendorXAndesVPackFPH())
      for (MVT VT : F16VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.hasVInstructionsBF16Minimal() ||
        Subtarget.hasVendorXAndesVBFHCvt())
      for (MVT VT : BF16VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.hasVInstructionsF32())
      for (MVT VT : F32VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.hasVInstructionsF64())
      for (MVT VT : F64VecVTs)
        addRegClassForRVV(VT);
 
    if (Subtarget.useRVVForFixedLengthVectors()) {
      auto addRegClassForFixedVectors = [this](MVT VT) {
        MVT ContainerVT = getContainerForFixedLengthVector(VT);
        unsigned RCID = getRegClassIDForVecVT(ContainerVT);
        const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
        addRegisterClass(VT, TRI.getRegClass(RCID));
      };
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
        if (useRVVForFixedLengthVectorVT(VT))
          addRegClassForFixedVectors(VT);
 
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
        if (useRVVForFixedLengthVectorVT(VT))
          addRegClassForFixedVectors(VT);
    }
 
    addRegisterClass(MVT::riscv_nxv1i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv1i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv2i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv4i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x2, &RISCV::VRN2M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x3, &RISCV::VRN3M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x4, &RISCV::VRN4M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x5, &RISCV::VRN5M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x6, &RISCV::VRN6M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x7, &RISCV::VRN7M1RegClass);
    addRegisterClass(MVT::riscv_nxv8i8x8, &RISCV::VRN8M1RegClass);
    addRegisterClass(MVT::riscv_nxv16i8x2, &RISCV::VRN2M2RegClass);
    addRegisterClass(MVT::riscv_nxv16i8x3, &RISCV::VRN3M2RegClass);
    addRegisterClass(MVT::riscv_nxv16i8x4, &RISCV::VRN4M2RegClass);
    addRegisterClass(MVT::riscv_nxv32i8x2, &RISCV::VRN2M4RegClass);
  }
 
  // fixed vector is stored in GPRs for P extension packed operations
  if (Subtarget.hasStdExtP()) {
    if (Subtarget.is64Bit()) {
      addRegisterClass(MVT::v2i32, &RISCV::GPRRegClass);
      addRegisterClass(MVT::v4i16, &RISCV::GPRRegClass);
      addRegisterClass(MVT::v8i8, &RISCV::GPRRegClass);
    } else {
      addRegisterClass(MVT::v2i16, &RISCV::GPRRegClass);
      addRegisterClass(MVT::v4i8, &RISCV::GPRRegClass);
 
      addRegisterClass(MVT::v2i32, &RISCV::GPRPairRegClass);
      addRegisterClass(MVT::v4i16, &RISCV::GPRPairRegClass);
      addRegisterClass(MVT::v8i8, &RISCV::GPRPairRegClass);
    }
  }
 
  // Compute derived properties from the register classes.
  computeRegisterProperties(STI.getRegisterInfo());
 
  setStackPointerRegisterToSaveRestore(RISCV::X2);
 
  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,
                   MVT::i1, Promote);
  // DAGCombiner can call isLoadExtLegal for types that aren't legal.
  setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
                   MVT::i1, Promote);
 
  // TODO: add all necessary setOperationAction calls.
  setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Custom);
 
  setOperationAction(ISD::BR_JT, MVT::Other, Expand);
  setOperationAction(ISD::BR_CC, XLenVT, Expand);
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
  setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
 
  setCondCodeAction(ISD::SETGT, XLenVT, Custom);
  setCondCodeAction(ISD::SETGE, XLenVT, Expand);
  setCondCodeAction(ISD::SETUGT, XLenVT, Custom);
  setCondCodeAction(ISD::SETUGE, XLenVT, Expand);
  if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
    setCondCodeAction(ISD::SETULE, XLenVT, Expand);
    setCondCodeAction(ISD::SETLE, XLenVT, Expand);
  }
 
  setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);
 
  setOperationAction(ISD::VASTART, MVT::Other, Custom);
  setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
 
  if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
      !Subtarget.hasVendorXAndesPerf())
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 
  setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
 
  if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
      !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
      !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
    setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
 
  if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
    setOperationAction(ISD::LOAD, MVT::i64, Custom);
    setOperationAction(ISD::STORE, MVT::i64, Custom);
  }
 
  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
 
    setOperationAction(ISD::LOAD, MVT::i32, Custom);
    setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
                       MVT::i32, Custom);
    setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Custom);
    setOperationAction({ISD::SADDO, ISD::SSUBO}, MVT::i32, Custom);
  } else if (Subtarget.hasStdExtP()) {
    // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
    setOperationAction({ISD::ADD, ISD::SUB}, MVT::i64, Custom);
    setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, MVT::i64, Custom);
  }
  if (!Subtarget.hasStdExtZmmul()) {
    setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);
  } else if (Subtarget.is64Bit()) {
    setOperationAction(ISD::MUL, MVT::i128, Custom);
    setOperationAction(ISD::MUL, MVT::i32, Custom);
  } else {
    setOperationAction(ISD::MUL, MVT::i64, Custom);
  }
 
  if (!Subtarget.hasStdExtM()) {
    setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, XLenVT,
                       Expand);
  } else if (Subtarget.is64Bit()) {
    setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},
                       {MVT::i8, MVT::i16, MVT::i32}, Custom);
  }
 
  setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, XLenVT, Expand);
 
  // On RV32, the P extension has a WMUL(U) instruction we can use for
  // (S/U)MUL_LOHI.
  // FIXME: Does P imply Zmmul?
  if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
      Subtarget.is64Bit())
    setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT, Expand);
 
  setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,
                     Custom);
 
  if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
    if (Subtarget.is64Bit())
      setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
  } else if (Subtarget.hasVendorXTHeadBb()) {
    if (Subtarget.is64Bit())
      setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
    setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom);
  } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
    setOperationAction(ISD::ROTL, XLenVT, Expand);
  } else {
    setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);
  }
 
  if (Subtarget.hasStdExtP())
    setOperationAction({ISD::FSHL, ISD::FSHR}, XLenVT, Legal);
 
  setOperationAction(ISD::BSWAP, XLenVT,
                     Subtarget.hasREV8Like() ? Legal : Expand);
 
  if (Subtarget.hasREVLike()) {
    setOperationAction(ISD::BITREVERSE, XLenVT, Legal);
  } else {
    // Zbkb can use rev8+brev8 to implement bitreverse.
    setOperationAction(ISD::BITREVERSE, XLenVT,
                       Subtarget.hasStdExtZbkb() ? Custom : Expand);
    if (Subtarget.hasStdExtZbkb())
      setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
  }
 
  if (Subtarget.hasStdExtZbb() ||
      (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
    setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,
                       Legal);
  }
 
  if (Subtarget.hasCTZLike()) {
    if (Subtarget.is64Bit())
      setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, MVT::i32, Custom);
  } else {
    setOperationAction(ISD::CTTZ, XLenVT, Expand);
  }
 
  if (!Subtarget.hasCPOPLike()) {
    // TODO: These should be set to LibCall, but this currently breaks
    //   the Linux kernel build. See #101786. Lacks i128 tests, too.
    if (Subtarget.is64Bit())
      setOperationAction(ISD::CTPOP, MVT::i128, Expand);
    else
      setOperationAction(ISD::CTPOP, MVT::i32, Expand);
    setOperationAction(ISD::CTPOP, MVT::i64, Expand);
  }
 
  if (Subtarget.hasCLZLike()) {
    // We need the custom lowering to make sure that the resulting sequence
    // for the 32bit case is efficient on 64bit targets.
    // Use default promotion for i32 without Zbb.
    if (Subtarget.is64Bit() &&
        (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
      setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, MVT::i32, Custom);
  } else {
    if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())
      setOperationAction(ISD::CTLZ_ZERO_POISON, XLenVT, Legal);
    setOperationAction(ISD::CTLZ, XLenVT, Expand);
  }
 
  if (Subtarget.hasStdExtP()) {
    setOperationAction(ISD::CTLS, XLenVT, Legal);
    if (Subtarget.is64Bit())
      setOperationAction(ISD::CTLS, MVT::i32, Custom);
  }
 
  if (Subtarget.hasStdExtP() ||
      (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
    setOperationAction(ISD::ABS, XLenVT, Legal);
    if (Subtarget.is64Bit())
      setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
  } else if (Subtarget.hasShortForwardBranchIALU()) {
    // We can use PseudoCCSUB to implement ABS.
    setOperationAction(ISD::ABS, XLenVT, Legal);
  } else if (Subtarget.is64Bit()) {
    setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
  }
 
  if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
    setOperationAction(ISD::SELECT, XLenVT, Custom);
 
  if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
      !Subtarget.is64Bit()) {
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i32, Legal);
  } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i32, Custom);
  } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i32, Custom);
  }
 
  if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
      !Subtarget.is64Bit()) {
    // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
    // pssha.w/psshl.w and extracting.
    setOperationAction(ISD::SSHLSAT, MVT::i32, Legal);
    setOperationAction(ISD::USHLSAT, MVT::i32, Legal);
  }
 
  if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
    // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
    // paadd.w, paaddu.w and extracting.
    setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, MVT::i32, Legal);
  }
 
  if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) {
    setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Legal);
    if (Subtarget.hasStdExtZbc())
      setOperationAction(ISD::CLMULR, XLenVT, Legal);
  } else if (Subtarget.hasStdExtZvbc() && Subtarget.is64Bit()) {
    // FIXME: Support i32 on RV32 by zexting from XLEN to i64 and extracting
    // half of the result (low for CLMUL, high for CLMULH).
    // TODO: Zvbc32e allows us to do a lot more here.
    setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Custom);
  }
 
  static const unsigned FPLegalNodeTypes[] = {
      ISD::FMINNUM,       ISD::FMAXNUM,        ISD::FMINIMUMNUM,
      ISD::FMAXIMUMNUM,   ISD::LRINT,          ISD::LLRINT,
      ISD::LROUND,        ISD::LLROUND,        ISD::STRICT_LRINT,
      ISD::STRICT_LLRINT, ISD::STRICT_LROUND,  ISD::STRICT_LLROUND,
      ISD::STRICT_FMA,    ISD::STRICT_FADD,    ISD::STRICT_FSUB,
      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,    ISD::STRICT_FSQRT,
      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
 
  static const ISD::CondCode FPCCToExpand[] = {
      ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
      ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
      ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
 
  static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
                                          ISD::FPOW};
  static const unsigned FPOpToLibCall[] = {ISD::FREM};
 
  static const unsigned FPRndMode[] = {
      ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
      ISD::FROUNDEVEN};
 
  static const unsigned ZfhminZfbfminPromoteOps[] = {
      ISD::FMINNUM,       ISD::FMAXNUM,        ISD::FMINIMUM,
      ISD::FMAXIMUM,      ISD::FMAXIMUMNUM,    ISD::FMINIMUMNUM,
      ISD::FADD,          ISD::FSUB,           ISD::FMUL,
      ISD::FMA,           ISD::FDIV,           ISD::FSQRT,
      ISD::STRICT_FMA,    ISD::STRICT_FADD,    ISD::STRICT_FSUB,
      ISD::STRICT_FMUL,   ISD::STRICT_FDIV,    ISD::STRICT_FSQRT,
      ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::SETCC,
      ISD::FCEIL,         ISD::FFLOOR,         ISD::FTRUNC,
      ISD::FRINT,         ISD::FROUND,         ISD::FROUNDEVEN,
      ISD::FCANONICALIZE};
 
  if (Subtarget.hasStdExtP()) {
    static const MVT P32VecVTs[] = {MVT::v2i16, MVT::v4i8};
    static const MVT P64VecVTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
    ArrayRef<MVT> VTs;
    if (Subtarget.is64Bit()) {
      VTs = P64VecVTs;
      // There's no instruction for vector shamt in P extension so we unroll to
      // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
      // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
      // widen for those operations that will be unrolled.
      setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA},
                         {MVT::v2i16, MVT::v4i8}, Custom);
      setOperationAction(ISD::INTRINSIC_WO_CHAIN, {MVT::v2i16, MVT::v4i8},
                         Custom);
      // Operand legalization queries the action using the illegal subvector.
      setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
                         Custom);
    } else {
      VTs = P32VecVTs;
    }
    // By default everything must be expanded.
    for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
      setOperationAction(Op, VTs, Expand);
 
    for (MVT VT : VTs) {
      for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
        setTruncStoreAction(VT, OtherVT, Expand);
        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                         OtherVT, Expand);
      }
    }
 
    setOperationAction({ISD::LOAD, ISD::STORE}, VTs, Legal);
    setOperationAction({ISD::ADD, ISD::SUB}, VTs, Legal);
    setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VTs, Legal);
    setOperationAction(ISD::UADDSAT, VTs, Legal);
    setOperationAction(ISD::SADDSAT, VTs, Legal);
    setOperationAction(ISD::USUBSAT, VTs, Legal);
    setOperationAction(ISD::SSUBSAT, VTs, Legal);
    setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Legal);
    setOperationAction(ISD::BITREVERSE, VTs, Legal);
    setOperationAction(ISD::VECTOR_SHUFFLE, VTs, Custom);
    setOperationAction(ISD::VECTOR_REVERSE, VTs, Legal);
    for (MVT VT : VTs) {
      if (VT != MVT::v2i32)
        setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Legal);
      if (VT.getVectorElementType() != MVT::i8) {
        setOperationAction(ISD::SSHLSAT, VT, Custom);
        setOperationAction(ISD::BSWAP, VT, Legal);
      }
    }
    setOperationAction({ISD::UNDEF, ISD::POISON}, VTs, Legal);
    setOperationAction(ISD::SPLAT_VECTOR, VTs, Legal);
    setOperationAction(ISD::BUILD_VECTOR, VTs, Legal);
    setOperationAction(ISD::SCALAR_TO_VECTOR, VTs, Legal);
    setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Custom);
    setOperationAction(ISD::BITCAST, VTs, Custom);
    setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
                       Custom);
    setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
                       Legal);
    setOperationAction(ISD::SELECT, VTs, Custom);
    setOperationAction(ISD::VSELECT, VTs, Legal);
    setOperationAction(ISD::SETCC, VTs, Legal);
    setCondCodeAction(
        {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE}, VTs,
        Expand);
    setCondCodeAction({ISD::SETNE, ISD::SETGT}, VTs, Custom);
 
    if (!Subtarget.is64Bit())
      setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v4i8}, Custom);
 
    // P extension vector comparisons produce all 1s for true, all 0s for false
    setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
    if (!Subtarget.is64Bit()) {
      // By default everything must be expanded.
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
        setOperationAction(Op, P64VecVTs, Expand);
 
      for (MVT VT : P64VecVTs) {
        for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
          setTruncStoreAction(VT, OtherVT, Expand);
          setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                           OtherVT, Expand);
        }
      }
 
      setOperationAction({ISD::UNDEF, ISD::POISON}, P64VecVTs, Legal);
      setOperationAction({ISD::LOAD, ISD::STORE}, P64VecVTs, Custom);
      setOperationAction(ISD::BITCAST, P64VecVTs, Custom);
      setOperationAction({ISD::ADD, ISD::SUB}, P64VecVTs, Legal);
      setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, {MVT::v4i16, MVT::v8i8},
                         Custom);
      setOperationAction(
          {ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, P64VecVTs,
          Legal);
      setOperationAction(ISD::INTRINSIC_WO_CHAIN, P64VecVTs, Legal);
      setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
      setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, P64VecVTs, Legal);
      setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
                         P64VecVTs, Legal);
      setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU},
                         {MVT::v4i16, MVT::v8i8}, Legal);
      setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, P64VecVTs, Custom);
      setOperationAction(ISD::SSHLSAT, {MVT::v2i32, MVT::v4i16}, Custom);
      setOperationAction(ISD::BSWAP, MVT::v4i16, Legal);
      setOperationAction(ISD::BITREVERSE, {MVT::v4i16, MVT::v8i8}, Legal);
      setOperationAction(ISD::VECTOR_SHUFFLE, P64VecVTs, Custom);
      setOperationAction(ISD::VECTOR_REVERSE, P64VecVTs, Custom);
      setOperationAction(ISD::SPLAT_VECTOR, P64VecVTs, Legal);
      setOperationAction(ISD::BUILD_VECTOR, P64VecVTs, Legal);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Legal);
      setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
                         {MVT::v4i16, MVT::v8i8}, Custom);
      setOperationAction(ISD::CONCAT_VECTORS, {MVT::v4i16, MVT::v8i8}, Legal);
      setOperationAction(ISD::EXTRACT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
                         Legal);
      setOperationAction({ISD::SELECT, ISD::VSELECT}, {MVT::v4i16, MVT::v8i8},
                         Custom);
      setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU},
                         {MVT::v4i16, MVT::v8i8}, Custom);
      setOperationAction(ISD::MUL, P32VecVTs, Custom);
      setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v4i8, Custom);
      setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v2i16, Legal);
      setOperationAction({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
                         {MVT::v4i16, MVT::v2i32}, Legal);
      setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Legal);
      setOperationAction(ISD::SETCC, P64VecVTs, Legal);
      setCondCodeAction(
          {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE},
          P64VecVTs, Expand);
      setCondCodeAction({ISD::SETNE, ISD::SETGT}, P64VecVTs, Custom);
      // Operation legalization queries the action using the result type.
      setOperationAction(ISD::INSERT_SUBVECTOR, {MVT::v4i16, MVT::v8i8},
                         Custom);
    } else {
      setOperationAction(ISD::MUL, P64VecVTs, Legal);
      setOperationAction({ISD::MULHS, ISD::MULHU}, {MVT::v2i32, MVT::v4i16},
                         Legal);
      setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v8i8, Custom);
      setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG,
                         {MVT::v4i16, MVT::v2i32}, Legal);
      setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, {MVT::v4i16, MVT::v2i32},
                         Custom);
      setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Custom);
    }
    // LegalizeVectorOps uses result VT, LegalizeDAG uses ExtVT.
    setOperationAction(
        ISD::SIGN_EXTEND_INREG,
        {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v4i16, MVT::v2i32}, Legal);
  }
 
  if (Subtarget.hasStdExtZfbfmin()) {
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
    setOperationAction(ISD::ConstantFP, MVT::bf16, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);
    setOperationAction(ISD::SELECT, MVT::bf16, Custom);
    setOperationAction(ISD::BR_CC, MVT::bf16, Expand);
    setOperationAction(ZfhminZfbfminPromoteOps, MVT::bf16, Promote);
    setOperationAction(ISD::FREM, MVT::bf16, Promote);
    setOperationAction(ISD::FABS, MVT::bf16, Custom);
    setOperationAction(ISD::FNEG, MVT::bf16, Custom);
    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom);
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
  }
 
  if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
    if (Subtarget.hasStdExtZfhOrZhinx()) {
      setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);
      setOperationAction(FPRndMode, MVT::f16,
                         Subtarget.hasStdExtZfa() ? Legal : Custom);
      setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16,
                         Subtarget.hasStdExtZfa() ? Legal : Custom);
      if (Subtarget.hasStdExtZfa())
        setOperationAction(ISD::ConstantFP, MVT::f16, Custom);
    } else {
      setOperationAction(ZfhminZfbfminPromoteOps, MVT::f16, Promote);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Promote);
      for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
                      ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
                      ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
        setOperationAction(Op, MVT::f16, Custom);
      setOperationAction(ISD::FABS, MVT::f16, Custom);
      setOperationAction(ISD::FNEG, MVT::f16, Custom);
      setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
      setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
    }
 
    if (!Subtarget.hasStdExtD()) {
      // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
      // instruction runs into double rounding issues, so this is wrong.
      // Normally we'd use an f64 fma, but without the D extension the f64 type
      // is not legal. This should probably be a libcall.
      AddPromotedToType(ISD::FMA, MVT::f16, MVT::f32);
      AddPromotedToType(ISD::STRICT_FMA, MVT::f16, MVT::f32);
    }
 
    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
 
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
    setCondCodeAction(FPCCToExpand, MVT::f16, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
    setOperationAction(ISD::SELECT, MVT::f16, Custom);
    setOperationAction(ISD::BR_CC, MVT::f16, Expand);
 
    setOperationAction(
        ISD::FNEARBYINT, MVT::f16,
        Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
    setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,
                        ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
                        ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
                        ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
                       MVT::f16, Promote);
 
    // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
    // complete support for all operations in LegalizeDAG.
    setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
                        ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
                        ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
                        ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
                       MVT::f16, Promote);
 
    // We need to custom promote this.
    if (Subtarget.is64Bit())
      setOperationAction(ISD::FPOWI, MVT::i32, Custom);
  }
 
  if (Subtarget.hasStdExtFOrZfinx()) {
    setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
    setOperationAction(FPRndMode, MVT::f32,
                       Subtarget.hasStdExtZfa() ? Legal : Custom);
    setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
    setOperationAction(ISD::SELECT, MVT::f32, Custom);
    setOperationAction(ISD::BR_CC, MVT::f32, Expand);
    setOperationAction(FPOpToExpand, MVT::f32, Expand);
    setOperationAction(FPOpToLibCall, MVT::f32, LibCall);
    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
    setTruncStoreAction(MVT::f32, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
    setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
    setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
    setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
    setOperationAction(ISD::FP_TO_BF16, MVT::f32,
                       Subtarget.isSoftFPABI() ? LibCall : Custom);
    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Custom);
 
    if (Subtarget.hasStdExtZfa()) {
      setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
      setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal);
    } else {
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom);
    }
  }
 
  if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
    setOperationAction(ISD::BITCAST, MVT::i32, Custom);
 
  if (Subtarget.hasStdExtDOrZdinx()) {
    setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
 
    if (!Subtarget.is64Bit())
      setOperationAction(ISD::BITCAST, MVT::i64, Custom);
 
    if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
        !Subtarget.is64Bit()) {
      setOperationAction(ISD::LOAD, MVT::f64, Custom);
      setOperationAction(ISD::STORE, MVT::f64, Custom);
    }
 
    if (Subtarget.hasStdExtZfa()) {
      setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
      setOperationAction(FPRndMode, MVT::f64, Legal);
      setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Legal);
    } else {
      if (Subtarget.is64Bit())
        setOperationAction(FPRndMode, MVT::f64, Custom);
 
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom);
    }
 
    setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
    setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
    setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
    setOperationAction(ISD::SELECT, MVT::f64, Custom);
    setOperationAction(ISD::BR_CC, MVT::f64, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
    setTruncStoreAction(MVT::f64, MVT::f32, Expand);
    setOperationAction(FPOpToExpand, MVT::f64, Expand);
    setOperationAction(FPOpToLibCall, MVT::f64, LibCall);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
    setTruncStoreAction(MVT::f64, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
    setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
    setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
    setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
    setOperationAction(ISD::FP_TO_BF16, MVT::f64,
                       Subtarget.isSoftFPABI() ? LibCall : Custom);
    setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
    setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
    setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
  }
 
  if (Subtarget.is64Bit()) {
    setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
                        ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
                       MVT::i32, Custom);
    setOperationAction(ISD::LROUND, MVT::i32, Custom);
  }
 
  if (Subtarget.hasStdExtFOrZfinx()) {
    setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
                       Custom);
 
    // f16/bf16 require custom handling.
    setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, XLenVT,
                       Custom);
    setOperationAction({ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, XLenVT,
                       Custom);
 
    setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
    setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
    setOperationAction(ISD::GET_FPENV, XLenVT, Custom);
    setOperationAction(ISD::SET_FPENV, XLenVT, Custom);
    setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
    setOperationAction(ISD::GET_FPMODE, XLenVT, Custom);
    setOperationAction(ISD::SET_FPMODE, XLenVT, Custom);
    setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
  }
 
  setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
                      ISD::JumpTable},
                     XLenVT, Custom);
 
  setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
 
  if (Subtarget.is64Bit())
    setOperationAction(ISD::Constant, MVT::i64, Custom);
 
  // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
  // Unfortunately this can't be determined just from the ISA naming string.
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
                     Subtarget.is64Bit() ? Legal : Custom);
  setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64,
                     Subtarget.is64Bit() ? Legal : Custom);
 
  if (Subtarget.is64Bit()) {
    setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
    setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
  }
 
  setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  if (Subtarget.is64Bit())
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
 
  if (Subtarget.hasVendorXMIPSCBOP())
    setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
  else
    setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
 
  if (Subtarget.hasStdExtZalrsc()) {
    setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
    if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
      setMinCmpXchgSizeInBits(8);
    else
      setMinCmpXchgSizeInBits(32);
  } else if (Subtarget.hasForcedAtomics()) {
    setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
  } else {
    setMaxAtomicSizeInBitsSupported(0);
  }
 
  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
 
  setBooleanContents(ZeroOrOneBooleanContent);
 
  if (getTargetMachine().getTargetTriple().isOSLinux()) {
    // Custom lowering of llvm.clear_cache.
    setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
  }
 
  if (Subtarget.hasVInstructions()) {
    setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
    setOperationAction(ISD::VSCALE, XLenVT, Custom);
 
    // RVV intrinsics may have illegal operands.
    // We also need to custom legalize vmv.x.s.
    setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
                        ISD::INTRINSIC_VOID},
                       {MVT::i8, MVT::i16}, Custom);
    if (Subtarget.is64Bit())
      setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
                         MVT::i32, Custom);
    else
      setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
                         MVT::i64, Custom);
 
    setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
                       MVT::Other, Custom);
 
    static const unsigned IntegerVPOps[] = {
        ISD::VP_SDIV,        ISD::VP_UDIV,        ISD::VP_SREM,
        ISD::VP_UREM,        ISD::VP_REDUCE_ADD,  ISD::VP_REDUCE_AND,
        ISD::VP_REDUCE_OR,   ISD::VP_REDUCE_XOR,  ISD::VP_REDUCE_SMAX,
        ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
        ISD::VP_MERGE,
        ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
        ISD::VP_CTTZ_ELTS,   ISD::VP_CTTZ_ELTS_ZERO_POISON};
 
    static const unsigned FloatingPointVPOps[] = {
        ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
        ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
        ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
 
    static const unsigned IntegerVecReduceOps[] = {
        ISD::VECREDUCE_ADD,  ISD::VECREDUCE_AND,  ISD::VECREDUCE_OR,
        ISD::VECREDUCE_XOR,  ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
        ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
 
    static const unsigned FloatingPointVecReduceOps[] = {
        ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
        ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
 
    static const unsigned FloatingPointLibCallOps[] = {
        ISD::FREM,  ISD::FPOW,   ISD::FCOS, ISD::FSIN,  ISD::FSINCOS, ISD::FEXP,
        ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
 
    if (!Subtarget.is64Bit()) {
      // We must custom-lower certain vXi64 operations on RV32 due to the vector
      // element type being illegal.
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
                         MVT::i64, Custom);
 
      setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);
 
      setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
                          ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
                          ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
                          ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
                         MVT::i64, Custom);
    }
 
    for (MVT VT : BoolVecVTs) {
      if (!isTypeLegal(VT))
        continue;
 
      setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
 
      // Mask VTs are custom-expanded into a series of standard nodes
      setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,
                          ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
                          ISD::SCALAR_TO_VECTOR},
                         VT, Custom);
 
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
                         Custom);
 
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction({ISD::SELECT_CC, ISD::VSELECT}, VT,
                         Expand);
      setOperationAction(ISD::VP_MERGE, VT, Custom);
 
      setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
                          ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON},
                         VT, Custom);
 
      setOperationAction(
          {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
          Custom);
 
      setOperationAction(
          {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
          Custom);
 
      // RVV has native int->float & float->int conversions where the
      // element type sizes are within one power-of-two of each other. Any
      // wider distances between type sizes have to be lowered as sequences
      // which progressively narrow the gap in stages.
      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
                          ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
                          ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
                          ISD::STRICT_FP_TO_UINT},
                         VT, Custom);
      setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                         Custom);
 
      // Expand all extending loads to types larger than this, and truncating
      // stores from types larger than this.
      for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
        setTruncStoreAction(VT, OtherVT, Expand);
        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                         OtherVT, Expand);
      }
 
      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
 
      setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
 
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
      setOperationPromotedToType(
          {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
          MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
    }
 
    for (MVT VT : IntVecVTs) {
      if (!isTypeLegal(VT))
        continue;
 
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
 
      // Vectors implement MULHS/MULHU.
      setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
 
      // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
      if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
        setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
 
      setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
                         Legal);
 
      if (Subtarget.hasStdExtZvabd()) {
        setOperationAction(ISD::ABS, VT, Legal);
        setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Legal);
      } else {
        setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
      }
 
      // Custom-lower extensions and truncations from/to mask types.
      setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
                         VT, Custom);
 
      // RVV has native int->float & float->int conversions where the
      // element type sizes are within one power-of-two of each other. Any
      // wider distances between type sizes have to be lowered as sequences
      // which progressively narrow the gap in stages.
      setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
                          ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
                          ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
                          ISD::STRICT_FP_TO_UINT},
                         VT, Custom);
      setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                         Custom);
      setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
                          ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
                          ISD::SSUBSAT, ISD::USUBSAT},
                         VT, Legal);
 
      // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
      // nodes which truncate by one power of two at a time.
      setOperationAction(
          {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
          Custom);
 
      // Custom-lower insert/extract operations to simplify patterns.
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
                         Custom);
 
      // Custom-lower reduction operations to set up the corresponding custom
      // nodes' operands.
      setOperationAction(IntegerVecReduceOps, VT, Custom);
 
      setOperationAction(IntegerVPOps, VT, Custom);
 
      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
 
      setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
                         VT, Custom);
 
      setOperationAction(
          {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
           ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
          VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                          ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
                         VT, Custom);
 
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
 
      setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
 
      for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
        setTruncStoreAction(VT, OtherVT, Expand);
        setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                         OtherVT, Expand);
      }
 
      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
 
      setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
                         VT, Custom);
 
      if (Subtarget.hasStdExtZvkb()) {
        setOperationAction(ISD::BSWAP, VT, Legal);
      } else {
        setOperationAction(ISD::BSWAP, VT, Expand);
        setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
      }
 
      if (Subtarget.hasStdExtZvbb()) {
        setOperationAction(ISD::BITREVERSE, VT, Legal);
      } else {
        setOperationAction(ISD::BITREVERSE, VT, Expand);
        setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);
 
        // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
        // range of f32.
        EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
        if (isTypeLegal(FloatVT)) {
          setOperationAction(
              {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
              Custom);
        }
      }
 
      if (VT.getVectorElementType() == MVT::i64) {
        if (Subtarget.hasStdExtZvbc())
          setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
      } else {
        if (Subtarget.hasStdExtZvbc32e()) {
          setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
        } else if (Subtarget.hasStdExtZvbc()) {
          // Promote to i64 if the lmul is small enough.
          // FIXME: Split if necessary to widen.
          // FIXME: Promote clmulh directly without legalizing to clmul first.
          MVT I64VecVT = MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
          if (isTypeLegal(I64VecVT))
            setOperationAction(ISD::CLMUL, VT, Custom);
        }
      }
 
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
      setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV, ISD::MASKED_UREM,
                          ISD::MASKED_SREM},
                         VT, Legal);
    }
 
    for (MVT VT : VecTupleVTs) {
      if (!isTypeLegal(VT))
        continue;
 
      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
    }
 
    // Expand various CCs to best match the RVV ISA, which natively supports UNE
    // but no other unordered comparisons, and supports all ordered comparisons
    // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
    // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
    // and we pattern-match those back to the "original", swapping operands once
    // more. This way we catch both operations and both "vf" and "fv" forms with
    // fewer patterns.
    static const ISD::CondCode VFPCCToExpand[] = {
        ISD::SETO,   ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
        ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
        ISD::SETGT,  ISD::SETOGT, ISD::SETGE,  ISD::SETOGE,
    };
 
    // TODO: support more ops.
    static const unsigned ZvfhminZvfbfminPromoteOps[] = {
        ISD::FMINNUM,
        ISD::FMAXNUM,
        ISD::FMINIMUMNUM,
        ISD::FMAXIMUMNUM,
        ISD::FADD,
        ISD::FSUB,
        ISD::FMUL,
        ISD::FMA,
        ISD::FDIV,
        ISD::FSQRT,
        ISD::FCEIL,
        ISD::FTRUNC,
        ISD::FFLOOR,
        ISD::FROUND,
        ISD::FROUNDEVEN,
        ISD::FRINT,
        ISD::FNEARBYINT,
        ISD::IS_FPCLASS,
        ISD::SETCC,
        ISD::FMAXIMUM,
        ISD::FMINIMUM,
        ISD::STRICT_FADD,
        ISD::STRICT_FSUB,
        ISD::STRICT_FMUL,
        ISD::STRICT_FDIV,
        ISD::STRICT_FSQRT,
        ISD::STRICT_FMA,
        ISD::VECREDUCE_FADD,
        ISD::VECREDUCE_FMIN,
        ISD::VECREDUCE_FMAX,
        ISD::VECREDUCE_FMINIMUM,
        ISD::VECREDUCE_FMAXIMUM,
        ISD::FCANONICALIZE};
 
    // TODO: Make more of these ops legal.
    static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
                                                ISD::FSQRT,
                                                ISD::FCEIL,
                                                ISD::FTRUNC,
                                                ISD::FFLOOR,
                                                ISD::FROUND,
                                                ISD::FROUNDEVEN,
                                                ISD::FRINT,
                                                ISD::FNEARBYINT,
                                                ISD::STRICT_FDIV,
                                                ISD::STRICT_FSQRT,
                                                ISD::VECREDUCE_FADD,
                                                ISD::VECREDUCE_FMIN,
                                                ISD::VECREDUCE_FMAX,
                                                ISD::VECREDUCE_FMINIMUM,
                                                ISD::VECREDUCE_FMAXIMUM};
 
    // TODO: support more vp ops.
    static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
        ISD::VP_REDUCE_FMIN,
        ISD::VP_REDUCE_FMAX,
        ISD::VP_REDUCE_FMINIMUM,
        ISD::VP_REDUCE_FMAXIMUM};
 
    // Sets common operation actions on RVV floating-point vector types.
    const auto SetCommonVFPActions = [&](MVT VT) {
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
      // sizes are within one power-of-two of each other. Therefore conversions
      // between vXf16 and vXf64 must be lowered as sequences which convert via
      // vXf32.
      setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
      setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
      // Custom-lower insert/extract operations to simplify patterns.
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
                         Custom);
      // Expand various condition codes (explained above).
      setCondCodeAction(VFPCCToExpand, VT, Expand);
 
      setOperationAction(
          {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
          Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
 
      setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
                          ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
                          ISD::IS_FPCLASS},
                         VT, Custom);
 
      setOperationAction(FloatingPointVecReduceOps, VT, Custom);
 
      // Expand FP operations that need libcalls.
      setOperationAction(FloatingPointLibCallOps, VT, Expand);
 
      setOperationAction(ISD::FCANONICALIZE, VT, Expand);
 
      setOperationAction(ISD::FCOPYSIGN, VT, Legal);
 
      setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
 
      setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
                         VT, Custom);
 
      setOperationAction(
          {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
           ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
          VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
 
      setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                          ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
                         VT, Custom);
 
      setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
      setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
 
      setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
                          ISD::VECTOR_SPLICE_RIGHT},
                         VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
      setOperationAction(FloatingPointVPOps, VT, Custom);
 
      setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
                         Custom);
      setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                          ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
                         VT, Legal);
      setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
                          ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
                          ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
                          ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
                         VT, Custom);
 
      setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
    };
 
    // Sets common extload/truncstore actions on RVV floating-point vector
    // types.
    const auto SetCommonVFPExtLoadTruncStoreActions =
        [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
          for (auto SmallVT : SmallerVTs) {
            setTruncStoreAction(VT, SmallVT, Expand);
            setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
          }
        };
 
    // Sets common actions for f16 and bf16 for when there's only
    // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
    const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
      setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
      setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
                         Custom);
      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
      setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
      setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
                         Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
                          ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
                          ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
                          ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
                          ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
                         VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
      MVT EltVT = VT.getVectorElementType();
      if (isTypeLegal(EltVT))
        setOperationAction({ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
                           VT, Custom);
      else
        setOperationAction(ISD::SPLAT_VECTOR, EltVT, Custom);
      setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
                          ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
                          ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                          ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
                          ISD::VP_SCATTER},
                         VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      setOperationAction(ISD::FNEG, VT, Expand);
      setOperationAction(ISD::FABS, VT, Expand);
      setOperationAction(ISD::FCOPYSIGN, VT, Expand);
 
      // Expand FP operations that need libcalls.
      setOperationAction(FloatingPointLibCallOps, VT, Expand);
 
      setOperationAction(ISD::FCANONICALIZE, VT, Expand);
 
      // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
      if (getLMUL(VT) == RISCVVType::LMUL_8) {
        setOperationAction(ZvfhminZvfbfminPromoteOps, VT, Custom);
        setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
      } else {
        MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
        setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
        setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
      }
    };
 
    // Sets common actions for zvfbfa, some of instructions are supported
    // natively so that we don't need to promote them.
    const auto SetZvfbfaActions = [&](MVT VT) {
      setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
      setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
                         Custom);
      setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
      setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
      setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
                         Custom);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
                          ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                          ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE,
                          ISD::VECTOR_INTERLEAVE, ISD::VECTOR_REVERSE,
                          ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
                          ISD::VECTOR_COMPRESS},
                         VT, Custom);
      setOperationAction(
          {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
          Legal);
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
      setOperationAction(ISD::IS_FPCLASS, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
      setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
      setOperationAction(ISD::FCOPYSIGN, VT, Legal);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                          ISD::STRICT_FMA},
                         VT, Legal);
      setCondCodeAction(VFPCCToExpand, VT, Expand);
 
      setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
                          ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
                          ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                          ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
                          ISD::VP_SCATTER},
                         VT, Custom);
      setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
      // Expand FP operations that need libcalls.
      setOperationAction(FloatingPointLibCallOps, VT, Expand);
 
      setOperationAction(ISD::FCANONICALIZE, VT, Expand);
 
      // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
      if (getLMUL(VT) == RISCVVType::LMUL_8) {
        setOperationAction(ZvfbfaPromoteOps, VT, Custom);
        setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
      } else {
        MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
        setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
        setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
      }
    };
 
    if (Subtarget.hasVInstructionsF16()) {
      for (MVT VT : F16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonVFPActions(VT);
      }
    } else if (Subtarget.hasVInstructionsF16Minimal()) {
      for (MVT VT : F16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonPromoteToF32Actions(VT);
      }
    }
 
    if (Subtarget.hasVInstructionsBF16()) {
      for (MVT VT : BF16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetZvfbfaActions(VT);
      }
    } else if (Subtarget.hasVInstructionsBF16Minimal()) {
      for (MVT VT : BF16VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonPromoteToF32Actions(VT);
      }
    }
 
    if (Subtarget.hasVInstructionsF32()) {
      for (MVT VT : F32VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonVFPActions(VT);
        SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
        SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
      }
    }
 
    if (Subtarget.hasVInstructionsF64()) {
      for (MVT VT : F64VecVTs) {
        if (!isTypeLegal(VT))
          continue;
        SetCommonVFPActions(VT);
        SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
        SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
        SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
      }
    }
 
    if (Subtarget.useRVVForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
        if (!useRVVForFixedLengthVectorVT(VT))
          continue;
 
        // By default everything must be expanded.
        for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
          setOperationAction(Op, VT, Expand);
        for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
          setTruncStoreAction(VT, OtherVT, Expand);
          setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
                           OtherVT, Expand);
        }
 
        // Custom lower fixed vector undefs to scalable vector undefs to avoid
        // expansion to a build_vector of 0s.
        setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
 
        // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
        setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
                           Custom);
 
        setOperationAction(
            {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
            Custom);
 
        setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
                           VT, Custom);
 
        setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
                           VT, Custom);
 
        setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
 
        setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
 
        setOperationAction(ISD::SETCC, VT, Custom);
 
        setOperationAction(ISD::SELECT, VT, Custom);
 
        setOperationAction(
            {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
            Custom);
 
        setOperationAction(ISD::BITCAST, VT, Custom);
 
        setOperationAction(
            {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
            Custom);
 
        setOperationAction(
            {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
            Custom);
 
        setOperationAction(
            {
                ISD::SINT_TO_FP,
                ISD::UINT_TO_FP,
                ISD::FP_TO_SINT,
                ISD::FP_TO_UINT,
                ISD::STRICT_SINT_TO_FP,
                ISD::STRICT_UINT_TO_FP,
                ISD::STRICT_FP_TO_SINT,
                ISD::STRICT_FP_TO_UINT,
            },
            VT, Custom);
        setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
                           Custom);
 
        setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
 
        // Operations below are different for between masks and other vectors.
        if (VT.getVectorElementType() == MVT::i1) {
          setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VT, Custom);
 
          setOperationAction(ISD::VP_MERGE, VT, Custom);
 
          setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
          setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
          setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
                             Custom);
          continue;
        }
 
        // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
        // it before type legalization for i64 vectors on RV32. It will then be
        // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
        // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
        // improvements first.
        if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
          setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
          setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
 
          // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
          setOperationAction(ISD::BUILD_VECTOR, MVT::i64, Custom);
        }
 
        setOperationAction(
            {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
 
        setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
                            ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                            ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
                            ISD::VP_SCATTER},
                           VT, Custom);
        setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
        setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
                            ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
                            ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
                           VT, Custom);
 
        setOperationAction(
            {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
 
        setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
 
        // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
        if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
          setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
 
        setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
                            ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
                            ISD::SSUBSAT, ISD::USUBSAT},
                           VT, Custom);
 
        setOperationAction(ISD::VSELECT, VT, Custom);
 
        setOperationAction(
            {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
 
        // Custom-lower reduction operations to set up the corresponding custom
        // nodes' operands.
        setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
                            ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
                            ISD::VECREDUCE_UMIN},
                           VT, Custom);
 
        setOperationAction(IntegerVPOps, VT, Custom);
 
        if (Subtarget.hasStdExtZvkb())
          setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom);
 
        if (Subtarget.hasStdExtZvbb()) {
          setOperationAction({ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
                              ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTPOP},
                             VT, Custom);
        } else {
          // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
          // range of f32.
          EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
          if (isTypeLegal(FloatVT))
            setOperationAction(
                {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
                Custom);
        }
 
        if (VT.getVectorElementType() == MVT::i64) {
          if (Subtarget.hasStdExtZvbc())
            setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
        } else {
          if (Subtarget.hasStdExtZvbc32e()) {
            setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Custom);
          } else if (Subtarget.hasStdExtZvbc()) {
            // Promote to i64 as is done for scalable vectors.
            MVT I64VecVT =
                MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
            if (I64VecVT.isValid() && useRVVForFixedLengthVectorVT(I64VecVT))
              setOperationAction(ISD::CLMUL, VT, Custom);
          }
        }
 
        setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
        setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV,
                            ISD::MASKED_UREM, ISD::MASKED_SREM},
                           VT, Custom);
      }
 
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
        // There are no extending loads or truncating stores.
        for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
          setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
          setTruncStoreAction(VT, InnerVT, Expand);
        }
 
        if (!useRVVForFixedLengthVectorVT(VT))
          continue;
 
        // By default everything must be expanded.
        for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
          setOperationAction(Op, VT, Expand);
 
        // Custom lower fixed vector undefs to scalable vector undefs to avoid
        // expansion to a build_vector of 0s.
        setOperationAction({ISD::UNDEF, ISD::POISON}, VT, Custom);
 
        setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
                            ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
                            ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
                            ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
                           VT, Custom);
        setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
        setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
 
        setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
                           VT, Custom);
 
        setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
                            ISD::MGATHER, ISD::MSCATTER},
                           VT, Custom);
        setOperationAction({ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
                            ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
                            ISD::EXPERIMENTAL_VP_STRIDED_STORE},
                           VT, Custom);
        setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
 
        setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
        setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
                           Custom);
 
        setOperationAction(ISD::BITCAST, VT, Custom);
 
        if (VT.getVectorElementType() == MVT::f16 &&
            !Subtarget.hasVInstructionsF16()) {
          setOperationAction(
              {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
              Custom);
          setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
          setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
          if (Subtarget.hasStdExtZfhmin()) {
            setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
          } else {
            // We need to custom legalize f16 build vectors if Zfhmin isn't
            // available.
            setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
          }
          setOperationAction(ISD::FNEG, VT, Expand);
          setOperationAction(ISD::FABS, VT, Expand);
          setOperationAction(ISD::FCOPYSIGN, VT, Expand);
          MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
          // Don't promote f16 vector operations to f32 if f32 vector type is
          // not legal.
          // Custom lower maximum LMUL case to split to 2 half LMUL operations.
          // TODO: Support more operations.
          if (!isTypeLegal(F32VecVT)) {
            setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
                                ISD::VECREDUCE_FMAXIMUM,
                                ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
                               VT, Custom);
            setOperationAction(ISD::SETCC, VT, Custom);
            continue;
          }
          setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
          setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
          continue;
        }
 
        if (VT.getVectorElementType() == MVT::bf16) {
          setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
          setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
          if (Subtarget.hasStdExtZfbfmin()) {
            setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
          } else {
            // We need to custom legalize bf16 build vectors if Zfbfmin isn't
            // available.
            setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
          }
          if (Subtarget.hasVInstructionsBF16()) {
            setOperationAction(ZvfbfaOps, VT, Custom);
            setCondCodeAction(VFPCCToExpand, VT, Expand);
          }
          setOperationAction(
              {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
              Custom);
          MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
          // Don't promote bf16 vector operations to f32 if f32 vector type is
          // not legal.
          // Custom lower maximum LMUL case to split to 2 half LMUL operations.
          // TODO: Support more operations.
          if (!isTypeLegal(F32VecVT)) {
            setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
                                ISD::VECREDUCE_FMAXIMUM,
                                ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
                               VT, Custom);
            setOperationAction(ISD::SETCC, VT, Custom);
            continue;
          }
 
          if (Subtarget.hasVInstructionsBF16())
            setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
          else
            setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
          setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
          continue;
        }
 
        setOperationAction({ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
                           Custom);
 
        setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
                            ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
                            ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
                            ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
                            ISD::FMAXIMUM, ISD::FMINIMUM},
                           VT, Custom);
 
        setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
                            ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
                            ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
                            ISD::FNEARBYINT, ISD::FCANONICALIZE},
                           VT, Custom);
 
        setCondCodeAction(VFPCCToExpand, VT, Expand);
 
        setOperationAction(ISD::SETCC, VT, Custom);
        setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
 
        setOperationAction(FloatingPointVecReduceOps, VT, Custom);
 
        setOperationAction(FloatingPointVPOps, VT, Custom);
 
        setOperationAction(
            {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
             ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
             ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
             ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
             ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
            VT, Custom);
      }
 
      // Custom-legalize bitcasts from fixed-length vectors to scalar types.
      setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32}, Custom);
      if (Subtarget.is64Bit())
        setOperationAction(ISD::BITCAST, MVT::i64, Custom);
      if (Subtarget.hasStdExtZfhminOrZhinxmin())
        setOperationAction(ISD::BITCAST, MVT::f16, Custom);
      if (Subtarget.hasStdExtZfbfmin())
        setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
      if (Subtarget.hasStdExtFOrZfinx())
        setOperationAction(ISD::BITCAST, MVT::f32, Custom);
      if (Subtarget.hasStdExtDOrZdinx())
        setOperationAction(ISD::BITCAST, MVT::f64, Custom);
    }
  }
 
  if (Subtarget.hasStdExtZaamo())
    setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand);
 
  if (Subtarget.hasForcedAtomics()) {
    // Force __sync libcalls to be emitted for atomic rmw/cas operations.
    setOperationAction(
        {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
         ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
         ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
         ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
        XLenVT, LibCall);
  }
 
  if (Subtarget.hasVendorXTHeadMemIdx()) {
    for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
      setIndexedLoadAction(im, MVT::i8, Legal);
      setIndexedStoreAction(im, MVT::i8, Legal);
      setIndexedLoadAction(im, MVT::i16, Legal);
      setIndexedStoreAction(im, MVT::i16, Legal);
      setIndexedLoadAction(im, MVT::i32, Legal);
      setIndexedStoreAction(im, MVT::i32, Legal);
 
      if (Subtarget.is64Bit()) {
        setIndexedLoadAction(im, MVT::i64, Legal);
        setIndexedStoreAction(im, MVT::i64, Legal);
      }
    }
  }
 
  if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
    setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
    setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
    setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
 
    setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
    setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
    setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
  }
 
  // zve32x is broken for partial_reduce_umla, but let's not make it worse.
  if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
    static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
                                      ISD::PARTIAL_REDUCE_UMLA,
                                      ISD::PARTIAL_REDUCE_SUMLA};
    setPartialReduceMLAAction(MLAOps, MVT::nxv1i32, MVT::nxv4i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv2i32, MVT::nxv8i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv16i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv8i32, MVT::nxv32i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv16i32, MVT::nxv64i8, Custom);
 
    // An i64 accumulator is handled by performing an i32 vdot4a* and widening
    // the result to i64 (see lowerPARTIAL_REDUCE_MLA).
    setPartialReduceMLAAction(MLAOps, MVT::nxv1i64, MVT::nxv8i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv2i64, MVT::nxv16i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv4i64, MVT::nxv32i8, Custom);
    setPartialReduceMLAAction(MLAOps, MVT::nxv8i64, MVT::nxv64i8, Custom);
 
    if (Subtarget.useRVVForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
        if ((VT.getVectorElementType() != MVT::i32 &&
             VT.getVectorElementType() != MVT::i64) ||
            !useRVVForFixedLengthVectorVT(VT))
          continue;
        ElementCount EC = VT.getVectorElementCount();
        unsigned Scale = VT.getVectorElementType() == MVT::i64 ? 8 : 4;
        MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(Scale));
        setPartialReduceMLAAction(MLAOps, VT, ArgVT, Custom);
      }
    }
  }
 
  // Customize load and store operation for bf16 if zfh isn't enabled.
  if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
    setOperationAction(ISD::LOAD, MVT::bf16, Custom);
    setOperationAction(ISD::STORE, MVT::bf16, Custom);
  }
 
  // Function alignments.
  const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
  setMinFunctionAlignment(FunctionAlignment);
  // Set preferred alignments.
  setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
  setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
 
  setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
                       ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
                       ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
                       ISD::SRA});
  setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
 
  if (Subtarget.hasStdExtFOrZfinx())
    setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
 
  // Allow scalar min/max to be combined with vector reductions.
  if (Subtarget.hasVInstructions())
    setTargetDAGCombine({ISD::UMAX, ISD::UMIN});
  if (Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
    setTargetDAGCombine({ISD::SMAX, ISD::SMIN});
 
  if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
      Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
    setTargetDAGCombine(ISD::TRUNCATE);
 
  if (Subtarget.hasStdExtZbkb())
    setTargetDAGCombine(ISD::BITREVERSE);
 
  if (Subtarget.hasStdExtFOrZfinx())
    setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
                         ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
  if (Subtarget.hasVInstructions())
    setTargetDAGCombine({ISD::FCOPYSIGN,
                         ISD::VECTOR_MATCH,
                         ISD::MGATHER,
                         ISD::MSCATTER,
                         ISD::MLOAD,
                         ISD::VP_GATHER,
                         ISD::VP_SCATTER,
                         ISD::SRL,
                         ISD::SHL,
                         ISD::STORE,
                         ISD::SPLAT_VECTOR,
                         ISD::BUILD_VECTOR,
                         ISD::CONCAT_VECTORS,
                         ISD::VP_STORE,
                         ISD::EXPERIMENTAL_VP_REVERSE,
                         ISD::SDIV,
                         ISD::UDIV,
                         ISD::SREM,
                         ISD::UREM,
                         ISD::INSERT_VECTOR_ELT,
                         ISD::ABS,
                         ISD::ABS_MIN_POISON,
                         ISD::CTPOP,
                         ISD::VECTOR_SHUFFLE,
                         ISD::FMA,
                         ISD::VSELECT,
                         ISD::VECREDUCE_ADD,
                         ISD::VECTOR_SPLICE_RIGHT});
 
  if (Subtarget.hasVendorXTHeadMemPair())
    setTargetDAGCombine({ISD::LOAD, ISD::STORE});
  if (Subtarget.useRVVForFixedLengthVectors() || Subtarget.hasStdExtP())
    setTargetDAGCombine(ISD::BITCAST);
 
  setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
 
  setMaxLargeFPConvertBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
 
  setJumpIsExpensive(Subtarget.isJumpExpensive());
 
  // Disable strict node mutation.
  IsStrictFPEnabled = true;
  EnableExtLdPromotion = true;
 
  // Let the subtarget decide if a predictable select is more expensive than the
  // corresponding branch. This information is used in CGP/SelectOpt to decide
  // when to convert selects into branches.
  PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
 
  MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
  MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
 
  MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
  MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
  MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
 
  MaxStoresPerMemmoveOptSize =
      Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
  MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
 
  MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
  MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
}
 
TargetLoweringBase::LegalizeTypeAction
RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
  if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
    if (VT == MVT::v2i16 || VT == MVT::v4i8)
      return TypeWidenVector;
 
  return TargetLoweringBase::getPreferredVectorAction(VT);
}
 
EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
                                            LLVMContext &Context,
                                            EVT VT) const {
  if (!VT.isVector())
    return getPointerTy(DL);
  if (Subtarget.hasVInstructions() &&
      (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
    return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
  return VT.changeVectorElementTypeToInteger();
}
 
TargetLoweringBase::CondMergingParams
RISCVTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
                                                   const Value *LHS,
                                                   const Value *RHS,
                                                   const Function *F) const {
  if (F->hasOptSize())
    return TargetLowering::getJumpConditionMergingParams(Opc, LHS, RHS, F);
 
  // Merging conditions eliminates a branch, so the budget we are willing to
  // spend eagerly computing the RHS condition should scale with how expensive a
  // mispredicted branch is. A branch only costs the full penalty when actually
  // mispredicted, so scale it down by an assumed misprediction rate (~25%).
  int BaseCost = Subtarget.getMispredictionPenalty() / 4;
  if (BrMergingBaseCostThresh.getNumOccurrences() > 1)
    BaseCost = BrMergingBaseCostThresh;
 
  return {BaseCost, BrMergingLikelyBias, BrMergingUnlikelyBias};
}
 
MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
  return Subtarget.getXLenVT();
}
 
// Return false if we can lower get_vector_length to a vsetvli intrinsic.
bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
                                                      unsigned VF,
                                                      bool IsScalable) const {
  if (!Subtarget.hasVInstructions())
    return true;
 
  if (!IsScalable)
    return true;
 
  if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
    return true;
 
  // Don't allow VF=1 if those types are't legal.
  if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
    return true;
 
  // VLEN=32 support is incomplete.
  if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
    return true;
 
  // The maximum VF is for the smallest element width with LMUL=8.
  // VF must be a power of 2.
  unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
  return VF > MaxVF || !isPowerOf2_32(VF);
}
 
void RISCVTargetLowering::getTgtMemIntrinsic(
    SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
    MachineFunction &MF, unsigned Intrinsic) const {
  IntrinsicInfo Info;
  auto &DL = I.getDataLayout();
 
  auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
                                 bool IsUnitStrided, bool UsePtrVal = false) {
    Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
    // We can't use ptrVal if the intrinsic can access memory before the
    // pointer. This means we can't use it for strided or indexed intrinsics.
    if (UsePtrVal)
      Info.ptrVal = I.getArgOperand(PtrOp);
    else
      Info.fallbackAddressSpace =
          I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
    Type *MemTy;
    if (IsStore) {
      // Store value is the first operand.
      MemTy = I.getArgOperand(0)->getType();
    } else {
      // Use return type. If it's segment load, return type is a struct.
      MemTy = I.getType();
      if (MemTy->isStructTy())
        MemTy = MemTy->getStructElementType(0);
    }
    if (!IsUnitStrided)
      MemTy = MemTy->getScalarType();
 
    Info.memVT = getValueType(DL, MemTy);
    if (MemTy->isTargetExtTy()) {
      // RISC-V vector tuple type's alignment type should be its element type.
      if (cast<TargetExtType>(MemTy)->getName() == "riscv.vector.tuple")
        MemTy = Type::getIntNTy(
            MemTy->getContext(),
            1 << cast<ConstantInt>(I.getArgOperand(I.arg_size() - 1))
                     ->getZExtValue());
      Info.align = DL.getABITypeAlign(MemTy);
    } else {
      Info.align = Align(DL.getTypeStoreSize(MemTy->getScalarType()));
    }
    Info.size = MemoryLocation::UnknownSize;
    Info.flags |=
        IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
    Infos.push_back(Info);
  };
 
  if (I.hasMetadata(LLVMContext::MD_nontemporal))
    Info.flags |= MachineMemOperand::MONonTemporal;
 
  Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
  switch (Intrinsic) {
  default:
    return;
  case Intrinsic::riscv_masked_atomicrmw_xchg:
  case Intrinsic::riscv_masked_atomicrmw_add:
  case Intrinsic::riscv_masked_atomicrmw_sub:
  case Intrinsic::riscv_masked_atomicrmw_nand:
  case Intrinsic::riscv_masked_atomicrmw_max:
  case Intrinsic::riscv_masked_atomicrmw_min:
  case Intrinsic::riscv_masked_atomicrmw_umax:
  case Intrinsic::riscv_masked_atomicrmw_umin:
  case Intrinsic::riscv_masked_cmpxchg:
    // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
    // narrow atomic operation. These will be expanded to an LR/SC loop that
    // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
    // will be used to modify the appropriate part of the 4 byte data and
    // preserve the rest.
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::i32;
    Info.ptrVal = I.getArgOperand(0);
    Info.offset = 0;
    Info.align = Align(4);
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
                 MachineMemOperand::MOVolatile;
    Infos.push_back(Info);
    return;
  case Intrinsic::riscv_seg2_load_mask:
  case Intrinsic::riscv_seg3_load_mask:
  case Intrinsic::riscv_seg4_load_mask:
  case Intrinsic::riscv_seg5_load_mask:
  case Intrinsic::riscv_seg6_load_mask:
  case Intrinsic::riscv_seg7_load_mask:
  case Intrinsic::riscv_seg8_load_mask:
  case Intrinsic::riscv_sseg2_load_mask:
  case Intrinsic::riscv_sseg3_load_mask:
  case Intrinsic::riscv_sseg4_load_mask:
  case Intrinsic::riscv_sseg5_load_mask:
  case Intrinsic::riscv_sseg6_load_mask:
  case Intrinsic::riscv_sseg7_load_mask:
  case Intrinsic::riscv_sseg8_load_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_seg2_store_mask:
  case Intrinsic::riscv_seg3_store_mask:
  case Intrinsic::riscv_seg4_store_mask:
  case Intrinsic::riscv_seg5_store_mask:
  case Intrinsic::riscv_seg6_store_mask:
  case Intrinsic::riscv_seg7_store_mask:
  case Intrinsic::riscv_seg8_store_mask:
    // Operands are (vec, ..., vec, ptr, mask, vl)
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_sseg2_store_mask:
  case Intrinsic::riscv_sseg3_store_mask:
  case Intrinsic::riscv_sseg4_store_mask:
  case Intrinsic::riscv_sseg5_store_mask:
  case Intrinsic::riscv_sseg6_store_mask:
  case Intrinsic::riscv_sseg7_store_mask:
  case Intrinsic::riscv_sseg8_store_mask:
    // Operands are (vec, ..., vec, ptr, offset, mask, vl)
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlm:
    SetRVVLoadStoreInfo(/*PtrOp*/ 0,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ true,
                        /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vle:
  case Intrinsic::riscv_vle_mask:
  case Intrinsic::riscv_vleff:
  case Intrinsic::riscv_vleff_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ true,
                        /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vsm:
  case Intrinsic::riscv_vse:
  case Intrinsic::riscv_vse_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ true,
                        /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlse:
  case Intrinsic::riscv_vlse_mask:
  case Intrinsic::riscv_vloxei:
  case Intrinsic::riscv_vloxei_mask:
  case Intrinsic::riscv_vluxei:
  case Intrinsic::riscv_vluxei_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vsse:
  case Intrinsic::riscv_vsse_mask:
  case Intrinsic::riscv_vsoxei:
  case Intrinsic::riscv_vsoxei_mask:
  case Intrinsic::riscv_vsuxei:
  case Intrinsic::riscv_vsuxei_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ 1,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vlseg2:
  case Intrinsic::riscv_vlseg3:
  case Intrinsic::riscv_vlseg4:
  case Intrinsic::riscv_vlseg5:
  case Intrinsic::riscv_vlseg6:
  case Intrinsic::riscv_vlseg7:
  case Intrinsic::riscv_vlseg8:
  case Intrinsic::riscv_vlseg2ff:
  case Intrinsic::riscv_vlseg3ff:
  case Intrinsic::riscv_vlseg4ff:
  case Intrinsic::riscv_vlseg5ff:
  case Intrinsic::riscv_vlseg6ff:
  case Intrinsic::riscv_vlseg7ff:
  case Intrinsic::riscv_vlseg8ff:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlseg2_mask:
  case Intrinsic::riscv_vlseg3_mask:
  case Intrinsic::riscv_vlseg4_mask:
  case Intrinsic::riscv_vlseg5_mask:
  case Intrinsic::riscv_vlseg6_mask:
  case Intrinsic::riscv_vlseg7_mask:
  case Intrinsic::riscv_vlseg8_mask:
  case Intrinsic::riscv_vlseg2ff_mask:
  case Intrinsic::riscv_vlseg3ff_mask:
  case Intrinsic::riscv_vlseg4ff_mask:
  case Intrinsic::riscv_vlseg5ff_mask:
  case Intrinsic::riscv_vlseg6ff_mask:
  case Intrinsic::riscv_vlseg7ff_mask:
  case Intrinsic::riscv_vlseg8ff_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
    return;
  case Intrinsic::riscv_vlsseg2:
  case Intrinsic::riscv_vlsseg3:
  case Intrinsic::riscv_vlsseg4:
  case Intrinsic::riscv_vlsseg5:
  case Intrinsic::riscv_vlsseg6:
  case Intrinsic::riscv_vlsseg7:
  case Intrinsic::riscv_vlsseg8:
  case Intrinsic::riscv_vloxseg2:
  case Intrinsic::riscv_vloxseg3:
  case Intrinsic::riscv_vloxseg4:
  case Intrinsic::riscv_vloxseg5:
  case Intrinsic::riscv_vloxseg6:
  case Intrinsic::riscv_vloxseg7:
  case Intrinsic::riscv_vloxseg8:
  case Intrinsic::riscv_vluxseg2:
  case Intrinsic::riscv_vluxseg3:
  case Intrinsic::riscv_vluxseg4:
  case Intrinsic::riscv_vluxseg5:
  case Intrinsic::riscv_vluxseg6:
  case Intrinsic::riscv_vluxseg7:
  case Intrinsic::riscv_vluxseg8:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vlsseg2_mask:
  case Intrinsic::riscv_vlsseg3_mask:
  case Intrinsic::riscv_vlsseg4_mask:
  case Intrinsic::riscv_vlsseg5_mask:
  case Intrinsic::riscv_vlsseg6_mask:
  case Intrinsic::riscv_vlsseg7_mask:
  case Intrinsic::riscv_vlsseg8_mask:
  case Intrinsic::riscv_vloxseg2_mask:
  case Intrinsic::riscv_vloxseg3_mask:
  case Intrinsic::riscv_vloxseg4_mask:
  case Intrinsic::riscv_vloxseg5_mask:
  case Intrinsic::riscv_vloxseg6_mask:
  case Intrinsic::riscv_vloxseg7_mask:
  case Intrinsic::riscv_vloxseg8_mask:
  case Intrinsic::riscv_vluxseg2_mask:
  case Intrinsic::riscv_vluxseg3_mask:
  case Intrinsic::riscv_vluxseg4_mask:
  case Intrinsic::riscv_vluxseg5_mask:
  case Intrinsic::riscv_vluxseg6_mask:
  case Intrinsic::riscv_vluxseg7_mask:
  case Intrinsic::riscv_vluxseg8_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
                        /*IsStore*/ false,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vsseg2:
  case Intrinsic::riscv_vsseg3:
  case Intrinsic::riscv_vsseg4:
  case Intrinsic::riscv_vsseg5:
  case Intrinsic::riscv_vsseg6:
  case Intrinsic::riscv_vsseg7:
  case Intrinsic::riscv_vsseg8:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vsseg2_mask:
  case Intrinsic::riscv_vsseg3_mask:
  case Intrinsic::riscv_vsseg4_mask:
  case Intrinsic::riscv_vsseg5_mask:
  case Intrinsic::riscv_vsseg6_mask:
  case Intrinsic::riscv_vsseg7_mask:
  case Intrinsic::riscv_vsseg8_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vssseg2:
  case Intrinsic::riscv_vssseg3:
  case Intrinsic::riscv_vssseg4:
  case Intrinsic::riscv_vssseg5:
  case Intrinsic::riscv_vssseg6:
  case Intrinsic::riscv_vssseg7:
  case Intrinsic::riscv_vssseg8:
  case Intrinsic::riscv_vsoxseg2:
  case Intrinsic::riscv_vsoxseg3:
  case Intrinsic::riscv_vsoxseg4:
  case Intrinsic::riscv_vsoxseg5:
  case Intrinsic::riscv_vsoxseg6:
  case Intrinsic::riscv_vsoxseg7:
  case Intrinsic::riscv_vsoxseg8:
  case Intrinsic::riscv_vsuxseg2:
  case Intrinsic::riscv_vsuxseg3:
  case Intrinsic::riscv_vsuxseg4:
  case Intrinsic::riscv_vsuxseg5:
  case Intrinsic::riscv_vsuxseg6:
  case Intrinsic::riscv_vsuxseg7:
  case Intrinsic::riscv_vsuxseg8:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_vssseg2_mask:
  case Intrinsic::riscv_vssseg3_mask:
  case Intrinsic::riscv_vssseg4_mask:
  case Intrinsic::riscv_vssseg5_mask:
  case Intrinsic::riscv_vssseg6_mask:
  case Intrinsic::riscv_vssseg7_mask:
  case Intrinsic::riscv_vssseg8_mask:
  case Intrinsic::riscv_vsoxseg2_mask:
  case Intrinsic::riscv_vsoxseg3_mask:
  case Intrinsic::riscv_vsoxseg4_mask:
  case Intrinsic::riscv_vsoxseg5_mask:
  case Intrinsic::riscv_vsoxseg6_mask:
  case Intrinsic::riscv_vsoxseg7_mask:
  case Intrinsic::riscv_vsoxseg8_mask:
  case Intrinsic::riscv_vsuxseg2_mask:
  case Intrinsic::riscv_vsuxseg3_mask:
  case Intrinsic::riscv_vsuxseg4_mask:
  case Intrinsic::riscv_vsuxseg5_mask:
  case Intrinsic::riscv_vsuxseg6_mask:
  case Intrinsic::riscv_vsuxseg7_mask:
  case Intrinsic::riscv_vsuxseg8_mask:
    SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
                        /*IsStore*/ true,
                        /*IsUnitStrided*/ false);
    return;
  case Intrinsic::riscv_sf_vlte8:
  case Intrinsic::riscv_sf_vlte16:
  case Intrinsic::riscv_sf_vlte32:
  case Intrinsic::riscv_sf_vlte64:
    Info.opc = ISD::INTRINSIC_VOID;
    Info.ptrVal = I.getArgOperand(1);
    switch (Intrinsic) {
    case Intrinsic::riscv_sf_vlte8:
      Info.memVT = MVT::i8;
      Info.align = Align(1);
      break;
    case Intrinsic::riscv_sf_vlte16:
      Info.memVT = MVT::i16;
      Info.align = Align(2);
      break;
    case Intrinsic::riscv_sf_vlte32:
      Info.memVT = MVT::i32;
      Info.align = Align(4);
      break;
    case Intrinsic::riscv_sf_vlte64:
      Info.memVT = MVT::i64;
      Info.align = Align(8);
      break;
    }
    Info.size = MemoryLocation::UnknownSize;
    Info.flags |= MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  case Intrinsic::riscv_sf_vste8:
  case Intrinsic::riscv_sf_vste16:
  case Intrinsic::riscv_sf_vste32:
  case Intrinsic::riscv_sf_vste64:
    Info.opc = ISD::INTRINSIC_VOID;
    Info.ptrVal = I.getArgOperand(1);
    switch (Intrinsic) {
    case Intrinsic::riscv_sf_vste8:
      Info.memVT = MVT::i8;
      Info.align = Align(1);
      break;
    case Intrinsic::riscv_sf_vste16:
      Info.memVT = MVT::i16;
      Info.align = Align(2);
      break;
    case Intrinsic::riscv_sf_vste32:
      Info.memVT = MVT::i32;
      Info.align = Align(4);
      break;
    case Intrinsic::riscv_sf_vste64:
      Info.memVT = MVT::i64;
      Info.align = Align(8);
      break;
    }
    Info.size = MemoryLocation::UnknownSize;
    Info.flags |= MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
}
 
bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                                const AddrMode &AM, Type *Ty,
                                                unsigned AS,
                                                Instruction *I) const {
  // No global is ever allowed as a base.
  if (AM.BaseGV)
    return false;
 
  // None of our addressing modes allows a scalable offset
  if (AM.ScalableOffset)
    return false;
 
  // RVV instructions only support register addressing.
  if (Subtarget.hasVInstructions() && isa<VectorType>(Ty))
    return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
 
  // The Xqcilo extension provides load/store instructions with a 26-bit signed
  // offset.
  if (Subtarget.hasVendorXqcilo()) {
    if (!isInt<26>(AM.BaseOffs))
      return false;
  } else if (!isInt<12>(AM.BaseOffs)) {
    // Otherwise require a 12-bit signed offset.
    return false;
  }
 
  switch (AM.Scale) {
  case 0: // "r+i" or just "i", depending on HasBaseReg.
    break;
  case 1:
    if (!AM.HasBaseReg) // allow "r+i".
      break;
    return false; // disallow "r+r" or "r+r+i".
  default:
    return false;
  }
 
  return true;
}
 
bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
  return isInt<12>(Imm);
}
 
bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
  // The Xqcilia extension provides add-immediate instructions with a 26-bit
  // signed immediate.
  if (Subtarget.hasVendorXqcilia())
    return isInt<26>(Imm);
  return isInt<12>(Imm);
}
 
// On RV32, 64-bit integers are split into their high and low parts and held
// in two different registers, so the trunc is free since the low register can
// just be used.
// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
// isTruncateFree?
bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
  if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
    return false;
  unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
  unsigned DestBits = DstTy->getPrimitiveSizeInBits();
  return (SrcBits == 64 && DestBits == 32);
}
 
bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
  // We consider i64->i32 free on RV64 since we have good selection of W
  // instructions that make promoting operations back to i64 free in many cases.
  if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
      !DstVT.isInteger())
    return false;
  unsigned SrcBits = SrcVT.getSizeInBits();
  unsigned DestBits = DstVT.getSizeInBits();
  return (SrcBits == 64 && DestBits == 32);
}
 
bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
  EVT SrcVT = Val.getValueType();
  // free truncate from vnsrl and vnsra
  if (Subtarget.hasVInstructions() &&
      (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
      SrcVT.isVector() && VT2.isVector()) {
    unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
    unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
    if (SrcBits == DestBits * 2) {
      return true;
    }
  }
  return TargetLowering::isTruncateFree(Val, VT2);
}
 
bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
  // Zexts are free if they can be combined with a load.
  // Don't advertise i32->i64 zextload as being free for RV64. It interacts
  // poorly with type legalization of compares preferring sext.
  if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
    EVT MemVT = LD->getMemoryVT();
    if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
        (LD->getExtensionType() == ISD::NON_EXTLOAD ||
         LD->getExtensionType() == ISD::ZEXTLOAD))
      return true;
  }
 
  return TargetLowering::isZExtFree(Val, VT2);
}
 
bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
  return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
}
 
bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
  return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
}
 
bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
  return Subtarget.hasCTZLike();
}
 
bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
  return Subtarget.hasCLZLike();
}
 
bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
    const Instruction &AndI) const {
  // We expect to be able to match a bit extraction instruction if the Zbs
  // extension is supported and the mask is a power of two. However, we
  // conservatively return false if the mask would fit in an ANDI instruction,
  // on the basis that it's possible the sinking+duplication of the AND in
  // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
  // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
  if (!Subtarget.hasBEXTILike())
    return false;
  ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
  if (!Mask)
    return false;
  return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
}
 
bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
  EVT VT = Y.getValueType();
 
  if (VT.isVector())
    return false;
 
  return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
         (!isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque());
}
 
bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
  EVT VT = Y.getValueType();
 
  if (!VT.isVector())
    return hasAndNotCompare(Y);
 
  return Subtarget.hasStdExtZvkb();
}
 
bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
  // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
  if (Subtarget.hasStdExtZbs())
    return X.getValueType().isScalarInteger();
  auto *C = dyn_cast<ConstantSDNode>(Y);
  // XTheadBs provides th.tst (similar to bexti), if Y is a constant
  if (Subtarget.hasVendorXTHeadBs())
    return C != nullptr;
  // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
  return C && C->getAPIntValue().ule(10);
}
 
bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
    unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
    SDValue Y) const {
  if (SelectOpcode != ISD::VSELECT)
    return false;
 
  // Only enable for rvv.
  if (!VT.isVector() || !Subtarget.hasVInstructions())
    return false;
 
  if (VT.isFixedLengthVector() && !isTypeLegal(VT))
    return false;
 
  return true;
}
 
bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                                            Type *Ty) const {
  assert(Ty->isIntegerTy());
 
  unsigned BitSize = Ty->getIntegerBitWidth();
  if (BitSize > Subtarget.getXLen())
    return false;
 
  // Fast path, assume 32-bit immediates are cheap.
  int64_t Val = Imm.getSExtValue();
  if (isInt<32>(Val))
    return true;
 
  // A constant pool entry may be more aligned than the load we're trying to
  // replace. If we don't support unaligned scalar mem, prefer the constant
  // pool.
  // TODO: Can the caller pass down the alignment?
  if (!Subtarget.enableUnalignedScalarMem())
    return true;
 
  // Prefer to keep the load if it would require many instructions.
  // This uses the same threshold we use for constant pools but doesn't
  // check useConstantPoolForLargeInts.
  // TODO: Should we keep the load only when we're definitely going to emit a
  // constant pool?
 
  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, Subtarget);
  return Seq.size() <= Subtarget.getMaxBuildIntsCost();
}
 
bool RISCVTargetLowering::
    shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
        SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
        unsigned OldShiftOpcode, unsigned NewShiftOpcode,
        SelectionDAG &DAG) const {
  // One interesting pattern that we'd want to form is 'bit extract':
  //   ((1 >> Y) & 1) ==/!= 0
  // But we also need to be careful not to try to reverse that fold.
 
  // Is this '((1 >> Y) & 1)'?
  if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
    return false; // Keep the 'bit extract' pattern.
 
  // Will this be '((1 >> Y) & 1)' after the transform?
  if (NewShiftOpcode == ISD::SRL && CC->isOne())
    return true; // Do form the 'bit extract' pattern.
 
  // If 'X' is a constant, and we transform, then we will immediately
  // try to undo the fold, thus causing endless combine loop.
  // So only do the transform if X is not a constant. This matches the default
  // implementation of this function.
  return !XC;
}
 
bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
  unsigned Opc = VecOp.getOpcode();
 
  // Assume target opcodes can't be scalarized.
  // TODO - do we have any exceptions?
  if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
    return false;
 
  // If the vector op is not supported, try to convert to scalar.
  EVT VecVT = VecOp.getValueType();
  if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
    return true;
 
  // If the vector op is supported, but the scalar op is not, the transform may
  // not be worthwhile.
  // Permit a vector binary operation can be converted to scalar binary
  // operation which is custom lowered with illegal type.
  EVT ScalarVT = VecVT.getScalarType();
  return isOperationLegalOrCustomOrPromote(Opc, ScalarVT) ||
         isOperationCustom(Opc, ScalarVT);
}
 
bool RISCVTargetLowering::isOffsetFoldingLegal(
    const GlobalAddressSDNode *GA) const {
  // In order to maximise the opportunity for common subexpression elimination,
  // keep a separate ADD node for the global address offset instead of folding
  // it in the global address node. Later peephole optimisations may choose to
  // fold it back in when profitable.
  return false;
}
 
// Returns 0-31 if the fli instruction is available for the type and this is
// legal FP immediate for the type. Returns -1 otherwise.
int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
  if (!Subtarget.hasStdExtZfa())
    return -1;
 
  bool IsSupportedVT = false;
  if (VT == MVT::f16) {
    IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
  } else if (VT == MVT::f32) {
    IsSupportedVT = true;
  } else if (VT == MVT::f64) {
    assert(Subtarget.hasStdExtD() && "Expect D extension");
    IsSupportedVT = true;
  }
 
  if (!IsSupportedVT)
    return -1;
 
  return RISCVLoadFPImm::getLoadFPImm(Imm);
}
 
bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                       bool ForCodeSize) const {
  bool IsLegalVT = false;
  if (VT == MVT::f16)
    IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
  else if (VT == MVT::f32)
    IsLegalVT = Subtarget.hasStdExtFOrZfinx();
  else if (VT == MVT::f64)
    IsLegalVT = Subtarget.hasStdExtDOrZdinx();
  else if (VT == MVT::bf16)
    IsLegalVT = Subtarget.hasStdExtZfbfmin();
 
  if (!IsLegalVT)
    return false;
 
  if (getLegalZfaFPImm(Imm, VT) >= 0)
    return true;
 
  // Some constants can be produced by fli+fneg.
  if (Imm.isNegative() && getLegalZfaFPImm(-Imm, VT) >= 0)
    return true;
 
  // Cannot create a 64 bit floating-point immediate value for rv32.
  if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
    // td can handle +0.0 or -0.0 already.
    // -0.0 can be created by fmv + fneg.
    return Imm.isZero();
  }
 
  // Special case: fmv + fneg
  if (Imm.isNegZero())
    return true;
 
  // Building an integer and then converting requires a fmv at the end of
  // the integer sequence. The fmv is not required for Zfinx.
  const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
  const int Cost =
      FmvCost + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(),
                                           Subtarget.getXLen(), Subtarget);
  return Cost <= FPImmCost;
}
 
// TODO: This is very conservative.
TargetLowering::ExtractSubvectorCost
RISCVTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
                                             unsigned Index) const {
  if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
      (ResVT == MVT::v4i8 || ResVT == MVT::v2i16)) {
    if ((Index % ResVT.getVectorNumElements()) == 0)
      return ExtractSubvectorCost::Free;
    return ExtractSubvectorCost::Expensive;
  }
 
  if (!Subtarget.hasVInstructions())
    return ExtractSubvectorCost::Expensive;
 
  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
    return ExtractSubvectorCost::Expensive;
 
  // Extracts from index 0 are just subreg extracts.
  if (Index == 0)
    return ExtractSubvectorCost::Free;
 
  // Only support extracting a fixed from a fixed vector for now.
  if (ResVT.isScalableVector() || SrcVT.isScalableVector())
    return ExtractSubvectorCost::Expensive;
 
  EVT EltVT = ResVT.getVectorElementType();
  assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
 
  // The smallest type we can slide is i8.
  if (EltVT == MVT::i1)
    return ExtractSubvectorCost::Expensive;
 
  unsigned ResElts = ResVT.getVectorNumElements();
  unsigned SrcElts = SrcVT.getVectorNumElements();
 
  unsigned MinVLen = Subtarget.getRealMinVLen();
  unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
 
  // If we're extracting only data from the first VLEN bits of the source
  // then we can always do this with an m1 vslidedown.vx.  Restricting the
  // Index ensures we can use a vslidedown.vi.
  // TODO: We can generalize this when the exact VLEN is known.
  if (Index + ResElts <= MinVLMAX && Index < 31)
    return ExtractSubvectorCost::Free;
 
  // Convervatively only handle extracting half of a vector.
  // TODO: We can do arbitrary slidedowns, but for now only support extracting
  // the upper half of a vector until we have more test coverage.
  // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
  // a cheap extract.  However, this case is important in practice for
  // shuffled extracts of longer vectors.  How resolve?
  if ((ResElts * 2) == SrcElts && Index == ResElts)
    return ExtractSubvectorCost::Free;
  return ExtractSubvectorCost::Expensive;
}
 
MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                      CallingConv::ID CC,
                                                      EVT VT) const {
  // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
  // We might still end up using a GPR but that will be decided based on ABI.
  if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
      !Subtarget.hasStdExtZfhminOrZhinxmin())
    return MVT::f32;
 
  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
 
unsigned
RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
                                     std::optional<MVT> RegisterVT) const {
  // Pair inline assembly operand
  if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
      *RegisterVT == MVT::Untyped)
    return 1;
 
  return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
}
 
unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                           CallingConv::ID CC,
                                                           EVT VT) const {
  // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
  // We might still end up using a GPR but that will be decided based on ABI.
  if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
      !Subtarget.hasStdExtZfhminOrZhinxmin())
    return 1;
 
  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
}
 
// Changes the condition code and swaps operands if necessary, so the SetCC
// operation matches one of the comparisons supported directly by branches
// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
// with 1/-1.
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
                                    ISD::CondCode &CC, SelectionDAG &DAG,
                                    const RISCVSubtarget &Subtarget) {
  // If this is a single bit test that can't be handled by ANDI, shift the
  // bit to be tested to the MSB and perform a signed compare with 0.
  if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
      LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
      isa<ConstantSDNode>(LHS.getOperand(1)) &&
      // XAndesPerf supports branch on test bit.
      !Subtarget.hasVendorXAndesPerf()) {
    uint64_t Mask = LHS.getConstantOperandVal(1);
    if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
      unsigned ShAmt = 0;
      if (isPowerOf2_64(Mask)) {
        CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
        ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
      } else {
        ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
      }
 
      LHS = LHS.getOperand(0);
      if (ShAmt != 0)
        LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
                          DAG.getConstant(ShAmt, DL, LHS.getValueType()));
      return;
    }
  }
 
  if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
    int64_t C = RHSC->getSExtValue();
    switch (CC) {
    default: break;
    case ISD::SETGT:
      // Convert X > -1 to X >= 0.
      if (C == -1) {
        RHS = DAG.getConstant(0, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
          C != INT64_MAX && isInt<5>(C + 1)) {
        // We have a conditional move instruction for SETGE but not SETGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
        RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(C + 1)) {
        // We have a branch immediate instruction for SETGE but not SETGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
        RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      break;
    case ISD::SETLT:
      // Convert X < 1 to 0 >= X.
      if (C == 1) {
        RHS = LHS;
        LHS = DAG.getConstant(0, DL, RHS.getValueType());
        CC = ISD::SETGE;
        return;
      }
      break;
    case ISD::SETUGT:
      if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
          C != INT64_MAX && isUInt<5>(C + 1)) {
        // We have a conditional move instruction for SETUGE but not SETUGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
        RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETUGE;
        return;
      }
      if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(C + 1)) {
        // We have a branch immediate instruction for SETUGE but not SETUGT.
        // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
        // immediate.
        RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
        CC = ISD::SETUGE;
        return;
      }
      break;
    }
  }
 
  switch (CC) {
  default:
    break;
  case ISD::SETGT:
  case ISD::SETLE:
  case ISD::SETUGT:
  case ISD::SETULE:
    CC = ISD::getSetCCSwappedOperands(CC);
    std::swap(LHS, RHS);
    break;
  }
}
 
RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
  if (VT.isRISCVVectorTuple()) {
    if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv1i8x8)
      return RISCVVType::LMUL_F8;
    if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv2i8x8)
      return RISCVVType::LMUL_F4;
    if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv4i8x8)
      return RISCVVType::LMUL_F2;
    if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv8i8x8)
      return RISCVVType::LMUL_1;
    if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
        VT.SimpleTy <= MVT::riscv_nxv16i8x4)
      return RISCVVType::LMUL_2;
    if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
      return RISCVVType::LMUL_4;
    llvm_unreachable("Invalid vector tuple type LMUL.");
  }
 
  assert(VT.isScalableVector() && "Expecting a scalable vector type");
  unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
  if (VT.getVectorElementType() == MVT::i1)
    KnownSize *= 8;
 
  switch (KnownSize) {
  default:
    llvm_unreachable("Invalid LMUL.");
  case 8:
    return RISCVVType::LMUL_F8;
  case 16:
    return RISCVVType::LMUL_F4;
  case 32:
    return RISCVVType::LMUL_F2;
  case 64:
    return RISCVVType::LMUL_1;
  case 128:
    return RISCVVType::LMUL_2;
  case 256:
    return RISCVVType::LMUL_4;
  case 512:
    return RISCVVType::LMUL_8;
  }
}
 
unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
  switch (LMul) {
  default:
    llvm_unreachable("Invalid LMUL.");
  case RISCVVType::LMUL_F8:
  case RISCVVType::LMUL_F4:
  case RISCVVType::LMUL_F2:
  case RISCVVType::LMUL_1:
    return RISCV::VRRegClassID;
  case RISCVVType::LMUL_2:
    return RISCV::VRM2RegClassID;
  case RISCVVType::LMUL_4:
    return RISCV::VRM4RegClassID;
  case RISCVVType::LMUL_8:
    return RISCV::VRM8RegClassID;
  }
}
 
unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
  RISCVVType::VLMUL LMUL = getLMUL(VT);
  if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
      LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
    static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
                  "Unexpected subreg numbering");
    return RISCV::sub_vrm1_0 + Index;
  }
  if (LMUL == RISCVVType::LMUL_2) {
    static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
                  "Unexpected subreg numbering");
    return RISCV::sub_vrm2_0 + Index;
  }
  if (LMUL == RISCVVType::LMUL_4) {
    static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
                  "Unexpected subreg numbering");
    return RISCV::sub_vrm4_0 + Index;
  }
  llvm_unreachable("Invalid vector type.");
}
 
unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
  if (VT.isRISCVVectorTuple()) {
    unsigned NF = VT.getRISCVVectorTupleNumFields();
    unsigned RegsPerField =
        std::max(1U, (unsigned)VT.getSizeInBits().getKnownMinValue() /
                         (NF * RISCV::RVVBitsPerBlock));
    switch (RegsPerField) {
    case 1:
      if (NF == 2)
        return RISCV::VRN2M1RegClassID;
      if (NF == 3)
        return RISCV::VRN3M1RegClassID;
      if (NF == 4)
        return RISCV::VRN4M1RegClassID;
      if (NF == 5)
        return RISCV::VRN5M1RegClassID;
      if (NF == 6)
        return RISCV::VRN6M1RegClassID;
      if (NF == 7)
        return RISCV::VRN7M1RegClassID;
      if (NF == 8)
        return RISCV::VRN8M1RegClassID;
      break;
    case 2:
      if (NF == 2)
        return RISCV::VRN2M2RegClassID;
      if (NF == 3)
        return RISCV::VRN3M2RegClassID;
      if (NF == 4)
        return RISCV::VRN4M2RegClassID;
      break;
    case 4:
      assert(NF == 2);
      return RISCV::VRN2M4RegClassID;
    default:
      break;
    }
    llvm_unreachable("Invalid vector tuple type RegClass.");
  }
 
  if (VT.getVectorElementType() == MVT::i1)
    return RISCV::VRRegClassID;
  return getRegClassIDForLMUL(getLMUL(VT));
}
 
// Attempt to decompose a subvector insert/extract between VecVT and
// SubVecVT via subregister indices. Returns the subregister index that
// can perform the subvector insert/extract with the given element index, as
// well as the index corresponding to any leftover subvectors that must be
// further inserted/extracted within the register class for SubVecVT.
std::pair<unsigned, unsigned>
RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
    MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
    const RISCVRegisterInfo *TRI) {
  static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
                 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
                 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
                "Register classes not ordered");
  unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
  unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
 
  // If VecVT is a vector tuple type, either it's the tuple type with same
  // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
  if (VecVT.isRISCVVectorTuple()) {
    if (VecRegClassID == SubRegClassID)
      return {RISCV::NoSubRegister, 0};
 
    assert(SubVecVT.isScalableVector() &&
           "Only allow scalable vector subvector.");
    assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
           "Invalid vector tuple insert/extract for vector and subvector with "
           "different LMUL.");
    return {getSubregIndexByMVT(VecVT, InsertExtractIdx), 0};
  }
 
  // Try to compose a subregister index that takes us from the incoming
  // LMUL>1 register class down to the outgoing one. At each step we half
  // the LMUL:
  //   nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
  // Note that this is not guaranteed to find a subregister index, such as
  // when we are extracting from one VR type to another.
  unsigned SubRegIdx = RISCV::NoSubRegister;
  for (const unsigned RCID :
       {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
    if (VecRegClassID > RCID && SubRegClassID <= RCID) {
      VecVT = VecVT.getHalfNumVectorElementsVT();
      bool IsHi =
          InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
      SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
                                            getSubregIndexByMVT(VecVT, IsHi));
      if (IsHi)
        InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
    }
  return {SubRegIdx, InsertExtractIdx};
}
 
// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
// stores for those types.
bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
  if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
      (VT == MVT::i32 || VT == MVT::v2i16 || VT == MVT::v4i8))
    return false;
 
  return !Subtarget.useRVVForFixedLengthVectors() ||
         VT.isFixedLengthVectorOf(MVT::i1);
}
 
bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
  if (!ScalarTy.isSimple())
    return false;
  switch (ScalarTy.getSimpleVT().SimpleTy) {
  case MVT::iPTR:
    return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
  case MVT::i8:
  case MVT::i16:
  case MVT::i32:
    return Subtarget.hasVInstructions();
  case MVT::i64:
    return Subtarget.hasVInstructionsI64();
  case MVT::f16:
    return Subtarget.hasVInstructionsF16Minimal();
  case MVT::bf16:
    return Subtarget.hasVInstructionsBF16Minimal();
  case MVT::f32:
    return Subtarget.hasVInstructionsF32();
  case MVT::f64:
    return Subtarget.hasVInstructionsF64();
  default:
    return false;
  }
}
 
 
unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
  return NumRepeatedDivisors;
}
 
static SDValue getVLOperand(SDValue Op) {
  assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
          Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
         "Unexpected opcode");
  bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
  unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
  const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
      RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
  if (!II)
    return SDValue();
  return Op.getOperand(II->VLOperand + 1 + HasChain);
}
 
static bool useRVVForFixedLengthVectorVT(MVT VT,
                                         const RISCVSubtarget &Subtarget) {
  assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
  if (!Subtarget.useRVVForFixedLengthVectors())
    return false;
 
  // We only support a set of vector types with a consistent maximum fixed size
  // across all supported vector element types to avoid legalization issues.
  // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
  // fixed-length vector type we support is 1024 bytes.
  if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
    return false;
 
  unsigned MinVLen = Subtarget.getRealMinVLen();
 
  MVT EltVT = VT.getVectorElementType();
 
  // Don't use RVV for vectors we cannot scalarize if required.
  switch (EltVT.SimpleTy) {
  // i1 is supported but has different rules.
  default:
    return false;
  case MVT::i1:
    // Masks can only use a single register.
    if (VT.getVectorNumElements() > MinVLen)
      return false;
    MinVLen /= 8;
    break;
  case MVT::i8:
  case MVT::i16:
  case MVT::i32:
    break;
  case MVT::i64:
    if (!Subtarget.hasVInstructionsI64())
      return false;
    break;
  case MVT::f16:
    if (!Subtarget.hasVInstructionsF16Minimal())
      return false;
    break;
  case MVT::bf16:
    if (!Subtarget.hasVInstructionsBF16Minimal())
      return false;
    break;
  case MVT::f32:
    if (!Subtarget.hasVInstructionsF32())
      return false;
    break;
  case MVT::f64:
    if (!Subtarget.hasVInstructionsF64())
      return false;
    break;
  }
 
  // Reject elements larger than ELEN.
  if (EltVT.getSizeInBits() > Subtarget.getELen())
    return false;
 
  unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
  // Don't use RVV for types that don't fit.
  if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
    return false;
 
  // TODO: Perhaps an artificial restriction, but worth having whilst getting
  // the base fixed length RVV support in place.
  if (!VT.isPow2VectorType())
    return false;
 
  return true;
}
 
bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
  return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
}
 
// Return the largest legal scalable vector type that matches VT's element type.
static MVT getContainerForFixedLengthVector(MVT VT,
                                            const RISCVSubtarget &Subtarget) {
  // This may be called before legal types are setup.
  assert(((VT.isFixedLengthVector() &&
           Subtarget.getTargetLowering()->isTypeLegal(VT)) ||
          useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
         "Expected legal fixed length vector!");
 
  unsigned MinVLen = Subtarget.getRealMinVLen();
  unsigned MaxELen = Subtarget.getELen();
 
  MVT EltVT = VT.getVectorElementType();
  switch (EltVT.SimpleTy) {
  default:
    llvm_unreachable("unexpected element type for RVV container");
  case MVT::i1:
  case MVT::i8:
  case MVT::i16:
  case MVT::i32:
  case MVT::i64:
  case MVT::bf16:
  case MVT::f16:
  case MVT::f32:
  case MVT::f64: {
    // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
    // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
    // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
    unsigned NumElts =
        (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
    NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
    assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
    return MVT::getScalableVectorVT(EltVT, NumElts);
  }
  }
}
 
MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
  return ::getContainerForFixedLengthVector(VT, getSubtarget());
}
 
// Grow V to consume an entire RVV register.
static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
                                       const RISCVSubtarget &Subtarget) {
  assert(VT.isScalableVector() &&
         "Expected to convert into a scalable vector!");
  assert(V.getValueType().isFixedLengthVector() &&
         "Expected a fixed length vector operand!");
  SDLoc DL(V);
  return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), V, 0);
}
 
// Shrink V so it's just big enough to maintain a VT's worth of data.
static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
                                         const RISCVSubtarget &Subtarget) {
  assert(VT.isFixedLengthVector() &&
         "Expected to convert into a fixed length vector!");
  assert(V.getValueType().isScalableVector() &&
         "Expected a scalable vector operand!");
  SDLoc DL(V);
  return DAG.getExtractSubvector(DL, VT, V, 0);
}
 
/// Return the type of the mask type suitable for masking the provided
/// vector type.  This is simply an i1 element type vector of the same
/// (possibly scalable) length.
static MVT getMaskTypeFor(MVT VecVT) {
  assert(VecVT.isVector());
  ElementCount EC = VecVT.getVectorElementCount();
  return MVT::getVectorVT(MVT::i1, EC);
}
 
/// Creates an all ones mask suitable for masking a vector of type VecTy with
/// vector length VL.  .
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
                              SelectionDAG &DAG) {
  MVT MaskVT = getMaskTypeFor(VecVT);
  return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
}
 
static std::pair<SDValue, SDValue>
getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
                        const RISCVSubtarget &Subtarget) {
  assert(VecVT.isScalableVector() && "Expecting a scalable vector");
  SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
  SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
  return {Mask, VL};
}
 
static std::pair<SDValue, SDValue>
getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
                SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
  assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
  SDValue VL = DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
  SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
  return {Mask, VL};
}
 
// Gets the two common "VL" operands: an all-ones mask and the vector length.
// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
// the vector type that the fixed-length vector is contained in. Otherwise if
// VecVT is scalable, then ContainerVT should be the same as VecVT.
static std::pair<SDValue, SDValue>
getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
                const RISCVSubtarget &Subtarget) {
  if (VecVT.isFixedLengthVector())
    return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
                           Subtarget);
  assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
  return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
}
 
SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
                                          SelectionDAG &DAG) const {
  assert(VecVT.isScalableVector() && "Expected scalable vector");
  return DAG.getElementCount(DL, Subtarget.getXLenVT(),
                             VecVT.getVectorElementCount());
}
 
std::pair<unsigned, unsigned>
RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
                                        const RISCVSubtarget &Subtarget) {
  assert(VecVT.isScalableVector() && "Expected scalable vector");
 
  unsigned EltSize = VecVT.getScalarSizeInBits();
  unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
 
  unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
  unsigned MaxVLMAX =
      RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
 
  unsigned VectorBitsMin = Subtarget.getRealMinVLen();
  unsigned MinVLMAX =
      RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
 
  return std::make_pair(MinVLMAX, MaxVLMAX);
}
 
// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
// of either is (currently) supported. This can get us into an infinite loop
// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
// as a ..., etc.
// Until either (or both) of these can reliably lower any node, reporting that
// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
// which is not desirable.
bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
    EVT VT, unsigned DefinedValues) const {
  return false;
}
 
InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
  // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
  // implementation-defined.
  if (!VT.isVector())
    return InstructionCost::getInvalid();
  unsigned DLenFactor = Subtarget.getDLenFactor();
  unsigned Cost;
  if (VT.isScalableVector()) {
    unsigned LMul;
    bool Fractional;
    std::tie(LMul, Fractional) =
        RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(VT));
    if (Fractional)
      Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
    else
      Cost = (LMul * DLenFactor);
  } else {
    Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor);
  }
  return Cost;
}
 
 
/// Return the cost of a vrgather.vv instruction for the type VT.  vrgather.vv
/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
/// be by default.  VRGatherCostModel reflects available options.  Note that
/// operand (index and possibly mask) are handled separately.
InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
  auto LMULCost = getLMULCost(VT);
  bool Log2CostModel =
      Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
  if (Log2CostModel && LMULCost.isValid()) {
    unsigned Log = Log2_64(LMULCost.getValue());
    if (Log > 0)
      return LMULCost * Log;
  }
  return LMULCost * LMULCost;
}
 
/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
/// or may track the vrgather.vv cost. It is implementation-dependent.
InstructionCost RISCVTargetLowering::getVRGatherVICost(MVT VT) const {
  return getLMULCost(VT);
}
 
/// Return the cost of a vslidedown.vx or vslideup.vx instruction
/// for the type VT.  (This does not cover the vslide1up or vslide1down
/// variants.)  Slides may be linear in the number of vregs implied by LMUL,
/// or may track the vrgather.vv cost. It is implementation-dependent.
InstructionCost RISCVTargetLowering::getVSlideVXCost(MVT VT) const {
  return getLMULCost(VT);
}
 
/// Return the cost of a vslidedown.vi or vslideup.vi instruction
/// for the type VT.  (This does not cover the vslide1up or vslide1down
/// variants.)  Slides may be linear in the number of vregs implied by LMUL,
/// or may track the vrgather.vv cost. It is implementation-dependent.
InstructionCost RISCVTargetLowering::getVSlideVICost(MVT VT) const {
  return getLMULCost(VT);
}
 
static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
  // bf16 conversions are always promoted to f32.
  if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
      Op.getValueType() == MVT::bf16) {
    bool IsStrict = Op->isStrictFPOpcode();
 
    SDLoc DL(Op);
    if (IsStrict) {
      SDValue Val = DAG.getNode(Op.getOpcode(), DL, {MVT::f32, MVT::Other},
                                {Op.getOperand(0), Op.getOperand(1)});
      return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
                         {Op.getValueType(), MVT::Other},
                         {Val.getValue(1), Val.getValue(0),
                          DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
    }
    return DAG.getNode(
        ISD::FP_ROUND, DL, Op.getValueType(),
        DAG.getNode(Op.getOpcode(), DL, MVT::f32, Op.getOperand(0)),
        DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
  }
 
  // Other operations are legal.
  return Op;
}
 
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
  // RISC-V FP-to-int conversions saturate to the destination register size, but
  // don't produce 0 for nan. We can use a conversion instruction and fix the
  // nan case with a compare and a select.
  SDValue Src = Op.getOperand(0);
 
  MVT DstVT = Op.getSimpleValueType();
  EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
 
  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
 
  if (!DstVT.isVector()) {
    // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
    // the result.
    if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
        Src.getValueType() == MVT::bf16) {
      Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
    }
 
    unsigned Opc;
    if (SatVT == DstVT)
      Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
    else if (DstVT == MVT::i64 && SatVT == MVT::i32)
      Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
    else
      return SDValue();
    // FIXME: Support other SatVTs by clamping before or after the conversion.
 
    SDLoc DL(Op);
    SDValue FpToInt = DAG.getNode(
        Opc, DL, DstVT, Src,
        DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
 
    if (Opc == RISCVISD::FCVT_WU_RV64)
      FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
 
    SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
    return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
                           ISD::CondCode::SETUO);
  }
 
  // Vectors.
 
  MVT DstEltVT = DstVT.getVectorElementType();
  MVT SrcVT = Src.getSimpleValueType();
  MVT SrcEltVT = SrcVT.getVectorElementType();
  unsigned SrcEltSize = SrcEltVT.getSizeInBits();
  unsigned DstEltSize = DstEltVT.getSizeInBits();
 
  // Only handle saturating to the destination type.
  if (SatVT != DstEltVT)
    return SDValue();
 
  MVT DstContainerVT = DstVT;
  MVT SrcContainerVT = SrcVT;
  if (DstVT.isFixedLengthVector()) {
    DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
    assert(DstContainerVT.getVectorElementCount() ==
               SrcContainerVT.getVectorElementCount() &&
           "Expected same element count");
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  SDLoc DL(Op);
 
  auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
 
  SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
                              {Src, Src, DAG.getCondCode(ISD::SETNE),
                               DAG.getUNDEF(Mask.getValueType()), Mask, VL});
 
  // Need to widen by more than 1 step, promote the FP type, then do a widening
  // convert.
  if (DstEltSize > (2 * SrcEltSize)) {
    assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
    MVT InterVT = SrcContainerVT.changeVectorElementType(MVT::f32);
    Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
  }
 
  MVT CvtContainerVT = DstContainerVT;
  MVT CvtEltVT = DstEltVT;
  if (SrcEltSize > (2 * DstEltSize)) {
    CvtEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
    CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
  }
 
  unsigned RVVOpc =
      IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
  SDValue Res = DAG.getNode(RVVOpc, DL, CvtContainerVT, Src, Mask, VL);
 
  while (CvtContainerVT != DstContainerVT) {
    CvtEltVT = MVT::getIntegerVT(CvtEltVT.getSizeInBits() / 2);
    CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
    // Rounding mode here is arbitrary since we aren't shifting out any bits.
    unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
                                : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
    Res = DAG.getNode(ClipOpc, DL, CvtContainerVT, Res, Mask, VL);
  }
 
  SDValue SplatZero = DAG.getNode(
      RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
      DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
  Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
                    Res, DAG.getUNDEF(DstContainerVT), VL);
 
  if (DstVT.isFixedLengthVector())
    Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
 
  return Res;
}
 
static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  bool IsStrict = Op->isStrictFPOpcode();
  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
 
  // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
  // bf16 conversions are always promoted to f32.
  if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
      SrcVal.getValueType() == MVT::bf16) {
    SDLoc DL(Op);
    if (IsStrict) {
      SDValue Ext =
          DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
                      {Op.getOperand(0), SrcVal});
      return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
                         {Ext.getValue(1), Ext.getValue(0)});
    }
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                       DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, SrcVal));
  }
 
  // Other operations are legal.
  return Op;
}
 
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
  switch (Opc) {
  case ISD::FROUNDEVEN:
  case ISD::STRICT_FROUNDEVEN:
    return RISCVFPRndMode::RNE;
  case ISD::FTRUNC:
  case ISD::STRICT_FTRUNC:
    return RISCVFPRndMode::RTZ;
  case ISD::FFLOOR:
  case ISD::STRICT_FFLOOR:
    return RISCVFPRndMode::RDN;
  case ISD::FCEIL:
  case ISD::STRICT_FCEIL:
    return RISCVFPRndMode::RUP;
  case ISD::FROUND:
  case ISD::LROUND:
  case ISD::LLROUND:
  case ISD::STRICT_FROUND:
  case ISD::STRICT_LROUND:
  case ISD::STRICT_LLROUND:
    return RISCVFPRndMode::RMM;
  case ISD::FRINT:
  case ISD::LRINT:
  case ISD::LLRINT:
  case ISD::STRICT_FRINT:
  case ISD::STRICT_LRINT:
  case ISD::STRICT_LLRINT:
    return RISCVFPRndMode::DYN;
  }
 
  return RISCVFPRndMode::Invalid;
}
 
// Expand vector FTRUNC, FCEIL, FFLOOR and FROUND by converting to
// the integer domain and back. Taking care to avoid converting values that are
// nan or already correct.
static SDValue
lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isVector() && "Unexpected type");
 
  SDLoc DL(Op);
 
  SDValue Src = Op.getOperand(0);
 
  // Freeze the source since we are increasing the number of uses.
  Src = DAG.getFreeze(Src);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  // We do the conversion on the absolute value and fix the sign at the end.
  SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
 
  // Determine the largest integer that can be represented exactly. This and
  // values larger than it don't have any fractional bits so don't need to
  // be converted.
  const fltSemantics &FltSem = ContainerVT.getFltSemantics();
  unsigned Precision = APFloat::semanticsPrecision(FltSem);
  APFloat MaxVal = APFloat(FltSem);
  MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
                          /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
  SDValue MaxValNode =
      DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
  SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
                                    DAG.getUNDEF(ContainerVT), MaxValNode, VL);
 
  // If abs(Src) was larger than MaxVal or nan, keep it.
  MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
  Mask =
      DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
                  {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
                   Mask, Mask, VL});
 
  // Truncate to integer and convert back to FP.
  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Truncated;
 
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode");
  case ISD::FRINT:
  case ISD::FCEIL:
  case ISD::FFLOOR:
  case ISD::FROUND:
  case ISD::FROUNDEVEN: {
    RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
    assert(FRM != RISCVFPRndMode::Invalid);
    Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
                            DAG.getTargetConstant(FRM, DL, XLenVT), VL);
    break;
  }
  case ISD::FTRUNC:
    Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
                            Mask, VL);
    break;
  case ISD::FNEARBYINT:
    Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
                            Mask, VL);
    break;
  }
 
  // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
  if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
    Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
                            Mask, VL);
 
  // Restore the original sign so that -0.0 is preserved.
  Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
                          Src, Src, Mask, VL);
 
  if (!VT.isFixedLengthVector())
    return Truncated;
 
  return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
}
 
// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
// qNan and converting the new source to integer and back to FP.
static SDValue
lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                            const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  SDValue Chain = Op.getOperand(0);
  SDValue Src = Op.getOperand(1);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  // Freeze the source since we are increasing the number of uses.
  Src = DAG.getFreeze(Src);
 
  // Convert sNan to qNan by executing x + x for all unordered element x in Src.
  MVT MaskVT = Mask.getSimpleValueType();
  SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
                                DAG.getVTList(MaskVT, MVT::Other),
                                {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
                                 DAG.getUNDEF(MaskVT), Mask, VL});
  Chain = Unorder.getValue(1);
  Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
                    DAG.getVTList(ContainerVT, MVT::Other),
                    {Chain, Src, Src, Src, Unorder, VL});
  Chain = Src.getValue(1);
 
  // We do the conversion on the absolute value and fix the sign at the end.
  SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
 
  // Determine the largest integer that can be represented exactly. This and
  // values larger than it don't have any fractional bits so don't need to
  // be converted.
  const fltSemantics &FltSem = ContainerVT.getFltSemantics();
  unsigned Precision = APFloat::semanticsPrecision(FltSem);
  APFloat MaxVal = APFloat(FltSem);
  MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
                          /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
  SDValue MaxValNode =
      DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
  SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
                                    DAG.getUNDEF(ContainerVT), MaxValNode, VL);
 
  // If abs(Src) was larger than MaxVal or nan, keep it.
  Mask = DAG.getNode(
      RISCVISD::SETCC_VL, DL, MaskVT,
      {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
 
  // Truncate to integer and convert back to FP.
  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Truncated;
 
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode");
  case ISD::STRICT_FCEIL:
  case ISD::STRICT_FFLOOR:
  case ISD::STRICT_FROUND:
  case ISD::STRICT_FROUNDEVEN: {
    RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
    assert(FRM != RISCVFPRndMode::Invalid);
    Truncated = DAG.getNode(
        RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
        {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
    break;
  }
  case ISD::STRICT_FTRUNC:
    Truncated =
        DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
                    DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
    break;
  case ISD::STRICT_FNEARBYINT:
    Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
                            DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
                            Mask, VL);
    break;
  }
  Chain = Truncated.getValue(1);
 
  // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
  if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
    Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
                            DAG.getVTList(ContainerVT, MVT::Other), Chain,
                            Truncated, Mask, VL);
    Chain = Truncated.getValue(1);
  }
 
  // Restore the original sign so that -0.0 is preserved.
  Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
                          Src, Src, Mask, VL);
 
  if (VT.isFixedLengthVector())
    Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
  return DAG.getMergeValues({Truncated, Chain}, DL);
}
 
static SDValue
lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  if (VT.isVector())
    return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
 
  if (DAG.shouldOptForSize())
    return SDValue();
 
  SDLoc DL(Op);
  SDValue Src = Op.getOperand(0);
 
  // Create an integer the size of the mantissa with the MSB set. This and all
  // values larger than it don't have any fractional bits so don't need to be
  // converted.
  const fltSemantics &FltSem = VT.getFltSemantics();
  unsigned Precision = APFloat::semanticsPrecision(FltSem);
  APFloat MaxVal = APFloat(FltSem);
  MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
                          /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
  SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
 
  RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
  return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
                     DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
}
 
// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
                                       const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT DstVT = Op.getSimpleValueType();
  SDValue Src = Op.getOperand(0);
  MVT SrcVT = Src.getSimpleValueType();
  assert(SrcVT.isVector() && DstVT.isVector() &&
         !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
         "Unexpected type");
 
  MVT DstContainerVT = DstVT;
  MVT SrcContainerVT = SrcVT;
 
  if (DstVT.isFixedLengthVector()) {
    DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
 
  // [b]f16 -> f32
  MVT SrcElemType = SrcVT.getVectorElementType();
  if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
    MVT F32VT = SrcContainerVT.changeVectorElementType(MVT::f32);
    Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, F32VT, Src, Mask, VL);
  }
 
  SDValue Res =
      DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, DstContainerVT, Src, Mask,
                  DAG.getTargetConstant(matchRoundingOp(Op.getOpcode()), DL,
                                        Subtarget.getXLenVT()),
                  VL);
 
  if (!DstVT.isFixedLengthVector())
    return Res;
 
  return convertFromScalableVector(DstVT, Res, DAG, Subtarget);
}
 
static SDValue
getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
              const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
              SDValue Offset, SDValue Mask, SDValue VL,
              unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
  if (Passthru.isUndef())
    Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
  SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
  return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
}
 
static SDValue
getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
            EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
            SDValue VL,
            unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
  if (Passthru.isUndef())
    Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
  SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
  SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
  return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
}
 
struct VIDSequence {
  int64_t StepNumerator;
  unsigned StepDenominator;
  int64_t Addend;
};
 
static std::optional<APInt> getExactInteger(const APFloat &APF,
                                            uint32_t BitWidth) {
  // We will use a SINT_TO_FP to materialize this constant so we should use a
  // signed APSInt here.
  APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
  // We use an arbitrary rounding mode here. If a floating-point is an exact
  // integer (e.g., 1.0), the rounding mode does not affect the output value. If
  // the rounding mode changes the output value, then it is not an exact
  // integer.
  RoundingMode ArbitraryRM = RoundingMode::TowardZero;
  bool IsExact;
  // If it is out of signed integer range, it will return an invalid operation.
  // If it is not an exact integer, IsExact is false.
  if ((APF.convertToInteger(ValInt, ArbitraryRM, &IsExact) ==
       APFloatBase::opInvalidOp) ||
      !IsExact)
    return std::nullopt;
  return ValInt.extractBits(BitWidth, 0);
}
 
// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
// to the (non-zero) step S and start value X. This can be then lowered as the
// RVV sequence (VID * S) + X, for example.
// The step S is represented as an integer numerator divided by a positive
// denominator. Note that the implementation currently only identifies
// sequences in which either the numerator is +/- 1 or the denominator is 1. It
// cannot detect 2/3, for example.
// Note that this method will also match potentially unappealing index
// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
// determine whether this is worth generating code for.
//
// EltSizeInBits is the size of the type that the sequence will be calculated
// in, i.e. SEW for build_vectors or XLEN for address calculations.
static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
                                                      unsigned EltSizeInBits) {
  assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
  if (!cast<BuildVectorSDNode>(Op)->isConstant())
    return std::nullopt;
  bool IsInteger = Op.getValueType().isInteger();
 
  std::optional<unsigned> SeqStepDenom;
  std::optional<APInt> SeqStepNum;
  std::optional<APInt> SeqAddend;
  std::optional<std::pair<APInt, unsigned>> PrevElt;
  assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
 
  // First extract the ops into a list of constant integer values. This may not
  // be possible for floats if they're not all representable as integers.
  SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
  const unsigned OpSize = Op.getScalarValueSizeInBits();
  for (auto [Idx, Elt] : enumerate(Op->op_values())) {
    if (Elt.isUndef()) {
      Elts[Idx] = std::nullopt;
      continue;
    }
    if (IsInteger) {
      Elts[Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
    } else {
      auto ExactInteger =
          getExactInteger(cast<ConstantFPSDNode>(Elt)->getValueAPF(), OpSize);
      if (!ExactInteger)
        return std::nullopt;
      Elts[Idx] = *ExactInteger;
    }
  }
 
  for (auto [Idx, Elt] : enumerate(Elts)) {
    // Assume undef elements match the sequence; we just have to be careful
    // when interpolating across them.
    if (!Elt)
      continue;
 
    if (PrevElt) {
      // Calculate the step since the last non-undef element, and ensure
      // it's consistent across the entire sequence.
      unsigned IdxDiff = Idx - PrevElt->second;
      APInt ValDiff = *Elt - PrevElt->first;
 
      // A zero-value value difference means that we're somewhere in the middle
      // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
      // step change before evaluating the sequence.
      if (ValDiff == 0)
        continue;
 
      int64_t Remainder = ValDiff.srem(IdxDiff);
      // Normalize the step if it's greater than 1.
      if (Remainder != ValDiff.getSExtValue()) {
        // The difference must cleanly divide the element span.
        if (Remainder != 0)
          return std::nullopt;
        ValDiff = ValDiff.sdiv(IdxDiff);
        IdxDiff = 1;
      }
 
      if (!SeqStepNum)
        SeqStepNum = ValDiff;
      else if (ValDiff != SeqStepNum)
        return std::nullopt;
 
      if (!SeqStepDenom)
        SeqStepDenom = IdxDiff;
      else if (IdxDiff != *SeqStepDenom)
        return std::nullopt;
    }
 
    // Record this non-undef element for later.
    if (!PrevElt || PrevElt->first != *Elt)
      PrevElt = std::make_pair(*Elt, Idx);
  }
 
  // We need to have logged a step for this to count as a legal index sequence.
  if (!SeqStepNum || !SeqStepDenom)
    return std::nullopt;
 
  // Loop back through the sequence and validate elements we might have skipped
  // while waiting for a valid step. While doing this, log any sequence addend.
  for (auto [Idx, Elt] : enumerate(Elts)) {
    if (!Elt)
      continue;
    APInt ExpectedVal =
        (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
         *SeqStepNum)
            .sdiv(*SeqStepDenom);
 
    APInt Addend = *Elt - ExpectedVal;
    if (!SeqAddend)
      SeqAddend = Addend;
    else if (Addend != SeqAddend)
      return std::nullopt;
  }
 
  assert(SeqAddend && "Must have an addend if we have a step");
 
  return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
                     SeqAddend->getSExtValue()};
}
 
// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
// and lower it as a VRGATHER_VX_VL from the source vector.
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
                                  SelectionDAG &DAG,
                                  const RISCVSubtarget &Subtarget) {
  if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
    return SDValue();
  SDValue Src = SplatVal.getOperand(0);
  // Don't perform this optimization for i1 vectors, or if the element types are
  // different
  // FIXME: Support i1 vectors, maybe by promoting to i8?
  MVT EltTy = VT.getVectorElementType();
  if (EltTy == MVT::i1 ||
      !DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
    return SDValue();
  MVT SrcVT = Src.getSimpleValueType();
  if (EltTy != SrcVT.getVectorElementType())
    return SDValue();
  SDValue Idx = SplatVal.getOperand(1);
  // The index must be a legal type.
  if (Idx.getValueType() != Subtarget.getXLenVT())
    return SDValue();
 
  // Check that we know Idx lies within VT
  if (!TypeSize::isKnownLE(SrcVT.getSizeInBits(), VT.getSizeInBits())) {
    auto *CIdx = dyn_cast<ConstantSDNode>(Idx);
    if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
      return SDValue();
  }
 
  // Convert fixed length vectors to scalable
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector())
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  MVT SrcContainerVT = SrcVT;
  if (SrcVT.isFixedLengthVector()) {
    SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
    Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
  }
 
  // Put Vec in a VT sized vector
  if (SrcContainerVT.getVectorMinNumElements() <
      ContainerVT.getVectorMinNumElements())
    Src = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), Src, 0);
  else
    Src = DAG.getExtractSubvector(DL, ContainerVT, Src, 0);
 
  // We checked that Idx fits inside VT earlier
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
  SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Src,
                               Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
  if (VT.isFixedLengthVector())
    Gather = convertFromScalableVector(VT, Gather, DAG, Subtarget);
  return Gather;
}
 
static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
    int64_t StepNumerator = SimpleVID->StepNumerator;
    unsigned StepDenominator = SimpleVID->StepDenominator;
    int64_t Addend = SimpleVID->Addend;
 
    assert(StepNumerator != 0 && "Invalid step");
    bool Negate = false;
    int64_t SplatStepVal = StepNumerator;
    unsigned StepOpcode = ISD::MUL;
    // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
    // anyway as the shift of 63 won't fit in uimm5.
    if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
        isPowerOf2_64(std::abs(StepNumerator))) {
      Negate = StepNumerator < 0;
      StepOpcode = ISD::SHL;
      SplatStepVal = Log2_64(std::abs(StepNumerator));
    }
 
    // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
    // since it's the immediate value many RVV instructions accept. There is
    // no vmul.vi instruction so ensure multiply constant can fit in a
    // single addi instruction.  For the addend, we allow up to 32 bits..
    if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
         (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
        isPowerOf2_32(StepDenominator) &&
        (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(Addend)) {
      MVT VIDVT =
          VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
      MVT VIDContainerVT = getContainerForFixedLengthVector(VIDVT, Subtarget);
      SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
      // Convert right out of the scalable type so we can use standard ISD
      // nodes for the rest of the computation. If we used scalable types with
      // these, we'd lose the fixed-length vector info and generate worse
      // vsetvli code.
      VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
      if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
          (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
        SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
        VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
      }
      if (StepDenominator != 1) {
        SDValue SplatStep =
            DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
        VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
      }
      if (Addend != 0 || Negate) {
        SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
        VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
                          VID);
      }
      if (VT.isFloatingPoint()) {
        // TODO: Use vfwcvt to reduce register pressure.
        VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
      }
      return VID;
    }
  }
 
  return SDValue();
}
 
/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
/// which constitute a large proportion of the elements. In such cases we can
/// splat a vector with the dominant element and make up the shortfall with
/// INSERT_VECTOR_ELTs.  Returns SDValue if not profitable.
/// Note that this includes vectors of 2 elements by association. The
/// upper-most element is the "dominant" one, allowing us to use a splat to
/// "insert" the upper element, and an insert of the lower element at position
/// 0, which improves codegen.
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
                                                 const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned NumElts = Op.getNumOperands();
 
  SDValue DominantValue;
  unsigned MostCommonCount = 0;
  DenseMap<SDValue, unsigned> ValueCounts;
  unsigned NumUndefElts =
      count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
 
  // Track the number of scalar loads we know we'd be inserting, estimated as
  // any non-zero floating-point constant. Other kinds of element are either
  // already in registers or are materialized on demand. The threshold at which
  // a vector load is more desirable than several scalar materializion and
  // vector-insertion instructions is not known.
  unsigned NumScalarLoads = 0;
 
  for (SDValue V : Op->op_values()) {
    if (V.isUndef())
      continue;
 
    unsigned &Count = ValueCounts[V];
    if (0 == Count)
      if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
        NumScalarLoads += !CFP->isPosZero();
 
    // Is this value dominant? In case of a tie, prefer the highest element as
    // it's cheaper to insert near the beginning of a vector than it is at the
    // end.
    if (++Count >= MostCommonCount) {
      DominantValue = V;
      MostCommonCount = Count;
    }
  }
 
  assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
  unsigned NumDefElts = NumElts - NumUndefElts;
  unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
 
  // Don't perform this optimization when optimizing for size, since
  // materializing elements and inserting them tends to cause code bloat.
  if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
      (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
      ((MostCommonCount > DominantValueCountThreshold) ||
       (ValueCounts.size() <= Log2_32(NumDefElts)))) {
    // Start by splatting the most common element.
    SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
 
    DenseSet<SDValue> Processed{DominantValue};
 
    // We can handle an insert into the last element (of a splat) via
    // v(f)slide1down.  This is slightly better than the vslideup insert
    // lowering as it avoids the need for a vector group temporary.  It
    // is also better than using vmerge.vx as it avoids the need to
    // materialize the mask in a vector register.
    if (SDValue LastOp = Op->getOperand(Op->getNumOperands() - 1);
        !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
        LastOp != DominantValue) {
      Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
      auto OpCode =
        VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
      if (!VT.isFloatingPoint())
        LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
      Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
                        LastOp, Mask, VL);
      Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
      Processed.insert(LastOp);
    }
 
    MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
    for (const auto &OpIdx : enumerate(Op->ops())) {
      const SDValue &V = OpIdx.value();
      if (V.isUndef() || !Processed.insert(V).second)
        continue;
      if (ValueCounts[V] == 1) {
        Vec = DAG.getInsertVectorElt(DL, Vec, V, OpIdx.index());
      } else {
        // Blend in all instances of this value using a VSELECT, using a
        // mask where each bit signals whether that element is the one
        // we're after.
        SmallVector<SDValue> Ops;
        transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
          return DAG.getConstant(V == V1, DL, XLenVT);
        });
        Vec = DAG.getNode(ISD::VSELECT, DL, VT,
                          DAG.getBuildVector(SelMaskTy, DL, Ops),
                          DAG.getSplatBuildVector(VT, DL, V), Vec);
      }
    }
 
    return Vec;
  }
 
  return SDValue();
}
 
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
                                           const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  SDLoc DL(Op);
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned NumElts = Op.getNumOperands();
 
  if (VT.getVectorElementType() == MVT::i1) {
    if (ISD::isBuildVectorAllZeros(Op.getNode())) {
      SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
      return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
    }
 
    if (ISD::isBuildVectorAllOnes(Op.getNode())) {
      SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
      return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
    }
 
    // Lower constant mask BUILD_VECTORs via an integer vector type, in
    // scalar integer chunks whose bit-width depends on the number of mask
    // bits and XLEN.
    // First, determine the most appropriate scalar integer type to use. This
    // is at most XLenVT, but may be shrunk to a smaller vector element type
    // according to the size of the final vector - use i8 chunks rather than
    // XLenVT if we're producing a v8i1. This results in more consistent
    // codegen across RV32 and RV64.
    unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
    NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen());
    // If we have to use more than one INSERT_VECTOR_ELT then this
    // optimization is likely to increase code size; avoid performing it in
    // such a case. We can use a load from a constant pool in this case.
    if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
      return SDValue();
    // Now we can create our integer vector type. Note that it may be larger
    // than the resulting mask type: v4i1 would use v1i8 as its integer type.
    unsigned IntegerViaVecElts = divideCeil(NumElts, NumViaIntegerBits);
    MVT IntegerViaVecVT =
      MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
                       IntegerViaVecElts);
 
    uint64_t Bits = 0;
    unsigned BitPos = 0, IntegerEltIdx = 0;
    SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
 
    for (unsigned I = 0; I < NumElts;) {
      SDValue V = Op.getOperand(I);
      bool BitValue = !V.isUndef() && V->getAsZExtVal();
      Bits |= ((uint64_t)BitValue << BitPos);
      ++BitPos;
      ++I;
 
      // Once we accumulate enough bits to fill our scalar type or process the
      // last element, insert into our vector and clear our accumulated data.
      if (I % NumViaIntegerBits == 0 || I == NumElts) {
        if (NumViaIntegerBits <= 32)
          Bits = SignExtend64<32>(Bits);
        SDValue Elt = DAG.getSignedConstant(Bits, DL, XLenVT);
        Elts[IntegerEltIdx] = Elt;
        Bits = 0;
        BitPos = 0;
        IntegerEltIdx++;
      }
    }
 
    SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
 
    if (NumElts < NumViaIntegerBits) {
      // If we're producing a smaller vector than our minimum legal integer
      // type, bitcast to the equivalent (known-legal) mask type, and extract
      // our final mask.
      assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
      Vec = DAG.getBitcast(MVT::v8i1, Vec);
      Vec = DAG.getExtractSubvector(DL, VT, Vec, 0);
    } else {
      // Else we must have produced an integer type with the same size as the
      // mask type; bitcast for the final result.
      assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
      Vec = DAG.getBitcast(VT, Vec);
    }
 
    return Vec;
  }
 
  if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
    unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
                                        : RISCVISD::VMV_V_X_VL;
    if (!VT.isFloatingPoint())
      Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
    Splat =
        DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
    return convertFromScalableVector(VT, Splat, DAG, Subtarget);
  }
 
  // Try and match index sequences, which we can lower to the vid instruction
  // with optional modifications. An all-undef vector is matched by
  // getSplatValue, above.
  if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
    return Res;
 
  // For very small build_vectors, use a single scalar insert of a constant.
  // TODO: Base this on constant rematerialization cost, not size.
  const unsigned EltBitSize = VT.getScalarSizeInBits();
  if (VT.getSizeInBits() <= 32 &&
      ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
    MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits());
    assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
           "Unexpected sequence type");
    // If we can use the original VL with the modified element type, this
    // means we only have a VTYPE toggle, not a VL toggle.  TODO: Should this
    // be moved into InsertVSETVLI?
    unsigned ViaVecLen =
      (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
    MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
 
    uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
    uint64_t SplatValue = 0;
    // Construct the amalgamated value at this larger vector type.
    for (const auto &OpIdx : enumerate(Op->op_values())) {
      const auto &SeqV = OpIdx.value();
      if (!SeqV.isUndef())
        SplatValue |=
            ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
    }
 
    // On RV64, sign-extend from 32 to 64 bits where possible in order to
    // achieve better constant materializion.
    // On RV32, we need to sign-extend to use getSignedConstant.
    if (ViaIntVT == MVT::i32)
      SplatValue = SignExtend64<32>(SplatValue);
 
    SDValue Vec = DAG.getInsertVectorElt(
        DL, DAG.getUNDEF(ViaVecVT),
        DAG.getSignedConstant(SplatValue, DL, XLenVT), 0);
    if (ViaVecLen != 1)
      Vec = DAG.getExtractSubvector(DL, MVT::getVectorVT(ViaIntVT, 1), Vec, 0);
    return DAG.getBitcast(VT, Vec);
  }
 
 
  // Attempt to detect "hidden" splats, which only reveal themselves as splats
  // when re-interpreted as a vector with a larger element type. For example,
  //   v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
  // could be instead splat as
  //   v2i32 = build_vector i32 0x00010000, i32 0x00010000
  // TODO: This optimization could also work on non-constant splats, but it
  // would require bit-manipulation instructions to construct the splat value.
  SmallVector<SDValue> Sequence;
  const auto *BV = cast<BuildVectorSDNode>(Op);
  if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
      ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
      BV->getRepeatedSequence(Sequence) &&
      (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
    unsigned SeqLen = Sequence.size();
    MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
    assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
            ViaIntVT == MVT::i64) &&
           "Unexpected sequence type");
 
    // If we can use the original VL with the modified element type, this
    // means we only have a VTYPE toggle, not a VL toggle.  TODO: Should this
    // be moved into InsertVSETVLI?
    const unsigned RequiredVL = NumElts / SeqLen;
    const unsigned ViaVecLen =
      (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
      NumElts : RequiredVL;
    MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
 
    unsigned EltIdx = 0;
    uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
    uint64_t SplatValue = 0;
    // Construct the amalgamated value which can be splatted as this larger
    // vector type.
    for (const auto &SeqV : Sequence) {
      if (!SeqV.isUndef())
        SplatValue |=
            ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
      EltIdx++;
    }
 
    // On RV64, sign-extend from 32 to 64 bits where possible in order to
    // achieve better constant materializion.
    // On RV32, we need to sign-extend to use getSignedConstant.
    if (ViaIntVT == MVT::i32)
      SplatValue = SignExtend64<32>(SplatValue);
 
    // Since we can't introduce illegal i64 types at this stage, we can only
    // perform an i64 splat on RV32 if it is its own sign-extended value. That
    // way we can use RVV instructions to splat.
    assert((ViaIntVT.bitsLE(XLenVT) ||
            (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
           "Unexpected bitcast sequence");
    if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
      SDValue ViaVL =
          DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
      MVT ViaContainerVT =
          getContainerForFixedLengthVector(ViaVecVT, Subtarget);
      SDValue Splat =
          DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
                      DAG.getUNDEF(ViaContainerVT),
                      DAG.getSignedConstant(SplatValue, DL, XLenVT), ViaVL);
      Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
      if (ViaVecLen != RequiredVL)
        Splat = DAG.getExtractSubvector(
            DL, MVT::getVectorVT(ViaIntVT, RequiredVL), Splat, 0);
      return DAG.getBitcast(VT, Splat);
    }
  }
 
  // If the number of signbits allows, see if we can lower as a <N x i8>.
  // Our main goal here is to reduce LMUL (and thus work) required to
  // build the constant, but we will also narrow if the resulting
  // narrow vector is known to materialize cheaply.
  // TODO: We really should be costing the smaller vector.  There are
  // profitable cases this misses.
  if (EltBitSize > 8 && VT.isInteger() &&
      (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
      DAG.ComputeMaxSignificantBits(Op) <= 8) {
    SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
                                        DL, Op->ops());
    Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
                                     Source, DAG, Subtarget);
    SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
    return convertFromScalableVector(VT, Res, DAG, Subtarget);
  }
 
  if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
    return Res;
 
  // For constant vectors, use generic constant pool lowering.  Otherwise,
  // we'd have to materialize constants in GPRs just to move them into the
  // vector.
  return SDValue();
}
 
static unsigned getPACKOpcode(unsigned DestBW,
                              const RISCVSubtarget &Subtarget) {
  switch (DestBW) {
  default:
    llvm_unreachable("Unsupported pack size");
  case 16:
    return RISCV::PACKH;
  case 32:
    return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
  case 64:
    assert(Subtarget.is64Bit());
    return RISCV::PACK;
  }
}
 
/// Double the element size of the build vector to reduce the number
/// of vslide1down in the build vector chain.  In the worst case, this
/// trades three scalar operations for 1 vector operation.  Scalar
/// operations are generally lower latency, and for out-of-order cores
/// we also benefit from additional parallelism.
static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
                                          const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
  MVT ElemVT = VT.getVectorElementType();
  if (!ElemVT.isInteger())
    return SDValue();
 
  // TODO: Relax these architectural restrictions, possibly with costing
  // of the actual instructions required.
  if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
    return SDValue();
 
  unsigned NumElts = VT.getVectorNumElements();
  unsigned ElemSizeInBits = ElemVT.getSizeInBits();
  if (ElemSizeInBits >= std::min(Subtarget.getELen(), Subtarget.getXLen()) ||
      NumElts % 2 != 0)
    return SDValue();
 
  // Produce [B,A] packed into a type twice as wide.  Note that all
  // scalars are XLenVT, possibly masked (see below).
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Mask = DAG.getConstant(
      APInt::getLowBitsSet(XLenVT.getSizeInBits(), ElemSizeInBits), DL, XLenVT);
  auto pack = [&](SDValue A, SDValue B) {
    // Bias the scheduling of the inserted operations to near the
    // definition of the element - this tends to reduce register
    // pressure overall.
    SDLoc ElemDL(B);
    if (Subtarget.hasStdExtZbkb())
      // Note that we're relying on the high bits of the result being
      // don't care.  For PACKW, the result is *sign* extended.
      return SDValue(
          DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget),
                             ElemDL, XLenVT, A, B),
          0);
 
    A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
    B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
    SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
    return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
                       DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt),
                       SDNodeFlags::Disjoint);
  };
 
  SmallVector<SDValue> NewOperands;
  NewOperands.reserve(NumElts / 2);
  for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
    NewOperands.push_back(pack(Op.getOperand(i), Op.getOperand(i + 1)));
  assert(NumElts == NewOperands.size() * 2);
  MVT WideVT = MVT::getIntegerVT(ElemSizeInBits * 2);
  MVT WideVecVT = MVT::getVectorVT(WideVT, NumElts / 2);
  return DAG.getNode(ISD::BITCAST, DL, VT,
                     DAG.getBuildVector(WideVecVT, DL, NewOperands));
}
 
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  MVT VT = Op.getSimpleValueType();
  assert(VT.isFixedLengthVector() && "Unexpected vector!");
 
  MVT EltVT = VT.getVectorElementType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  SDLoc DL(Op);
 
  if (Subtarget.isRV32() && Subtarget.hasStdExtP()) {
    if (VT == MVT::v2i16) {
      SDValue Lo = DAG.getBitcast(
          MVT::v2i16,
          DAG.getAnyExtOrTrunc(Op->getOperand(0), DL, MVT::i32));
      SDValue Hi = DAG.getBitcast(
          MVT::v2i16,
          DAG.getAnyExtOrTrunc(Op->getOperand(1), DL, MVT::i32));
      return DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16, Lo, Hi);
    }
 
    if (VT == MVT::v4i8) {
      // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b,
      // d))
      SDValue Val0 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(0));
      SDValue Val1 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(1));
      SDValue Val2 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(2));
      SDValue Val3 =
          DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(3));
      SDValue Concat1 =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val0, Val2);
      SDValue Concat2 =
          DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Val1, Val3);
      SDValue PPairE =
          DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v8i8, Concat1, Concat2);
 
      SDValue Lo = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 0);
      SDValue Hi = DAG.getExtractSubvector(DL, MVT::v4i8, PPairE, 4);
 
      return DAG.getBitcast(MVT::v4i8,
                            DAG.getNode(RISCVISD::PPAIRE, DL, MVT::v2i16,
                                        DAG.getBitcast(MVT::v2i16, Lo),
                                        DAG.getBitcast(MVT::v2i16, Hi)));
    }
 
    llvm_unreachable("Unexpected RV32 P BUILD_VECTOR type");
  }
 
  // Proper support for f16 requires Zvfh. bf16 always requires special
  // handling. We need to cast the scalar to integer and create an integer
  // build_vector.
  if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
      (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
    MVT IVT = VT.changeVectorElementType(MVT::i16);
    SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
    for (const auto &[I, U] : enumerate(Op->ops())) {
      SDValue Elem = U.get();
      if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
          (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
        // Called by LegalizeDAG, we need to use XLenVT operations since we
        // can't create illegal types.
        if (auto *C = dyn_cast<ConstantFPSDNode>(Elem)) {
          // Manually constant fold so the integer build_vector can be lowered
          // better. Waiting for DAGCombine will be too late.
          APInt V =
              C->getValueAPF().bitcastToAPInt().sext(XLenVT.getSizeInBits());
          NewOps[I] = DAG.getConstant(V, DL, XLenVT);
        } else {
          NewOps[I] = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Elem);
        }
      } else {
        // Called by scalar type legalizer, we can use i16.
        NewOps[I] = DAG.getBitcast(MVT::i16, Op.getOperand(I));
      }
    }
    SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, DL, IVT, NewOps);
    return DAG.getBitcast(VT, Res);
  }
 
  if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
      ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
    return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  if (VT.getVectorElementType() == MVT::i1) {
    // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
    // vector type, we have a legal equivalently-sized i8 type, so we can use
    // that.
    MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
    SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
 
    SDValue WideVec;
    if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
      // For a splat, perform a scalar truncate before creating the wider
      // vector.
      Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
                          DAG.getConstant(1, DL, Splat.getValueType()));
      WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
    } else {
      SmallVector<SDValue, 8> Ops(Op->op_values());
      WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
      SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
      WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
    }
 
    return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
  }
 
  if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
    if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
      return Gather;
 
    if (!VT.isFloatingPoint())
      Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
 
    // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
    // pressure at high LMUL.
    bool IsScalar = all_of(Op->ops().drop_front(),
                           [](const SDUse &U) { return U.get().isUndef(); });
    unsigned Opc =
        VT.isFloatingPoint()
            ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
            : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
    Splat =
        DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
    return convertFromScalableVector(VT, Splat, DAG, Subtarget);
  }
 
  if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
    return Res;
 
  // If we're compiling for an exact VLEN value, we can split our work per
  // register in the register group.
  if (const auto VLen = Subtarget.getRealVLen();
      VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
    MVT ElemVT = VT.getVectorElementType();
    unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
    EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
    MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
    assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
 
    // The following semantically builds up a fixed length concat_vector
    // of the component build_vectors.  We eagerly lower to scalable and
    // insert_subvector here to avoid DAG combining it back to a large
    // build_vector.
    SmallVector<SDValue> BuildVectorOps(Op->ops());
    unsigned NumOpElts = M1VT.getVectorMinNumElements();
    SDValue Vec = DAG.getUNDEF(ContainerVT);
    for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
      auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(i, ElemsPerVReg);
      SDValue SubBV =
          DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
      SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget);
      unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
      Vec = DAG.getInsertSubvector(DL, Vec, SubBV, InsertIdx);
    }
    return convertFromScalableVector(VT, Vec, DAG, Subtarget);
  }
 
  // If we're about to resort to vslide1down (or stack usage), pack our
  // elements into the widest scalar type we can.  This will force a VL/VTYPE
  // toggle, but reduces the critical path, the number of vslide1down ops
  // required, and possibly enables scalar folds of the values.
  if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
    return Res;
 
  // For m1 vectors, if we have non-undef values in both halves of our vector,
  // split the vector into low and high halves, build them separately, then
  // use a vselect to combine them.  For long vectors, this cuts the critical
  // path of the vslide1down sequence in half, and gives us an opportunity
  // to special case each half independently.  Note that we don't change the
  // length of the sub-vectors here, so if both fallback to the generic
  // vslide1down path, we should be able to fold the vselect into the final
  // vslidedown (for the undef tail) for the first half w/ masking.
  unsigned NumElts = VT.getVectorNumElements();
  unsigned NumUndefElts =
      count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
  unsigned NumDefElts = NumElts - NumUndefElts;
  if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
      ContainerVT.bitsLE(RISCVTargetLowering::getM1VT(ContainerVT))) {
    SmallVector<SDValue> SubVecAOps, SubVecBOps;
    SmallVector<SDValue> MaskVals;
    SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
    SubVecAOps.reserve(NumElts);
    SubVecBOps.reserve(NumElts);
    for (const auto &[Idx, U] : enumerate(Op->ops())) {
      SDValue Elem = U.get();
      if (Idx < NumElts / 2) {
        SubVecAOps.push_back(Elem);
        SubVecBOps.push_back(UndefElem);
      } else {
        SubVecAOps.push_back(UndefElem);
        SubVecBOps.push_back(Elem);
      }
      bool SelectMaskVal = (Idx < NumElts / 2);
      MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
    }
    assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
           MaskVals.size() == NumElts);
 
    SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
    SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
    MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
    SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
    return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
  }
 
  // Cap the cost at a value linear to the number of elements in the vector.
  // The default lowering is to use the stack.  The vector store + scalar loads
  // is linear in VL.  However, at high lmuls vslide1down and vslidedown end up
  // being (at least) linear in LMUL.  As a result, using the vslidedown
  // lowering for every element ends up being VL*LMUL..
  // TODO: Should we be directly costing the stack alternative?  Doing so might
  // give us a more accurate upper bound.
  InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
 
  // TODO: unify with TTI getSlideCost.
  InstructionCost PerSlideCost = 1;
  switch (RISCVTargetLowering::getLMUL(ContainerVT)) {
  default: break;
  case RISCVVType::LMUL_2:
    PerSlideCost = 2;
    break;
  case RISCVVType::LMUL_4:
    PerSlideCost = 4;
    break;
  case RISCVVType::LMUL_8:
    PerSlideCost = 8;
    break;
  }
 
  // TODO: Should we be using the build instseq then cost + evaluate scheme
  // we use for integer constants here?
  unsigned UndefCount = 0;
  for (const SDValue &V : Op->ops()) {
    if (V.isUndef()) {
      UndefCount++;
      continue;
    }
    if (UndefCount) {
      LinearBudget -= PerSlideCost;
      UndefCount = 0;
    }
    LinearBudget -= PerSlideCost;
  }
  if (UndefCount) {
    LinearBudget -= PerSlideCost;
  }
 
  if (LinearBudget < 0)
    return SDValue();
 
  assert((!VT.isFloatingPoint() ||
          VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
         "Illegal type which will result in reserved encoding");
 
  const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
 
  // General case: splat the first operand and slide other operands down one
  // by one to form a vector. Alternatively, if every operand is an
  // extraction from element 0 of a vector, we use that vector from the last
  // extraction as the start value and slide up instead of slide down. Such that
  // (1) we can avoid the initial splat (2) we can turn those vslide1up into
  // vslideup of 1 later and eliminate the vector to scalar movement, which is
  // something we cannot do with vslide1down/vslidedown.
  // Of course, using vslide1up/vslideup might increase the register pressure,
  // and that's why we conservatively limit to cases where every operand is an
  // extraction from the first element.
  SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
  SDValue EVec;
  bool SlideUp = false;
  auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
                       SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
    if (SlideUp)
      return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
                         Mask, VL, Policy);
    return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
                         Mask, VL, Policy);
  };
 
  // The reason we don't use all_of here is because we're also capturing EVec
  // from the last non-undef operand. If the std::execution_policy of the
  // underlying std::all_of is anything but std::sequenced_policy we might
  // capture the wrong EVec.
  for (SDValue V : Operands) {
    using namespace SDPatternMatch;
    SlideUp = V.isUndef() || sd_match(V, m_ExtractElt(m_Value(EVec), m_Zero()));
    if (!SlideUp)
      break;
  }
 
  // Do not slideup if the element type of EVec is different.
  if (SlideUp) {
    MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
    MVT ContainerEltVT = ContainerVT.getVectorElementType();
    if (EVecEltVT != ContainerEltVT)
      SlideUp = false;
  }
 
  if (SlideUp) {
    MVT EVecContainerVT = EVec.getSimpleValueType();
    // Make sure the original vector has scalable vector type.
    if (EVecContainerVT.isFixedLengthVector()) {
      EVecContainerVT =
          getContainerForFixedLengthVector(EVecContainerVT, Subtarget);
      EVec = convertToScalableVector(EVecContainerVT, EVec, DAG, Subtarget);
    }
 
    // Adapt EVec's type into ContainerVT.
    if (EVecContainerVT.getVectorMinNumElements() <
        ContainerVT.getVectorMinNumElements())
      EVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), EVec, 0);
    else
      EVec = DAG.getExtractSubvector(DL, ContainerVT, EVec, 0);
 
    // Reverse the elements as we're going to slide up from the last element.
    std::reverse(Operands.begin(), Operands.end());
  }
 
  SDValue Vec;
  UndefCount = 0;
  for (SDValue V : Operands) {
    if (V.isUndef()) {
      UndefCount++;
      continue;
    }
 
    // Start our sequence with either a TA splat or extract source in the
    // hopes that hardware is able to recognize there's no dependency on the
    // prior value of our temporary register.
    if (!Vec) {
      if (SlideUp) {
        Vec = EVec;
      } else {
        Vec = DAG.getSplatVector(VT, DL, V);
        Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
      }
 
      UndefCount = 0;
      continue;
    }
 
    if (UndefCount) {
      const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
      Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
                      VL);
      UndefCount = 0;
    }
 
    unsigned Opcode;
    if (VT.isFloatingPoint())
      Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
    else
      Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
 
    if (!VT.isFloatingPoint())
      V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
    Vec = DAG.getNode(Opcode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
                      V, Mask, VL);
  }
  if (UndefCount) {
    const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
    Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
                    VL);
  }
  return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}
 
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
                                   SDValue Lo, SDValue Hi, SDValue VL,
                                   SelectionDAG &DAG) {
  if (!Passthru)
    Passthru = DAG.getUNDEF(VT);
  if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
    int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
    int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
    // If Hi constant is all the same sign bit as Lo, lower this as a custom
    // node in order to try and match RVV vector/scalar instructions.
    if ((LoC >> 31) == HiC)
      return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
 
    // Use vmv.v.x with EEW=32.  Use either a vsetivli or vsetvli to change
    // VL.  This can temporarily increase VL if VL less than VLMAX.
    if (LoC == HiC) {
      SDValue NewVL;
      if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
        NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
      else
        NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
      MVT InterVT =
          MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
      auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
                                  DAG.getUNDEF(InterVT), Lo, NewVL);
      return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
    }
  }
 
  // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
  if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
      isa<ConstantSDNode>(Hi.getOperand(1)) &&
      Hi.getConstantOperandVal(1) == 31)
    return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
 
  // If the hi bits of the splat are undefined, then it's fine to just splat Lo
  // even if it might be sign extended.
  if (Hi.isUndef())
    return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
 
  // Fall back to a stack store and stride x0 vector load.
  return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
                     Hi, VL);
}
 
// Called by type legalization to handle splat of i64 on RV32.
// FIXME: We can optimize this when the type has sign or zero bits in one
// of the halves.
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
                                   SDValue Scalar, SDValue VL,
                                   SelectionDAG &DAG) {
  assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
  SDValue Lo, Hi;
  std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32);
  return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
}
 
// This function lowers a splat of a scalar operand Splat with the vector
// length VL. It ensures the final sequence is type legal, which is useful when
// lowering a splat after type legalization.
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
                                MVT VT, const SDLoc &DL, SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  bool HasPassthru = Passthru && !Passthru.isUndef();
  if (!HasPassthru && !Passthru)
    Passthru = DAG.getUNDEF(VT);
 
  MVT EltVT = VT.getVectorElementType();
  MVT XLenVT = Subtarget.getXLenVT();
 
  if (VT.isFloatingPoint()) {
    if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
        (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
      if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
          (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
        Scalar = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Scalar);
      else
        Scalar = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Scalar);
      MVT IVT = VT.changeVectorElementType(MVT::i16);
      Passthru = DAG.getNode(ISD::BITCAST, DL, IVT, Passthru);
      SDValue Splat =
          lowerScalarSplat(Passthru, Scalar, VL, IVT, DL, DAG, Subtarget);
      return DAG.getNode(ISD::BITCAST, DL, VT, Splat);
    }
    return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
  }
 
  // Simplest case is that the operand needs to be promoted to XLenVT.
  if (Scalar.getValueType().bitsLE(XLenVT)) {
    // If the operand is a constant, sign extend to increase our chances
    // of being able to use a .vi instruction. ANY_EXTEND would become a
    // a zero extend and the simm5 check in isel would fail.
    // FIXME: Should we ignore the upper bits in isel instead?
    unsigned ExtOpc =
        isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
    Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
    return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
  }
 
  assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
         "Unexpected scalar for splat lowering!");
 
  if (isOneConstant(VL) && isNullConstant(Scalar))
    return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
                       DAG.getConstant(0, DL, XLenVT), VL);
 
  // Otherwise use the more complicated splatting algorithm.
  return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
}
 
// This function lowers an insert of a scalar operand Scalar into lane
// 0 of the vector regardless of the value of VL.  The contents of the
// remaining lanes of the result vector are unspecified.  VL is assumed
// to be non-zero.
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
                                 const SDLoc &DL, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
 
  const MVT XLenVT = Subtarget.getXLenVT();
  SDValue Passthru = DAG.getUNDEF(VT);
 
  if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
      isNullConstant(Scalar.getOperand(1))) {
    SDValue ExtractedVal = Scalar.getOperand(0);
    // The element types must be the same.
    if (ExtractedVal.getValueType().getVectorElementType() ==
        VT.getVectorElementType()) {
      MVT ExtractedVT = ExtractedVal.getSimpleValueType();
      MVT ExtractedContainerVT = ExtractedVT;
      if (ExtractedContainerVT.isFixedLengthVector()) {
        ExtractedContainerVT =
            getContainerForFixedLengthVector(ExtractedContainerVT, Subtarget);
        ExtractedVal = convertToScalableVector(ExtractedContainerVT,
                                               ExtractedVal, DAG, Subtarget);
      }
      if (ExtractedContainerVT.bitsLE(VT))
        return DAG.getInsertSubvector(DL, Passthru, ExtractedVal, 0);
      return DAG.getExtractSubvector(DL, VT, ExtractedVal, 0);
    }
  }
 
  if (VT.isFloatingPoint())
    return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
                       VL);
 
  // Avoid the tricky legalization cases by falling back to using the
  // splat code which already handles it gracefully.
  if (!Scalar.getValueType().bitsLE(XLenVT))
    return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
                            DAG.getConstant(1, DL, XLenVT),
                            VT, DL, DAG, Subtarget);
 
  // If the operand is a constant, sign extend to increase our chances
  // of being able to use a .vi instruction. ANY_EXTEND would become a
  // a zero extend and the simm5 check in isel would fail.
  // FIXME: Should we ignore the upper bits in isel instead?
  unsigned ExtOpc =
      isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
  Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
  return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
                     VL);
}
 
/// If concat_vector(V1,V2) could be folded away to some existing
/// vector source, return it.  Note that the source may be larger
/// than the requested concat_vector (i.e. a extract_subvector
/// might be required.)
static SDValue foldConcatVector(SDValue V1, SDValue V2) {
  EVT VT = V1.getValueType();
  assert(VT == V2.getValueType() && "argument types must match");
  // Both input must be extracts.
  if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
      V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
    return SDValue();
 
  // Extracting from the same source.
  SDValue Src = V1.getOperand(0);
  if (Src != V2.getOperand(0) ||
      VT.isScalableVector() != Src.getValueType().isScalableVector())
    return SDValue();
 
  // The extracts must extract the two halves of the source.
  if (V1.getConstantOperandVal(1) != 0 ||
      V2.getConstantOperandVal(1) != VT.getVectorMinNumElements())
    return SDValue();
 
  return Src;
}
 
// Can this shuffle be performed on exactly one (possibly larger) input?
static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
 
  if (V2.isUndef())
    return V1;
 
  unsigned NumElts = VT.getVectorNumElements();
  // Src needs to have twice the number of elements.
  // TODO: Update shuffle lowering to add the extract subvector
  if (SDValue Src = foldConcatVector(V1, V2);
      Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
    return Src;
 
  return SDValue();
}
 
static bool isLegalVTForZvzipOperand(MVT VT, const RISCVSubtarget &Subtarget) {
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector())
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  // Determine LMUL of the container vector.
  return RISCVTargetLowering::getLMUL(ContainerVT) != RISCVVType::LMUL_8;
}
 
/// Is this shuffle interleaving contiguous elements from one vector into the
/// even elements and contiguous elements from another vector into the odd
/// elements. \p EvenSrc will contain the element that should be in the first
/// even element. \p OddSrc will contain the element that should be in the first
/// odd element. These can be the first element in a source or the element half
/// way through the source.
static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
                                int &OddSrc, const RISCVSubtarget &Subtarget) {
  // We need to be able to widen elements to the next larger integer type or
  // use the vzip instruction at e64.
  if (VT.getScalarSizeInBits() >= Subtarget.getELen()) {
    if (!Subtarget.hasStdExtZvzip())
      return false;
    if (!isLegalVTForZvzipOperand(VT, Subtarget))
      return false;
  }
 
  int Size = Mask.size();
  int NumElts = VT.getVectorNumElements();
  assert(Size == (int)NumElts && "Unexpected mask size");
 
  SmallVector<unsigned, 2> StartIndexes;
  if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, Size * 2, StartIndexes))
    return false;
 
  EvenSrc = StartIndexes[0];
  OddSrc = StartIndexes[1];
 
  // One source should be low half of first vector.
  if (EvenSrc != 0 && OddSrc != 0)
    return false;
 
  // Subvectors will be subtracted from either at the start of the two input
  // vectors, or at the start and middle of the first vector if it's an unary
  // interleave.
  // In both cases, HalfNumElts will be extracted.
  // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
  // we'll create an illegal extract_subvector.
  // FIXME: We could support other values using a slidedown first.
  int HalfNumElts = NumElts / 2;
  return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
}
 
/// Is this mask representing a masked combination of two slides?
static bool isMaskedSlidePair(ArrayRef<int> Mask,
                              std::array<std::pair<int, int>, 2> &SrcInfo) {
  if (!llvm::isMaskedSlidePair(Mask, Mask.size(), SrcInfo))
    return false;
 
  // Avoid matching vselect idioms
  if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
    return false;
  // Prefer vslideup as the second instruction, and identity
  // only as the initial instruction.
  if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
      SrcInfo[1].second == 0)
    std::swap(SrcInfo[0], SrcInfo[1]);
  assert(SrcInfo[0].first != -1 && "Must find one slide");
  return true;
}
 
// Exactly matches the semantics of a previously existing custom matcher
// to allow migration to new matcher without changing output.
static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
                            unsigned NumElts) {
  if (SrcInfo[1].first == -1)
    return true;
  return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
         SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
}
 
static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
                          ArrayRef<int> Mask, unsigned Factor,
                          bool RequiredPolarity) {
  int NumElts = Mask.size();
  for (const auto &[Idx, M] : enumerate(Mask)) {
    if (M < 0)
      continue;
    int Src = M >= NumElts;
    int Diff = (int)Idx - (M % NumElts);
    bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
    assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
           "Must match exactly one of the two slides");
    if (RequiredPolarity != (C == (Idx / Factor) % 2))
      return false;
  }
  return true;
}
 
/// Given a shuffle which can be represented as a pair of two slides,
/// see if it is a pair-even idiom.
/// Pair-even is:
/// vs2: a0 a1 a2 a3
/// vs1: b0 b1 b2 b3
/// vd:  a0 b0 a2 b2
static bool isPairEven(const std::array<std::pair<int, int>, 2> &SrcInfo,
                       ArrayRef<int> Mask, unsigned &Factor) {
  Factor = SrcInfo[1].second;
  return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
         Mask.size() % Factor == 0 &&
         isAlternating(SrcInfo, Mask, Factor, true);
}
 
/// Given a shuffle which can be represented as a pair of two slides,
/// see if it is a pair-odd idiom.
/// Pair-odd is:
/// vs2: a0 a1 a2 a3
/// vs1: b0 b1 b2 b3
/// vd:  a1 b1 a3 b3
/// Note that the operand order is swapped due to the way we canonicalize
/// the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.
static bool isPairOdd(const std::array<std::pair<int, int>, 2> &SrcInfo,
                      ArrayRef<int> Mask, unsigned &Factor) {
  Factor = -SrcInfo[1].second;
  return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
         Mask.size() % Factor == 0 &&
         isAlternating(SrcInfo, Mask, Factor, false);
}
 
// Lower a deinterleave shuffle to SRL and TRUNC.  Factor must be
// 2, 4, 8 and the integer type Factor-times larger than VT's
// element type must be a legal element type.
// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
//                          -> [p, q, r, s] (Factor=2, Index=1)
static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
                                            SDValue Src, unsigned Factor,
                                            unsigned Index, SelectionDAG &DAG) {
  unsigned EltBits = VT.getScalarSizeInBits();
  ElementCount SrcEC = Src.getValueType().getVectorElementCount();
  MVT WideSrcVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor),
                                   SrcEC.divideCoefficientBy(Factor));
  MVT ResVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits),
                               SrcEC.divideCoefficientBy(Factor));
  Src = DAG.getBitcast(WideSrcVT, Src);
 
  unsigned Shift = Index * EltBits;
  SDValue Res = DAG.getNode(ISD::SRL, DL, WideSrcVT, Src,
                            DAG.getConstant(Shift, DL, WideSrcVT));
  Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT, Res);
  MVT CastVT = ResVT.changeVectorElementType(VT.getVectorElementType());
  Res = DAG.getBitcast(CastVT, Res);
  return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
}
 
/// Match a single source shuffle which is an identity except that some
/// particular element is repeated.  This can be lowered as a masked
/// vrgather.vi/vx.  Note that the two source form of this is handled
/// by the recursive splitting logic and doesn't need special handling.
static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
                                               const RISCVSubtarget &Subtarget,
                                               SelectionDAG &DAG) {
 
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  SDValue V1 = SVN->getOperand(0);
  assert(SVN->getOperand(1).isUndef());
  ArrayRef<int> Mask = SVN->getMask();
  const unsigned NumElts = VT.getVectorNumElements();
  MVT XLenVT = Subtarget.getXLenVT();
 
  std::optional<int> SplatIdx;
  for (auto [I, M] : enumerate(Mask)) {
    if (M == -1 || I == (unsigned)M)
      continue;
    if (SplatIdx && *SplatIdx != M)
      return SDValue();
    SplatIdx = M;
  }
 
  if (!SplatIdx)
    return SDValue();
 
  SmallVector<SDValue> MaskVals;
  for (int MaskIndex : Mask) {
    bool SelectMaskVal = MaskIndex == *SplatIdx;
    MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
  }
  assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
  SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
  SDValue Splat = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT),
                                       SmallVector<int>(NumElts, *SplatIdx));
  return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, Splat, V1);
}
 
// Lower the following shuffle to vslidedown.
// a)
// t49: v8i8 = extract_subvector t13, Constant:i64<0>
// t109: v8i8 = extract_subvector t13, Constant:i64<8>
// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
// b)
// t69: v16i16 = extract_subvector t68, Constant:i64<0>
// t23: v8i16 = extract_subvector t69, Constant:i64<0>
// t29: v4i16 = extract_subvector t23, Constant:i64<4>
// t26: v8i16 = extract_subvector t69, Constant:i64<8>
// t30: v4i16 = extract_subvector t26, Constant:i64<0>
// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
                                               SDValue V1, SDValue V2,
                                               ArrayRef<int> Mask,
                                               const RISCVSubtarget &Subtarget,
                                               SelectionDAG &DAG) {
  auto findNonEXTRACT_SUBVECTORParent =
      [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
    uint64_t Offset = 0;
    while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
           // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
           // a scalable vector. But we don't want to match the case.
           Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
      Offset += Parent.getConstantOperandVal(1);
      Parent = Parent.getOperand(0);
    }
    return std::make_pair(Parent, Offset);
  };
 
  auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
  auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
 
  // Extracting from the same source.
  SDValue Src = V1Src;
  if (Src != V2Src)
    return SDValue();
 
  // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
  SmallVector<int, 16> NewMask(Mask);
  for (size_t i = 0; i != NewMask.size(); ++i) {
    if (NewMask[i] == -1)
      continue;
 
    if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
      NewMask[i] = NewMask[i] + V1IndexOffset;
    } else {
      // Minus NewMask.size() is needed. Otherwise, the b case would be
      // <5,6,7,12> instead of <5,6,7,8>.
      NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
    }
  }
 
  // First index must be known and non-zero. It will be used as the slidedown
  // amount.
  if (NewMask[0] <= 0)
    return SDValue();
 
  // NewMask is also continuous.
  for (unsigned i = 1; i != NewMask.size(); ++i)
    if (NewMask[i - 1] + 1 != NewMask[i])
      return SDValue();
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT SrcVT = Src.getSimpleValueType();
  MVT ContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
  auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
  SDValue Slidedown =
      getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
                    convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
                    DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
  return DAG.getExtractSubvector(
      DL, VT, convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget), 0);
}
 
// Because vslideup leaves the destination elements at the start intact, we can
// use it to perform shuffles that insert subvectors:
//
// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
// ->
// vsetvli zero, 8, e8, mf2, ta, ma
// vslideup.vi v8, v9, 4
//
// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
// ->
// vsetvli zero, 5, e8, mf2, tu, ma
// vslideup.v1 v8, v9, 2
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
                                             SDValue V1, SDValue V2,
                                             ArrayRef<int> Mask,
                                             const RISCVSubtarget &Subtarget,
                                             SelectionDAG &DAG) {
  unsigned NumElts = VT.getVectorNumElements();
  int NumSubElts, Index;
  if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumElts, NumSubElts,
                                                Index))
    return SDValue();
 
  bool OpsSwapped = Mask[Index] < (int)NumElts;
  SDValue InPlace = OpsSwapped ? V2 : V1;
  SDValue ToInsert = OpsSwapped ? V1 : V2;
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
  // We slide up by the index that the subvector is being inserted at, and set
  // VL to the index + the number of elements being inserted.
  unsigned Policy =
      RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
  // If the we're adding a suffix to the in place vector, i.e. inserting right
  // up to the very end of it, then we don't actually care about the tail.
  if (NumSubElts + Index >= (int)NumElts)
    Policy |= RISCVVType::TAIL_AGNOSTIC;
 
  InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget);
  ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget);
  SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT);
 
  SDValue Res;
  // If we're inserting into the lowest elements, use a tail undisturbed
  // vmv.v.v.
  if (Index == 0)
    Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
                      VL);
  else
    Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
                      DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy);
  return convertFromScalableVector(VT, Res, DAG, Subtarget);
}
 
// A shuffle of shuffles where the final data only is drawn from 2 input ops
// can be compressed into a single shuffle
static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
                                         const RISCVSubtarget &Subtarget,
                                         SelectionDAG &DAG) {
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
 
  if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
      V2.getOpcode() != ISD::VECTOR_SHUFFLE)
    return SDValue();
 
  if (!V1.hasOneUse() || !V2.hasOneUse())
    return SDValue();
 
  ArrayRef<int> Mask = SVN->getMask();
  ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(V1.getNode())->getMask();
  ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(V2.getNode())->getMask();
  unsigned NumElts = Mask.size();
  SmallVector<int> NewMask(NumElts, -1);
  for (unsigned Idx : seq<unsigned>(NumElts)) {
    int Lane = Mask[Idx];
    // Don't assign if poison
    if (Lane == -1)
      continue;
    int OrigLane;
    bool SecondOp = false;
    if ((unsigned)Lane < NumElts) {
      OrigLane = V1Mask[Lane];
    } else {
      OrigLane = V2Mask[Lane - NumElts];
      SecondOp = true;
    }
    if (OrigLane == -1)
      continue;
    // Don't handle if shuffling from a second operand
    if ((unsigned)OrigLane >= NumElts)
      return SDValue();
    if (SecondOp)
      OrigLane += NumElts;
    NewMask[Idx] = OrigLane;
  }
 
  EVT VT = SVN->getValueType(0);
  SDLoc DL(SVN);
 
  return DAG.getVectorShuffle(VT, DL, V1->getOperand(0), V2->getOperand(0),
                              NewMask);
}
 
/// Match v(f)slide1up/down idioms.  These operations involve sliding
/// N-1 elements to make room for an inserted scalar at one end.
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
                                            SDValue V1, SDValue V2,
                                            ArrayRef<int> Mask,
                                            const RISCVSubtarget &Subtarget,
                                            SelectionDAG &DAG) {
  bool OpsSwapped = false;
  if (!isa<BuildVectorSDNode>(V1)) {
    if (!isa<BuildVectorSDNode>(V2))
      return SDValue();
    std::swap(V1, V2);
    OpsSwapped = true;
  }
  SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
  if (!Splat)
    return SDValue();
 
  // Return true if the mask could describe a slide of Mask.size() - 1
  // elements from concat_vector(V1, V2)[Base:] to [Offset:].
  auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
    const unsigned S = (Offset > 0) ? 0 : -Offset;
    const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
    for (unsigned i = S; i != E; ++i)
      if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
        return false;
    return true;
  };
 
  const unsigned NumElts = VT.getVectorNumElements();
  bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
  if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
    return SDValue();
 
  const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
  // Inserted lane must come from splat, undef scalar is legal but not profitable.
  if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
    return SDValue();
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
  // vslide1{down,up}.vx instead.
  if ((VT.getVectorElementType() == MVT::bf16 &&
       !Subtarget.hasVInstructionsBF16()) ||
      (VT.getVectorElementType() == MVT::f16 &&
       !Subtarget.hasVInstructionsF16())) {
    MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
    Splat =
        DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(), Splat);
    V2 = DAG.getBitcast(
        IntVT, convertToScalableVector(ContainerVT, V2, DAG, Subtarget));
    SDValue Vec = DAG.getNode(
        IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
        IntVT, DAG.getUNDEF(IntVT), V2, Splat, TrueMask, VL);
    Vec = DAG.getBitcast(ContainerVT, Vec);
    return convertFromScalableVector(VT, Vec, DAG, Subtarget);
  }
 
  auto OpCode = IsVSlidedown ?
    (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
    (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
  if (!VT.isFloatingPoint())
    Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
  auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
                         DAG.getUNDEF(ContainerVT),
                         convertToScalableVector(ContainerVT, V2, DAG, Subtarget),
                         Splat, TrueMask, VL);
  return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}
 
/// Match a mask which "spreads" the leading elements of a vector evenly
/// across the result.  Factor is the spread amount, and Index is the
/// offset applied.  (on success, Index < Factor)  This is the inverse
/// of a deinterleave with the same Factor and Index.  This is analogous
/// to an interleave, except that all but one lane is undef.
bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
                                       unsigned &Index) {
  SmallVector<bool> LaneIsUndef(Factor, true);
  for (unsigned i = 0; i < Mask.size(); i++)
    LaneIsUndef[i % Factor] &= (Mask[i] == -1);
 
  bool Found = false;
  for (unsigned i = 0; i < Factor; i++) {
    if (LaneIsUndef[i])
      continue;
    if (Found)
      return false;
    Index = i;
    Found = true;
  }
  if (!Found)
    return false;
 
  for (unsigned i = 0; i < Mask.size() / Factor; i++) {
    unsigned j = i * Factor + Index;
    if (Mask[j] != -1 && (unsigned)Mask[j] != i)
      return false;
  }
  return true;
}
 
static SDValue lowerZvzipVPAIR(unsigned Opc, SDValue Op0, SDValue Op1,
                               const SDLoc &DL, SelectionDAG &DAG,
                               const RISCVSubtarget &Subtarget) {
  assert(RISCVISD::VPAIRE_VL == Opc || RISCVISD::VPAIRO_VL == Opc);
  assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
 
  MVT VT = Op0.getSimpleValueType();
  MVT IntVT = VT.changeVectorElementTypeToInteger();
  Op0 = DAG.getBitcast(IntVT, Op0);
  Op1 = DAG.getBitcast(IntVT, Op1);
 
  MVT ContainerVT = IntVT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
    Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
  }
 
  MVT InnerVT = ContainerVT;
  auto [Mask, VL] = getDefaultVLOps(IntVT, InnerVT, DL, DAG, Subtarget);
 
  SDValue Passthru = DAG.getUNDEF(InnerVT);
  SDValue Res = DAG.getNode(Opc, DL, InnerVT, Op0, Op1, Passthru, Mask, VL);
  if (IntVT.isFixedLengthVector())
    Res = convertFromScalableVector(IntVT, Res, DAG, Subtarget);
  Res = DAG.getBitcast(VT, Res);
  return Res;
}
 
static SDValue lowerZvzipVZIP(SDValue Op0, SDValue Op1, const SDLoc &DL,
                              SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
  MVT VT = Op0.getSimpleValueType();
  MVT IntVT = VT.changeVectorElementTypeToInteger();
  Op0 = DAG.getBitcast(IntVT, Op0);
  Op1 = DAG.getBitcast(IntVT, Op1);
  MVT ContainerVT = IntVT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
    Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
    Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
  }
  MVT ResVT = ContainerVT.getDoubleNumVectorElementsVT();
  auto [Mask, VL] = getDefaultVLOps(IntVT, ContainerVT, DL, DAG, Subtarget);
  SDValue Passthru = DAG.getUNDEF(ResVT);
  SDValue Res =
      DAG.getNode(RISCVISD::VZIP_VL, DL, ResVT, Op0, Op1, Passthru, Mask, VL);
  if (IntVT.isFixedLengthVector())
    Res = convertFromScalableVector(IntVT.getDoubleNumVectorElementsVT(), Res,
                                    DAG, Subtarget);
  Res = DAG.getBitcast(VT.getDoubleNumVectorElementsVT(), Res);
  return Res;
}
 
static SDValue lowerZvzipVUNZIP(unsigned Opc, SDValue Op, const SDLoc &DL,
                                SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
  assert(Opc == RISCVISD::VUNZIPE_VL || Opc == RISCVISD::VUNZIPO_VL);
  MVT VT = Op.getSimpleValueType();
  assert(VT.getVectorMinNumElements() >= 2);
 
  MVT IntVT = VT.changeVectorElementTypeToInteger();
  Op = DAG.getBitcast(IntVT, Op);
  MVT ContainerVT = IntVT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
    // For E64 with LMUL <= 1, we can't represent a smaller fractional LMUL for
    // the result (LMUL <= 1/2 is not valid for E64). We must widen the input
    // container to at least LMUL=2 so the result can be LMUL=1.
    if (ContainerVT.getVectorElementType() == MVT::i64 &&
        RISCVTargetLowering::getLMUL(ContainerVT) == RISCVVType::LMUL_1) {
      ContainerVT = MVT::getScalableVectorVT(MVT::i64, 2);
    }
    Op = convertToScalableVector(ContainerVT, Op, DAG, Subtarget);
  }
 
  MVT ResVT = ContainerVT.getHalfNumVectorElementsVT();
  MVT HalfVT = VT.getHalfNumVectorElementsVT();
  MVT HalfIntVT = IntVT.getHalfNumVectorElementsVT();
  auto [Mask, VL] = getDefaultVLOps(ResVT, ResVT, DL, DAG, Subtarget);
  if (VT.isFixedLengthVector())
    VL = DAG.getConstant(VT.getVectorNumElements() / 2, DL,
                         Subtarget.getXLenVT());
  SDValue Passthru = DAG.getUNDEF(ResVT);
  SDValue Res = DAG.getNode(Opc, DL, ResVT, Op, Passthru, Mask, VL);
  if (HalfIntVT.isFixedLengthVector())
    Res = convertFromScalableVector(HalfIntVT, Res, DAG, Subtarget);
  Res = DAG.getBitcast(HalfVT, Res);
  return Res;
}
 
// Given a vector a, b, c, d return a vector Factor times longer
// with Factor-1 undef's between elements. Ex:
//   a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
//   undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
                                 const SDLoc &DL, SelectionDAG &DAG) {
 
  MVT VT = V.getSimpleValueType();
  unsigned EltBits = VT.getScalarSizeInBits();
  ElementCount EC = VT.getVectorElementCount();
  V = DAG.getBitcast(VT.changeTypeToInteger(), V);
 
  MVT WideVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor), EC);
 
  SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, V);
  // TODO: On rv32, the constant becomes a splat_vector_parts which does not
  // allow the SHL to fold away if Index is 0.
  if (Index != 0)
    Result = DAG.getNode(ISD::SHL, DL, WideVT, Result,
                         DAG.getConstant(EltBits * Index, DL, WideVT));
  // Make sure to use original element type
  MVT ResultVT = MVT::getVectorVT(VT.getVectorElementType(),
                                  EC.multiplyCoefficientBy(Factor));
  return DAG.getBitcast(ResultVT, Result);
}
 
// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
// to create an interleaved vector of <[vscale x] n*2 x ty>.
// This requires that the size of ty is less than the subtarget's maximum ELEN.
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
                                     const SDLoc &DL, SelectionDAG &DAG,
                                     const RISCVSubtarget &Subtarget) {
 
  // FIXME: Not only does this optimize the code, it fixes some correctness
  // issues because MIR does not have freeze.
  if (EvenV.isUndef())
    return getWideningSpread(OddV, 2, 1, DL, DAG);
  if (OddV.isUndef())
    return getWideningSpread(EvenV, 2, 0, DL, DAG);
 
  MVT VecVT = EvenV.getSimpleValueType();
  MVT VecContainerVT = VecVT; // <vscale x n x ty>
  // Convert fixed vectors to scalable if needed
  if (VecContainerVT.isFixedLengthVector()) {
    VecContainerVT = getContainerForFixedLengthVector(VecVT, Subtarget);
    EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget);
    OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget);
  }
 
  assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
 
  // We're working with a vector of the same size as the resulting
  // interleaved vector, but with half the number of elements and
  // twice the SEW (Hence the restriction on not using the maximum
  // ELEN)
  MVT WideVT =
      MVT::getVectorVT(MVT::getIntegerVT(VecVT.getScalarSizeInBits() * 2),
                       VecVT.getVectorElementCount());
  MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
  if (WideContainerVT.isFixedLengthVector())
    WideContainerVT = getContainerForFixedLengthVector(WideVT, Subtarget);
 
  // Bitcast the input vectors to integers in case they are FP
  VecContainerVT = VecContainerVT.changeTypeToInteger();
  EvenV = DAG.getBitcast(VecContainerVT, EvenV);
  OddV = DAG.getBitcast(VecContainerVT, OddV);
 
  auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
  SDValue Passthru = DAG.getUNDEF(WideContainerVT);
 
  SDValue Interleaved;
  if (Subtarget.hasStdExtZvbb()) {
    // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
    SDValue OffsetVec =
        DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
    Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
                              OffsetVec, Passthru, Mask, VL);
    Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
                              Interleaved, EvenV, Passthru, Mask, VL);
  } else {
    // FIXME: We should freeze the odd vector here. We already handled the case
    // of provably undef/poison above.
 
    // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
    // vwaddu.vv
    Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
                              OddV, Passthru, Mask, VL);
 
    // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
    SDValue AllOnesVec = DAG.getSplatVector(
        VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
    SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
                                  OddV, AllOnesVec, Passthru, Mask, VL);
 
    // Add the two together so we get
    //   (OddV * 0xff...ff) + (OddV + EvenV)
    // = (OddV * 0x100...00) + EvenV
    // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
    // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
    Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
                              Interleaved, OddsMul, Passthru, Mask, VL);
  }
 
  // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
  MVT ResultContainerVT = MVT::getVectorVT(
      VecVT.getVectorElementType(), // Make sure to use original type
      VecContainerVT.getVectorElementCount().multiplyCoefficientBy(2));
  Interleaved = DAG.getBitcast(ResultContainerVT, Interleaved);
 
  // Convert back to a fixed vector if needed
  MVT ResultVT =
      MVT::getVectorVT(VecVT.getVectorElementType(),
                       VecVT.getVectorElementCount().multiplyCoefficientBy(2));
  if (ResultVT.isFixedLengthVector())
    Interleaved =
        convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget);
 
  return Interleaved;
}
 
// If we have a vector of bits that we want to reverse, we can use a vbrev on a
// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
                                      SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  SDValue V = SVN->getOperand(0);
  unsigned NumElts = VT.getVectorNumElements();
 
  assert(VT.getVectorElementType() == MVT::i1);
 
  if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
                                        SVN->getMask().size()) ||
      !SVN->getOperand(1).isUndef())
    return SDValue();
 
  unsigned ViaEltSize = std::max((uint64_t)8, PowerOf2Ceil(NumElts));
  EVT ViaVT = EVT::getVectorVT(
      *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), ViaEltSize), 1);
  EVT ViaBitVT =
      EVT::getVectorVT(*DAG.getContext(), MVT::i1, ViaVT.getScalarSizeInBits());
 
  // If we don't have zvbb or the larger element type > ELEN, the operation will
  // be illegal.
  if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE,
                                                               ViaVT) ||
      !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT))
    return SDValue();
 
  // If the bit vector doesn't fit exactly into the larger element type, we need
  // to insert it into the larger vector and then shift up the reversed bits
  // afterwards to get rid of the gap introduced.
  if (ViaEltSize > NumElts)
    V = DAG.getInsertSubvector(DL, DAG.getUNDEF(ViaBitVT), V, 0);
 
  SDValue Res =
      DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
 
  // Shift up the reversed bits if the vector didn't exactly fit into the larger
  // element type.
  if (ViaEltSize > NumElts)
    Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
                      DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
 
  Res = DAG.getBitcast(ViaBitVT, Res);
 
  if (ViaEltSize > NumElts)
    Res = DAG.getExtractSubvector(DL, VT, Res, 0);
  return Res;
}
 
static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
                             const RISCVSubtarget &Subtarget,
                             MVT &RotateVT, unsigned &RotateAmt) {
  unsigned NumElts = VT.getVectorNumElements();
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  unsigned NumSubElts;
  if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, 2,
                                          NumElts, NumSubElts, RotateAmt))
    return false;
  RotateVT = MVT::getVectorVT(MVT::getIntegerVT(EltSizeInBits * NumSubElts),
                              NumElts / NumSubElts);
 
  // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
  return Subtarget.getTargetLowering()->isTypeLegal(RotateVT);
}
 
// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
                                           SelectionDAG &DAG,
                                           const RISCVSubtarget &Subtarget) {
  SDLoc DL(SVN);
 
  EVT VT = SVN->getValueType(0);
  unsigned RotateAmt;
  MVT RotateVT;
  if (!isLegalBitRotate(SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
    return SDValue();
 
  SDValue Op = DAG.getBitcast(RotateVT, SVN->getOperand(0));
 
  SDValue Rotate;
  // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
  // so canonicalize to vrev8.
  if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
    Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
  else
    Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
                         DAG.getConstant(RotateAmt, DL, RotateVT));
 
  return DAG.getBitcast(VT, Rotate);
}
 
// If compiling with an exactly known VLEN, see if we can split a
// shuffle on m2 or larger into a small number of m1 sized shuffles
// which write each destination registers exactly once.
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
                                            SelectionDAG &DAG,
                                            const RISCVSubtarget &Subtarget) {
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  ArrayRef<int> Mask = SVN->getMask();
 
  // If we don't know exact data layout, not much we can do.  If this
  // is already m1 or smaller, no point in splitting further.
  const auto VLen = Subtarget.getRealVLen();
  if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
    return SDValue();
 
  // Avoid picking up bitrotate patterns which we have a linear-in-lmul
  // expansion for.
  unsigned RotateAmt;
  MVT RotateVT;
  if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
    return SDValue();
 
  MVT ElemVT = VT.getVectorElementType();
  unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
 
  EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
  MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
  MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
  assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
  unsigned NumOpElts = M1VT.getVectorMinNumElements();
  unsigned NumElts = ContainerVT.getVectorMinNumElements();
  unsigned NumOfSrcRegs = NumElts / NumOpElts;
  unsigned NumOfDestRegs = NumElts / NumOpElts;
  // The following semantically builds up a fixed length concat_vector
  // of the component shuffle_vectors.  We eagerly lower to scalable here
  // to avoid DAG combining it back to a large shuffle_vector again.
  V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
  V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
  SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
      Operands;
  processShuffleMasks(
      Mask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs,
      [&]() { Operands.emplace_back(); },
      [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
        Operands.emplace_back().emplace_back(SrcVecIdx, UINT_MAX,
                                             SmallVector<int>(SrcSubMask));
      },
      [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
        if (NewReg)
          Operands.emplace_back();
        Operands.back().emplace_back(Idx1, Idx2, SmallVector<int>(SrcSubMask));
      });
  assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
  // Note: check that we do not emit too many shuffles here to prevent code
  // size explosion.
  // TODO: investigate, if it can be improved by extra analysis of the masks to
  // check if the code is more profitable.
  unsigned NumShuffles = std::accumulate(
      Operands.begin(), Operands.end(), 0u,
      [&](unsigned N,
          ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
        if (Data.empty())
          return N;
        N += Data.size();
        for (const auto &P : Data) {
          unsigned Idx2 = std::get<1>(P);
          ArrayRef<int> Mask = std::get<2>(P);
          if (Idx2 != UINT_MAX)
            ++N;
          else if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
            --N;
        }
        return N;
      });
  if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
      (NumOfDestRegs <= 2 && NumShuffles >= 4))
    return SDValue();
  auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
    SDValue SubVec = DAG.getExtractSubvector(DL, M1VT, SrcVec, ExtractIdx);
    SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget);
    return SubVec;
  };
  auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
                                        ArrayRef<int> Mask) {
    SDValue SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec1, SubVec2, Mask);
    return SubVec;
  };
  SDValue Vec = DAG.getUNDEF(ContainerVT);
  for (auto [I, Data] : enumerate(Operands)) {
    if (Data.empty())
      continue;
    SmallDenseMap<unsigned, SDValue, 4> Values;
    for (unsigned I : seq<unsigned>(Data.size())) {
      const auto &[Idx1, Idx2, _] = Data[I];
      // If the shuffle contains permutation of odd number of elements,
      // Idx1 might be used already in the first iteration.
      //
      // Idx1 = shuffle Idx1, Idx2
      // Idx1 = shuffle Idx1, Idx3
      SDValue &V = Values.try_emplace(Idx1).first->getSecond();
      if (!V)
        V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
                         (Idx1 % NumOfSrcRegs) * NumOpElts);
      if (Idx2 != UINT_MAX) {
        SDValue &V = Values.try_emplace(Idx2).first->getSecond();
        if (!V)
          V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
                           (Idx2 % NumOfSrcRegs) * NumOpElts);
      }
    }
    SDValue V;
    for (const auto &[Idx1, Idx2, Mask] : Data) {
      SDValue V1 = Values.at(Idx1);
      SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Idx2);
      V = PerformShuffle(V1, V2, Mask);
      Values[Idx1] = V;
    }
 
    unsigned InsertIdx = I * NumOpElts;
    V = convertToScalableVector(M1VT, V, DAG, Subtarget);
    Vec = DAG.getInsertSubvector(DL, Vec, V, InsertIdx);
  }
  return convertFromScalableVector(VT, Vec, DAG, Subtarget);
}
 
// Matches a subset of compress masks with a contiguous prefix of output
// elements.  This could be extended to allow gaps by deciding which
// source elements to spuriously demand.
static bool isCompressMask(ArrayRef<int> Mask) {
  int Last = -1;
  bool SawUndef = false;
  for (const auto &[Idx, M] : enumerate(Mask)) {
    if (M == -1) {
      SawUndef = true;
      continue;
    }
    if (SawUndef)
      return false;
    if (Idx > (unsigned)M)
      return false;
    if (M <= Last)
      return false;
    Last = M;
  }
  return true;
}
 
/// Given a shuffle where the indices are disjoint between the two sources,
/// e.g.:
///
/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
///
/// Merge the two sources into one and do a single source shuffle:
///
/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
///
/// A vselect will either be merged into a masked instruction or be lowered as a
/// vmerge.vvm, which is cheaper than a vrgather.vv.
static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
                                           SelectionDAG &DAG,
                                           const RISCVSubtarget &Subtarget) {
  MVT VT = SVN->getSimpleValueType(0);
  MVT XLenVT = Subtarget.getXLenVT();
  SDLoc DL(SVN);
 
  const ArrayRef<int> Mask = SVN->getMask();
 
  // Work out which source each lane will come from.
  SmallVector<int, 16> Srcs(Mask.size(), -1);
 
  for (int Idx : Mask) {
    if (Idx == -1)
      continue;
    unsigned SrcIdx = Idx % Mask.size();
    int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
    if (Srcs[SrcIdx] == -1)
      // Mark this source as using this lane.
      Srcs[SrcIdx] = Src;
    else if (Srcs[SrcIdx] != Src)
      // The other source is using this lane: not disjoint.
      return SDValue();
  }
 
  SmallVector<SDValue> SelectMaskVals;
  for (int Lane : Srcs) {
    if (Lane == -1)
      SelectMaskVals.push_back(DAG.getUNDEF(XLenVT));
    else
      SelectMaskVals.push_back(DAG.getConstant(Lane ? 0 : 1, DL, XLenVT));
  }
  MVT MaskVT = VT.changeVectorElementType(MVT::i1);
  SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, SelectMaskVals);
  SDValue Select = DAG.getNode(ISD::VSELECT, DL, VT, SelectMask,
                               SVN->getOperand(0), SVN->getOperand(1));
 
  // Move all indices relative to the first source.
  SmallVector<int> NewMask(Mask.size());
  for (unsigned I = 0; I < Mask.size(); I++) {
    if (Mask[I] == -1)
      NewMask[I] = -1;
    else
      NewMask[I] = Mask[I] % Mask.size();
  }
 
  return DAG.getVectorShuffle(VT, DL, Select, DAG.getUNDEF(VT), NewMask);
}
 
/// Is this mask local (i.e. elements only move within their local span), and
/// repeating (that is, the same rearrangement is being done within each span)?
static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
  // Require a prefix from the original mask until the consumer code
  // is adjusted to rewrite the mask instead of just taking a prefix.
  for (auto [I, M] : enumerate(Mask)) {
    if (M == -1)
      continue;
    if ((M / Span) != (int)(I / Span))
      return false;
    int SpanIdx = I % Span;
    int Expected = M % Span;
    if (Mask[SpanIdx] != Expected)
      return false;
  }
  return true;
}
 
/// Is this mask only using elements from the first span of the input?
static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
  return all_of(Mask, [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
}
 
/// Return true for a mask which performs an arbitrary shuffle within the first
/// span, and then repeats that same result across all remaining spans.  Note
/// that this doesn't check if all the inputs come from a single span!
static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
  // Require a prefix from the original mask until the consumer code
  // is adjusted to rewrite the mask instead of just taking a prefix.
  for (auto [I, M] : enumerate(Mask)) {
    if (M == -1)
      continue;
    int SpanIdx = I % Span;
    if (Mask[SpanIdx] != M)
      return false;
  }
  return true;
}
 
/// Try to widen element type to get a new mask value for a better permutation
/// sequence.  This doesn't try to inspect the widened mask for profitability;
/// we speculate the widened form is equal or better.  This has the effect of
/// reducing mask constant sizes - allowing cheaper materialization sequences
/// - and index sequence sizes - reducing register pressure and materialization
/// cost, at the cost of (possibly) an extra VTYPE toggle.
static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  MVT ScalarVT = VT.getVectorElementType();
  unsigned ElementSize = ScalarVT.getFixedSizeInBits();
  SDValue V0 = Op.getOperand(0);
  SDValue V1 = Op.getOperand(1);
  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
 
  // Avoid wasted work leading to isTypeLegal check failing below
  if (ElementSize > 32)
    return SDValue();
 
  SmallVector<int, 8> NewMask;
  if (!widenShuffleMaskElts(Mask, NewMask))
    return SDValue();
 
  MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(ElementSize * 2)
                                      : MVT::getIntegerVT(ElementSize * 2);
  MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
  if (!DAG.getTargetLoweringInfo().isTypeLegal(NewVT))
    return SDValue();
  V0 = DAG.getBitcast(NewVT, V0);
  V1 = DAG.getBitcast(NewVT, V1);
  return DAG.getBitcast(VT, DAG.getVectorShuffle(NewVT, DL, V0, V1, NewMask));
}
 
// Match an interleave shuffle that forms a P-extension packed zip:
//   <a0, b0, a1, b1, ...> -> zip*p/wzip*p
static SDValue lowerVECTOR_SHUFFLEAsPZip(ShuffleVectorSDNode *SVN,
                                         const RISCVSubtarget &Subtarget,
                                         SelectionDAG &DAG) {
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  MVT VT = SVN->getSimpleValueType(0);
  unsigned NumElts = VT.getVectorNumElements();
  ArrayRef<int> Mask = SVN->getMask();
 
  if (VT != MVT::v8i8 && VT != MVT::v4i16)
    return SDValue();
 
  SmallVector<unsigned, 2> StartIndexes;
  if (!V2.isUndef() &&
      ShuffleVectorInst::isInterleaveMask(Mask, 2, NumElts * 2, StartIndexes)) {
    unsigned EvenSrc = StartIndexes[0];
    unsigned OddSrc = StartIndexes[1];
    if (EvenSrc == 0 && OddSrc == NumElts) {
      if (Subtarget.is64Bit())
        return DAG.getNode(RISCVISD::PZIP, DL, VT, V1, V2);
      EVT HalfVT = VT.getHalfNumVectorElementsVT();
      V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
      V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
      return DAG.getNode(RISCVISD::PWZIP, DL, VT, V1, V2);
    }
    if (EvenSrc == NumElts && OddSrc == 0) {
      if (Subtarget.is64Bit())
        return DAG.getNode(RISCVISD::PZIP, DL, VT, V2, V1);
      EVT HalfVT = VT.getHalfNumVectorElementsVT();
      V1 = DAG.getExtractSubvector(DL, HalfVT, V1, 0);
      V2 = DAG.getExtractSubvector(DL, HalfVT, V2, 0);
      return DAG.getNode(RISCVISD::PWZIP, DL, VT, V2, V1);
    }
  }
 
  return SDValue();
}
 
// Match a deinterleave shuffle that forms a P-extension packed unzip:
//   <a0, a2, ..., b0, b2, ...> -> unzip*p
//   <a1, a3, ..., b1, b3, ...> -> unzip*hp
static SDValue lowerVECTOR_SHUFFLEAsPUnzip(ShuffleVectorSDNode *SVN,
                                           SelectionDAG &DAG, bool IsRV64) {
  MVT VT = SVN->getSimpleValueType(0);
  if (!IsRV64 || (VT != MVT::v8i8 && VT != MVT::v4i16))
    return SDValue();
 
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  ArrayRef<int> Mask = SVN->getMask();
 
  unsigned Index = 0;
  if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index))
    return SDValue();
 
  unsigned Opc = Index == 0 ? RISCVISD::PUNZIPE : RISCVISD::PUNZIPO;
  return DAG.getNode(Opc, DL, VT, V1, V2);
}
 
// Match the packed zero-extend shuffle mask <0, N, 2, N+2, ...>: even result
// lanes keep operand 0's even lanes and odd result lanes come from operand 1.
// The odd lanes may select any element of operand 1, which is looser than a
// strict pair-even mask; DAGCombiner::XformToShuffleWithZero forms exactly this
// from a packed zero-extend `and`, leaving each zeroed lane at its own
// position, and lowerVECTOR_SHUFFLEAsPPair then forms the PPAIRE. Undef lanes
// always match.
static bool isPackedZExtShuffleMask(ArrayRef<int> Mask) {
  unsigned NumElts = Mask.size();
  if (NumElts % 2 != 0)
    return false;
  for (unsigned I = 0; I != NumElts / 2; ++I)
    if ((Mask[2 * I] >= 0 && Mask[2 * I] != (int)(2 * I)) ||
        (Mask[2 * I + 1] >= 0 && Mask[2 * I + 1] < (int)NumElts))
      return false;
  return true;
}
 
// Match a legalized deinterleave shuffle on two RV32 vector halves and lower
// it to an RV32 P narrowing shift on the concatenated source.
static SDValue
lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(ShuffleVectorSDNode *SVN,
                                         const RISCVSubtarget &Subtarget,
                                         SelectionDAG &DAG) {
  MVT VT = SVN->getSimpleValueType(0);
  if (Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
    return SDValue();
 
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  unsigned NumElts = VT.getVectorNumElements();
 
  SDValue Src = foldConcatVector(V1, V2);
  if (!Src) {
    MVT SrcVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
    Src = DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, V1, V2);
  }
 
  // The source vector should be twice the size.
  if (Src.getValueType().getVectorNumElements() != 2 * NumElts)
    return SDValue();
 
  unsigned Index = 0;
  if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(SVN->getMask(), 2, Index))
    return SDValue();
 
  unsigned EltBits = VT.getVectorElementType().getSizeInBits();
  return DAG.getNode(RISCVISD::PNSRL, DL, VT, Src,
                     DAG.getConstant(Index * EltBits, DL, MVT::i32));
}
 
// Match a strided-interleave shuffle that forms a P-extension packed pair:
//   <a0, b0, a2, b2, ...> -> ppaire.*
//   <a0, b1, a2, b3, ...> -> ppaireo.*
//   <a1, b0, a3, b2, ...> -> ppairoe.*
//   <a1, b1, a3, b3, ...> -> ppairo.*
static SDValue lowerVECTOR_SHUFFLEAsPPair(ShuffleVectorSDNode *SVN,
                                          SelectionDAG &DAG) {
  MVT VT = SVN->getSimpleValueType(0);
  if (VT != MVT::v4i8 && VT != MVT::v8i8 && VT != MVT::v4i16)
    return SDValue();
 
  SDValue V1 = SVN->getOperand(0);
  SDValue V2 = SVN->getOperand(1);
  SDLoc DL(SVN);
  unsigned NumElts = VT.getVectorNumElements();
  ArrayRef<int> Mask = SVN->getMask();
 
  // A splat operand's lanes are all equal, so a lane selecting from it matches
  // any of its positions. This covers the zero operand XformToShuffleWithZero
  // forms for a packed zero-extend, which keeps each zeroed lane at its own
  // position rather than the strided one.
  bool V1IsSplat = DAG.isSplatValue(V1);
  bool V2IsSplat = DAG.isSplatValue(V2);
 
  // Walk the mask once, tracking the operand feeding the destination's even
  // lanes (index 0) and the operand feeding its odd lanes (index 1) — either
  // may turn out to be V1 or V2 — along with whether each pulls the even or
  // odd element out of its pair. All even (resp. odd) lanes must agree on
  // both the operand and the parity used; a splat operand's lanes are all
  // equal so it never constrains the parity.
  SDValue Src[2];
  std::optional<bool> Parity[2];
  for (unsigned I = 0; I != NumElts; ++I) {
    int M = Mask[I];
    if (M < 0)
      continue;
    unsigned Lane = I % 2;
    bool FromV1 = (unsigned)M < NumElts;
    SDValue Cand = FromV1 ? V1 : V2;
    unsigned Local = (unsigned)M % NumElts;
    if (!Src[Lane])
      Src[Lane] = Cand;
    else if (Src[Lane] != Cand)
      return SDValue();
 
    // Splats don't constrain parity.
    if (FromV1 ? V1IsSplat : V2IsSplat)
      continue;
 
    // The index must be from the even/odd element of its pair.
    if (Local / 2 != I / 2)
      return SDValue();
 
    bool P = Local % 2;
    if (!Parity[Lane])
      Parity[Lane] = P;
    else if (*Parity[Lane] != P)
      return SDValue();
  }
 
  // Make sure we have a source for both lanes.
  if (!Src[0] || !Src[1])
    return SDValue();
 
  bool EvenIsOdd = Parity[0].value_or(false);
  bool OddIsOdd = Parity[1].value_or(false);
  unsigned Opc;
  if (!EvenIsOdd && !OddIsOdd)
    Opc = RISCVISD::PPAIRE;
  else if (EvenIsOdd && OddIsOdd)
    Opc = RISCVISD::PPAIRO;
  else if (!EvenIsOdd && OddIsOdd)
    Opc = RISCVISD::PPAIREO;
  else
    Opc = RISCVISD::PPAIROE;
 
  return DAG.getNode(Opc, DL, VT, Src[0], Src[1]);
}
 
SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDValue V1 = Op.getOperand(0);
  SDValue V2 = Op.getOperand(1);
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  unsigned NumElts = VT.getVectorNumElements();
  ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
 
  // Select RVP-specific packed shuffles before falling back to the generic
  // fixed/scalable-vector lowering below.
  if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
    ArrayRef<int> Mask = SVN->getMask();
 
    // Select an element reverse shuffle to VECTOR_REVERSE. The tablegen
    // patterns select rev8/rev16/ppairoe.* from VECTOR_REVERSE.
    // Reverse of the low L lanes, higher lanes poison. L == NumElts is a plain
    // reverse; L == NumElts/2 is a widened RV64 v4i8/v2i16 reverse.
    auto IsLowReverse = [&](unsigned L) {
      return V2.isUndef() &&
             ShuffleVectorInst::isReverseMask(Mask.take_front(L), L) &&
             all_of(Mask.drop_front(L), [](int M) { return M < 0; });
    };
    if (IsLowReverse(NumElts))
      return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
    if (Subtarget.is64Bit() && VT == MVT::v4i16 && IsLowReverse(/*L=*/2))
      return DAG.getNode(RISCVISD::PPAIROE_H, DL, VT, V1, V1);
    // Widened: reversing sends the low-half lanes to the top half, so shift
    // them back down by half the register. Only the 64-bit packed types are
    // legal here, so the register is XLen (i64).
    if (Subtarget.is64Bit() && VT.getSizeInBits() == 64 &&
        IsLowReverse(NumElts / 2)) {
      SDValue Rev = DAG.getBitcast(
          MVT::i64, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1));
      SDValue Srl =
          DAG.getNode(ISD::SRL, DL, MVT::i64, Rev,
                      DAG.getConstant(VT.getSizeInBits() / 2, DL, MVT::i64));
      return DAG.getBitcast(VT, Srl);
    }
 
    if (SDValue V = lowerVECTOR_SHUFFLEAsPUnzip(SVN, DAG, Subtarget.is64Bit()))
      return V;
    if (SDValue V = lowerVECTOR_SHUFFLEAsPZip(SVN, Subtarget, DAG))
      return V;
    if (SDValue V =
            lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(SVN, Subtarget, DAG))
      return V;
    if (SDValue V = lowerVECTOR_SHUFFLEAsPPair(SVN, DAG))
      return V;
    return SDValue();
  }
 
  if (VT.getVectorElementType() == MVT::i1) {
    // Lower to a vror.vi of a larger element type if possible before we promote
    // i1s to i8s.
    if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
      return V;
    if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
      return V;
 
    // Promote i1 shuffle to i8 shuffle.
    MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount());
    V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
    V2 = V2.isUndef() ? DAG.getUNDEF(WidenVT)
                      : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
    SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
    return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
                        ISD::SETNE);
  }
 
  MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
  // Store the return value in a single variable instead of structured bindings
  // so that we can pass it to GetSlide below, which cannot capture structured
  // bindings until C++20.
  auto TrueMaskVL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
  auto [TrueMask, VL] = TrueMaskVL;
 
  if (SVN->isSplat()) {
    const int Lane = SVN->getSplatIndex();
    if (Lane >= 0) {
      MVT SVT = VT.getVectorElementType();
 
      // Turn splatted vector load into a strided load with an X0 stride.
      SDValue V = V1;
      // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
      // with undef.
      // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
      int Offset = Lane;
      if (V.getOpcode() == ISD::CONCAT_VECTORS) {
        int OpElements =
            V.getOperand(0).getSimpleValueType().getVectorNumElements();
        V = V.getOperand(Offset / OpElements);
        Offset %= OpElements;
      }
 
      // We need to ensure the load isn't atomic or volatile.
      if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
        auto *Ld = cast<LoadSDNode>(V);
        Offset *= SVT.getStoreSize();
        SDValue NewAddr = DAG.getMemBasePlusOffset(
            Ld->getBasePtr(), TypeSize::getFixed(Offset), DL);
 
        // If this is SEW=64 on RV32, use a strided load with a stride of x0.
        if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
          SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
          SDValue IntID =
              DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
          SDValue Ops[] = {Ld->getChain(),
                           IntID,
                           DAG.getUNDEF(ContainerVT),
                           NewAddr,
                           DAG.getRegister(RISCV::X0, XLenVT),
                           VL};
          SDValue NewLoad = DAG.getMemIntrinsicNode(
              ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
              DAG.getMachineFunction().getMachineMemOperand(
                  Ld->getMemOperand(), Offset, SVT.getStoreSize()));
          DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
          return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
        }
 
        MVT SplatVT = ContainerVT;
 
        // f16 with zvfhmin and bf16 need to use an integer scalar load.
        if (SVT == MVT::bf16 ||
            (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
          SVT = MVT::i16;
          SplatVT = ContainerVT.changeVectorElementType(SVT);
        }
 
        // Otherwise use a scalar load and splat. This will give the best
        // opportunity to fold a splat into the operation. ISel can turn it into
        // the x0 strided load if we aren't able to fold away the select.
        if (SVT.isFloatingPoint())
          V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
                          Ld->getPointerInfo().getWithOffset(Offset),
                          Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
        else
          V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
                             Ld->getPointerInfo().getWithOffset(Offset), SVT,
                             Ld->getBaseAlign(),
                             Ld->getMemOperand()->getFlags());
        DAG.makeEquivalentMemoryOrdering(Ld, V);
 
        unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
                                                 : RISCVISD::VMV_V_X_VL;
        SDValue Splat =
            DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
        Splat = DAG.getBitcast(ContainerVT, Splat);
        return convertFromScalableVector(VT, Splat, DAG, Subtarget);
      }
 
      V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
      assert(Lane < (int)NumElts && "Unexpected lane!");
      SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
                                   V1, DAG.getConstant(Lane, DL, XLenVT),
                                   DAG.getUNDEF(ContainerVT), TrueMask, VL);
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
  }
 
  // For exact VLEN m2 or greater, try to split to m1 operations if we
  // can split cleanly.
  if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
    return V;
 
  ArrayRef<int> Mask = SVN->getMask();
 
  if (SDValue V =
          lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
    return V;
 
  if (SDValue V =
          lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
    return V;
 
  // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
  // available.
  if (Subtarget.hasStdExtZvkb())
    if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
      return V;
 
  if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef() &&
      NumElts != 2)
    return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
 
  // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
  // use shift and truncate to perform the shuffle.
  // TODO: For Factor=6, we can perform the first step of the deinterleave via
  // shift-and-trunc reducing total cost for everything except an mf8 result.
  // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
  // to do the entire operation.
  if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
    const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
    assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
    for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
      unsigned Index = 0;
      if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
          1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
        if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
          return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
        if (1 < count_if(Mask,
                         [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
            1 < count_if(Mask, [&Mask](int Idx) {
              return Idx >= (int)Mask.size();
            })) {
          // Narrow each source and concatenate them.
          // FIXME: For small LMUL it is better to concatenate first.
          MVT EltVT = VT.getVectorElementType();
          auto EltCnt = VT.getVectorElementCount();
          MVT SubVT =
              MVT::getVectorVT(EltVT, EltCnt.divideCoefficientBy(Factor));
 
          SDValue Lo =
              getDeinterleaveShiftAndTrunc(DL, SubVT, V1, Factor, Index, DAG);
          SDValue Hi =
              getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG);
 
          SDValue Concat =
              DAG.getNode(ISD::CONCAT_VECTORS, DL,
                          SubVT.getDoubleNumVectorElementsVT(), Lo, Hi);
          if (Factor == 2)
            return Concat;
 
          SDValue Vec = DAG.getUNDEF(VT);
          return DAG.getInsertSubvector(DL, Vec, Concat, 0);
        }
      }
    }
  }
 
  // If this is a deinterleave(2), try using vunzip{e,o}. This mostly catches
  // e64 which can't match above.
  unsigned Index = 0;
  if (Subtarget.hasStdExtZvzip() &&
      ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index) &&
      1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
    bool UsesBothSources =
        1 < count_if(Mask,
                     [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
        1 < count_if(Mask,
                     [&Mask](int Idx) { return Idx >= (int)Mask.size(); });
 
    if (isLegalVTForZvzipOperand(VT, Subtarget)) {
      unsigned Opc = Index == 0 ? RISCVISD::VUNZIPE_VL : RISCVISD::VUNZIPO_VL;
      MVT NewVT = VT.getDoubleNumVectorElementsVT();
      if (isTypeLegal(NewVT)) {
        SDValue Op;
        if (V2.isUndef()) {
          Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
        } else if (auto VLEN = Subtarget.getRealVLen();
                   VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0) {
          Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
        } else if (SDValue Src = foldConcatVector(V1, V2)) {
          Op = DAG.getExtractSubvector(DL, NewVT, Src, 0);
        }
        if (Op)
          return lowerZvzipVUNZIP(Opc, Op, DL, DAG, Subtarget);
      }
 
      if (UsesBothSources &&
          V1.getSimpleValueType().getVectorMinNumElements() >= 2 &&
          V2.getSimpleValueType().getVectorMinNumElements() >= 2) {
        SDValue Lo = lowerZvzipVUNZIP(Opc, V1, DL, DAG, Subtarget);
        SDValue Hi = lowerZvzipVUNZIP(Opc, V2, DL, DAG, Subtarget);
        return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
      }
    }
  }
 
  if (SDValue V =
          lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
    return V;
 
  // Detect an interleave shuffle and lower to
  // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
  int EvenSrc, OddSrc;
  if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
      !(NumElts == 2 &&
        ShuffleVectorInst::isSingleSourceMask(Mask, Mask.size()))) {
    // Extract the halves of the vectors.
    MVT HalfVT = VT.getHalfNumVectorElementsVT();
 
    // Recognize if one half is actually undef; the matching above will
    // otherwise reuse the even stream for the undef one.  This improves
    // spread(2) shuffles.
    bool LaneIsUndef[2] = { true, true};
    for (const auto &[Idx, M] : enumerate(Mask))
      LaneIsUndef[Idx % 2] &= (M == -1);
 
    int Size = Mask.size();
    SDValue EvenV, OddV;
    if (LaneIsUndef[0]) {
      EvenV = DAG.getUNDEF(HalfVT);
    } else {
      assert(EvenSrc >= 0 && "Undef source?");
      EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
      EvenV = DAG.getExtractSubvector(DL, HalfVT, EvenV, EvenSrc % Size);
    }
 
    if (LaneIsUndef[1]) {
      OddV = DAG.getUNDEF(HalfVT);
    } else {
      assert(OddSrc >= 0 && "Undef source?");
      OddV = (OddSrc / Size) == 0 ? V1 : V2;
      OddV = DAG.getExtractSubvector(DL, HalfVT, OddV, OddSrc % Size);
    }
 
    // Prefer vzip if available.
    // TODO: Extend to matching vzip if EvenSrc and OddSrc allow.
    if (Subtarget.hasStdExtZvzip() && isLegalVTForZvzipOperand(VT, Subtarget))
      return lowerZvzipVZIP(EvenV, OddV, DL, DAG, Subtarget);
    return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
  }
 
  // Recognize a pattern which can handled via a pair of vslideup/vslidedown
  // instructions (in any combination) with masking on the second instruction.
  // Also handles masked slides into an identity source, and single slides
  // without masking.  Avoid matching bit rotates (which are not also element
  // rotates) as slide pairs.  This is a performance heuristic, not a
  // functional check.
  std::array<std::pair<int, int>, 2> SrcInfo;
  unsigned RotateAmt;
  MVT RotateVT;
  if (::isMaskedSlidePair(Mask, SrcInfo) &&
      (isElementRotate(SrcInfo, NumElts) ||
       !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
    SDValue Sources[2];
    auto GetSourceFor = [&](const std::pair<int, int> &Info) {
      int SrcIdx = Info.first;
      assert(SrcIdx == 0 || SrcIdx == 1);
      SDValue &Src = Sources[SrcIdx];
      if (!Src) {
        SDValue SrcV = SrcIdx == 0 ? V1 : V2;
        Src = convertToScalableVector(ContainerVT, SrcV, DAG, Subtarget);
      }
      return Src;
    };
    auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
                        SDValue Passthru) {
      auto [TrueMask, VL] = TrueMaskVL;
      SDValue SrcV = GetSourceFor(Src);
      int SlideAmt = Src.second;
      if (SlideAmt == 0) {
        // Should never be second operation
        assert(Mask == TrueMask);
        return SrcV;
      }
      if (SlideAmt < 0)
        return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
                             DAG.getConstant(-SlideAmt, DL, XLenVT), Mask, VL,
                             RISCVVType::TAIL_AGNOSTIC);
      return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
                         DAG.getConstant(SlideAmt, DL, XLenVT), Mask, VL,
                         RISCVVType::TAIL_AGNOSTIC);
    };
 
    if (SrcInfo[1].first == -1) {
      SDValue Res = DAG.getUNDEF(ContainerVT);
      Res = GetSlide(SrcInfo[0], TrueMask, Res);
      return convertFromScalableVector(VT, Res, DAG, Subtarget);
    }
 
    if (Subtarget.hasStdExtZvzip()) {
      bool TryWiden = false;
      unsigned Factor;
      if (isPairEven(SrcInfo, Mask, Factor)) {
        if (Factor == 1) {
          SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
          SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
          return lowerZvzipVPAIR(RISCVISD::VPAIRE_VL, Src1, Src2, DL, DAG,
                                 Subtarget);
        }
        TryWiden = true;
      }
      if (isPairOdd(SrcInfo, Mask, Factor)) {
        if (Factor == 1) {
          SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
          SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
          return lowerZvzipVPAIR(RISCVISD::VPAIRO_VL, Src1, Src2, DL, DAG,
                                 Subtarget);
        }
        TryWiden = true;
      }
      // If we found a widening oppurtunity which would let us form a
      // pair-even or pair-odd, use the generic code to widen the shuffle
      // and recurse through this logic.
      if (TryWiden)
        if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
          return V;
    }
 
    // Build the mask.  Note that vslideup unconditionally preserves elements
    // below the slide amount in the destination, and thus those elements are
    // undefined in the mask.  If the mask ends up all true (or undef), it
    // will be folded away by general logic.
    SmallVector<SDValue> MaskVals;
    for (const auto &[Idx, M] : enumerate(Mask)) {
      if (M < 0 ||
          (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
        MaskVals.push_back(DAG.getUNDEF(XLenVT));
        continue;
      }
      int Src = M >= (int)NumElts;
      int Diff = (int)Idx - (M % NumElts);
      bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
      assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
             "Must match exactly one of the two slides");
      MaskVals.push_back(DAG.getConstant(C, DL, XLenVT));
    }
    assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
    MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
    SDValue SelectMask = convertToScalableVector(
        ContainerVT.changeVectorElementType(MVT::i1),
        DAG.getBuildVector(MaskVT, DL, MaskVals), DAG, Subtarget);
 
    SDValue Res = DAG.getUNDEF(ContainerVT);
    Res = GetSlide(SrcInfo[0], TrueMask, Res);
    Res = GetSlide(SrcInfo[1], SelectMask, Res);
    return convertFromScalableVector(VT, Res, DAG, Subtarget);
  }
 
  // Handle any remaining single source shuffles
  assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
  if (V2.isUndef()) {
    // We might be able to express the shuffle as a bitrotate. But even if we
    // don't have Zvkb and have to expand, the expanded sequence of approx. 2
    // shifts and a vor will have a higher throughput than a vrgather.
    if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
      return V;
 
    if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
      return V;
 
    // Match a spread(4,8) which can be done via extend and shift.  Spread(2)
    // is fully covered in interleave(2) above, so it is ignored here.
    if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
      unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
      assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
      for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
        unsigned Index;
        if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
          MVT NarrowVT =
              MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
          SDValue Src = DAG.getExtractSubvector(DL, NarrowVT, V1, 0);
          return getWideningSpread(Src, Factor, Index, DL, DAG);
        }
      }
    }
 
    // If only a prefix of the source elements influence a prefix of the
    // destination elements, try to see if we can reduce the required LMUL
    unsigned MinVLen = Subtarget.getRealMinVLen();
    unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
    if (NumElts > MinVLMAX) {
      unsigned MaxIdx = 0;
      for (auto [I, M] : enumerate(Mask)) {
        if (M == -1)
          continue;
        MaxIdx = std::max({(unsigned)I, (unsigned)M, MaxIdx});
      }
      unsigned NewNumElts =
          std::max((uint64_t)MinVLMAX, PowerOf2Ceil(MaxIdx + 1));
      if (NewNumElts != NumElts) {
        MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
        V1 = DAG.getExtractSubvector(DL, NewVT, V1, 0);
        SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
                                           Mask.take_front(NewNumElts));
        return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
      }
    }
 
    // Before hitting generic lowering fallbacks, try to widen the mask
    // to a wider SEW.
    if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
      return V;
 
    // Can we generate a vcompress instead of a vrgather?  These scale better
    // at high LMUL, at the cost of not being able to fold a following select
    // into them.  The mask constants are also smaller than the index vector
    // constants, and thus easier to materialize.
    if (isCompressMask(Mask)) {
      SmallVector<SDValue> MaskVals(NumElts,
                                    DAG.getConstant(false, DL, XLenVT));
      for (auto Idx : Mask) {
        if (Idx == -1)
          break;
        assert(Idx >= 0 && (unsigned)Idx < NumElts);
        MaskVals[Idx] = DAG.getConstant(true, DL, XLenVT);
      }
      MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
      SDValue CompressMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
      return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
                         DAG.getUNDEF(VT));
    }
 
    if (VT.getScalarSizeInBits() == 8 &&
        any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
      // On such a vector we're unable to use i8 as the index type.
      // FIXME: We could promote the index to i16 and use vrgatherei16, but that
      // may involve vector splitting if we're already at LMUL=8, or our
      // user-supplied maximum fixed-length LMUL.
      return SDValue();
    }
 
    // Base case for the two operand recursion below - handle the worst case
    // single source shuffle.
    unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
    MVT IndexVT = VT.changeTypeToInteger();
    // Since we can't introduce illegal index types at this stage, use i16 and
    // vrgatherei16 if the corresponding index type for plain vrgather is greater
    // than XLenVT.
    if (IndexVT.getScalarType().bitsGT(XLenVT)) {
      GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
      IndexVT = IndexVT.changeVectorElementType(MVT::i16);
    }
 
    // If the mask allows, we can do all the index computation in 16 bits.  This
    // requires less work and less register pressure at high LMUL, and creates
    // smaller constants which may be cheaper to materialize.
    if (IndexVT.getScalarType().bitsGT(MVT::i16) && isUInt<16>(NumElts - 1) &&
        (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
      GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
      IndexVT = IndexVT.changeVectorElementType(MVT::i16);
    }
 
    MVT IndexContainerVT =
      ContainerVT.changeVectorElementType(IndexVT.getScalarType());
 
    V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
    SmallVector<SDValue> GatherIndicesLHS;
    for (int MaskIndex : Mask) {
      bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
      GatherIndicesLHS.push_back(IsLHSIndex
                                 ? DAG.getConstant(MaskIndex, DL, XLenVT)
                                 : DAG.getUNDEF(XLenVT));
    }
    SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
    LHSIndices =
        convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
    // At m1 and less, there's no point trying any of the high LMUL splitting
    // techniques.  TODO: Should we reconsider this for DLEN < VLEN?
    if (NumElts <= MinVLMAX) {
      SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
                                   DAG.getUNDEF(ContainerVT), TrueMask, VL);
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    const MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
    EVT SubIndexVT = M1VT.changeVectorElementType(IndexVT.getScalarType());
    auto [InnerTrueMask, InnerVL] =
        getDefaultScalableVLOps(M1VT, DL, DAG, Subtarget);
    int N =
        ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
    assert(isPowerOf2_32(N) && N <= 8);
 
    // If we have a locally repeating mask, then we can reuse the first
    // register in the index register group for all registers within the
    // source register group.  TODO: This generalizes to m2, and m4.
    if (isLocalRepeatingShuffle(Mask, MinVLMAX)) {
      SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
      SDValue Gather = DAG.getUNDEF(ContainerVT);
      for (int i = 0; i < N; i++) {
        unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
        SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, SubIdx);
        SDValue SubVec =
            DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
                        DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
        Gather = DAG.getInsertSubvector(DL, Gather, SubVec, SubIdx);
      }
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    // If we have a shuffle which only uses the first register in our source
    // register group, and repeats the same index across all spans, we can
    // use a single vrgather (and possibly some register moves).
    // TODO: This can be generalized for m2 or m4, or for any shuffle for
    // which we can do a linear number of shuffles to form an m1 which
    // contains all the output elements.
    if (isLowSourceShuffle(Mask, MinVLMAX) &&
        isSpanSplatShuffle(Mask, MinVLMAX)) {
      SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
      SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
      SDValue SubVec = DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
                                   DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
      SDValue Gather = DAG.getUNDEF(ContainerVT);
      for (int i = 0; i < N; i++)
        Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
                                        M1VT.getVectorMinNumElements() * i);
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    // If we have a shuffle which only uses the first register in our
    // source register group, we can do a linear number of m1 vrgathers
    // reusing the same source register (but with different indices)
    // TODO: This can be generalized for m2 or m4, or for any shuffle
    // for which we can do a vslidedown followed by this expansion.
    if (isLowSourceShuffle(Mask, MinVLMAX)) {
      SDValue SlideAmt =
          DAG.getElementCount(DL, XLenVT, M1VT.getVectorElementCount());
      SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
      SDValue Gather = DAG.getUNDEF(ContainerVT);
      for (int i = 0; i < N; i++) {
        if (i != 0)
          LHSIndices = getVSlidedown(DAG, Subtarget, DL, IndexContainerVT,
                                     DAG.getUNDEF(IndexContainerVT), LHSIndices,
                                     SlideAmt, TrueMask, VL);
        SDValue SubIndex =
            DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
        SDValue SubVec =
            DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
                        DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
        Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
                                        M1VT.getVectorMinNumElements() * i);
      }
      return convertFromScalableVector(VT, Gather, DAG, Subtarget);
    }
 
    // Fallback to generic vrgather if we can't find anything better.
    // On many machines, this will be O(LMUL^2)
    SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
                                 DAG.getUNDEF(ContainerVT), TrueMask, VL);
    return convertFromScalableVector(VT, Gather, DAG, Subtarget);
  }
 
  // As a backup, shuffles can be lowered via a vrgather instruction, possibly
  // merged with a second vrgather.
  SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
 
  // Now construct the mask that will be used by the blended vrgather operation.
  // Construct the appropriate indices into each vector.
  for (int MaskIndex : Mask) {
    bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
    ShuffleMaskLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
                             ? MaskIndex : -1);
    ShuffleMaskRHS.push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
  }
 
  // If the mask indices are disjoint between the two sources, we can lower it
  // as a vselect + a single source vrgather.vv. Don't do this if we think the
  // operands may end up being lowered to something cheaper than a vrgather.vv.
  if (!DAG.isSplatValue(V2) && !DAG.isSplatValue(V1) &&
      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS) &&
      !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS) &&
      !ShuffleVectorInst::isIdentityMask(ShuffleMaskLHS, NumElts) &&
      !ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts))
    if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
      return V;
 
  // Before hitting generic lowering fallbacks, try to widen the mask
  // to a wider SEW.
  if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
    return V;
 
  // Try to pick a profitable operand order.
  bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
  SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts);
 
  // Recursively invoke lowering for each operand if we had two
  // independent single source shuffles, and then combine the result via a
  // vselect.  Note that the vselect will likely be folded back into the
  // second permute (vrgather, or other) by the post-isel combine.
  V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
  V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
 
  SmallVector<SDValue> MaskVals;
  for (int MaskIndex : Mask) {
    bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
    MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
  }
 
  assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
  SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
 
  if (SwapOps)
    return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
  return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
}
 
bool RISCVTargetLowering::isVectorClearMaskLegal(ArrayRef<int> M,
                                                 EVT VT) const {
  // Enable DAGCombiner::XformToShuffleWithZero to rewrite a packed zero-extend
  // `and` into shuffle(src, zero, ...), which lowerVECTOR_SHUFFLE turns into
  // RISCVISD::PPAIRE. Accept the packed byte/halfword views that lowering
  // handles; the 32-bit views are illegal on RV64 but reachable before type
  // legalization widens them to the legal 64-bit view.
  if (!Subtarget.hasStdExtP() || !VT.isSimple())
    return false;
  MVT SVT = VT.getSimpleVT();
  return (SVT == MVT::v4i8 || SVT == MVT::v8i8 || SVT == MVT::v4i16) &&
         isPackedZExtShuffleMask(M);
}
 
bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
  // Only support legal VTs for other shuffles for now.
  if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
    return false;
 
  // Support splats for any type. These should type legalize well.
  if (ShuffleVectorSDNode::isSplatMask(M))
    return true;
 
  const unsigned NumElts = M.size();
  MVT SVT = VT.getSimpleVT();
 
  // Not for i1 vectors.
  if (SVT.getScalarType() == MVT::i1)
    return false;
 
  std::array<std::pair<int, int>, 2> SrcInfo;
  int Dummy1, Dummy2;
  return ShuffleVectorInst::isReverseMask(M, NumElts) ||
         (::isMaskedSlidePair(M, SrcInfo) &&
          isElementRotate(SrcInfo, NumElts)) ||
         isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
}
 
// Lower CTLZ_ZERO_POISON or CTTZ_ZERO_POISON by converting to FP and extracting
// the exponent.
SDValue
RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_POISON(SDValue Op,
                                                SelectionDAG &DAG) const {
  MVT VT = Op.getSimpleValueType();
  unsigned EltSize = VT.getScalarSizeInBits();
  SDValue Src = Op.getOperand(0);
  SDLoc DL(Op);
  MVT ContainerVT = VT;
 
  // We choose FP type that can represent the value if possible. Otherwise, we
  // use rounding to zero conversion for correct exponent of the result.
  // TODO: Use f16 for i8 when possible?
  MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
  if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount())))
    FloatEltVT = MVT::f32;
  MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
 
  // Legal types should have been checked in the RISCVTargetLowering
  // constructor.
  // TODO: Splitting may make sense in some cases.
  assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
         "Expected legal float type!");
 
  // For CTTZ_ZERO_POISON, we need to extract the lowest set bit using X & -X.
  // The trailing zero count is equal to log2 of this single bit value.
  if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON) {
    SDValue Neg = DAG.getNegative(Src, DL, VT);
    Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
  }
 
  // We have a legal FP type, convert to it.
  SDValue FloatVal;
  if (FloatVT.bitsGT(VT)) {
    FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
  } else {
    // Use RTZ to avoid rounding influencing exponent of FloatVal.
    if (VT.isFixedLengthVector()) {
      ContainerVT = getContainerForFixedLengthVector(VT);
      Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
    }
    auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
    SDValue RTZRM =
        DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
    MVT ContainerFloatVT =
        MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount());
    FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
                           Src, Mask, RTZRM, VL);
    if (VT.isFixedLengthVector())
      FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget);
  }
  // Bitcast to integer and shift the exponent to the LSB.
  EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
  SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
  unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
 
  // Restore back to original type. Truncation after SRL is to generate vnsrl.
  SDValue Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
                            DAG.getConstant(ShiftAmt, DL, IntVT));
  if (IntVT.bitsLT(VT))
    Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
  else if (IntVT.bitsGT(VT))
    Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
 
  // The exponent contains log2 of the value in biased form.
  unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
  // For trailing zeros, we just need to subtract the bias.
  if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON)
    return DAG.getNode(ISD::SUB, DL, VT, Exp,
                       DAG.getConstant(ExponentBias, DL, VT));
 
  // For leading zeros, we need to remove the bias and convert from log2 to
  // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
  unsigned Adjust = ExponentBias + (EltSize - 1);
  SDValue Res =
      DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
 
  // The above result with zero input equals to Adjust which is greater than
  // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
  if (Op.getOpcode() == ISD::CTLZ)
    Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
 
  return Res;
}
 
SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
                                                 SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT XLenVT = Subtarget.getXLenVT();
  SDValue Source = Op->getOperand(0);
  MVT SrcVT = Source.getSimpleValueType();
  SDValue Mask = Op->getOperand(1);
  SDValue EVL = Op->getOperand(2);
 
  if (SrcVT.isFixedLengthVector()) {
    MVT ContainerVT = getContainerForFixedLengthVector(SrcVT);
    Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget);
    Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
                                   Subtarget);
    SrcVT = ContainerVT;
  }
 
  // Convert to boolean vector.
  if (SrcVT.getScalarType() != MVT::i1) {
    SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
    SrcVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorElementCount());
    Source = DAG.getNode(RISCVISD::SETCC_VL, DL, SrcVT,
                         {Source, AllZero, DAG.getCondCode(ISD::SETNE),
                          DAG.getUNDEF(SrcVT), Mask, EVL});
  }
 
  SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Source, Mask, EVL);
  if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_POISON)
    // In this case, we can interpret poison as -1, so nothing to do further.
    return Res;
 
  // Convert -1 to VL.
  SDValue SetCC =
      DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
  Res = DAG.getSelect(DL, XLenVT, SetCC, EVL, Res);
  return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
}
 
// While RVV has alignment restrictions, we should always be able to load as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
// the load is already correctly-aligned, it returns SDValue().
SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
                                                    SelectionDAG &DAG) const {
  auto *Load = cast<LoadSDNode>(Op);
  assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Load->getMemoryVT(),
                                     *Load->getMemOperand()))
    return SDValue();
 
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV load type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
  SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
                          Load->getPointerInfo(), Load->getBaseAlign(),
                          Load->getMemOperand()->getFlags());
  return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
}
 
// While RVV has alignment restrictions, we should always be able to store as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
// returns SDValue() if the store is already correctly aligned.
SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
                                                     SelectionDAG &DAG) const {
  auto *Store = cast<StoreSDNode>(Op);
  assert(Store && Store->getValue().getValueType().isVector() &&
         "Expected vector store");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Store->getMemoryVT(),
                                     *Store->getMemOperand()))
    return SDValue();
 
  SDLoc DL(Op);
  SDValue StoredVal = Store->getValue();
  MVT VT = StoredVal.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV store type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
  StoredVal = DAG.getBitcast(NewVT, StoredVal);
  return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
                      Store->getPointerInfo(), Store->getBaseAlign(),
                      Store->getMemOperand()->getFlags());
}
 
// While RVV has alignment restrictions, we should always be able to load as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
// the load is already correctly-aligned, it returns SDValue().
SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
                                                   SelectionDAG &DAG) const {
  auto *Load = cast<VPLoadSDNode>(Op);
  assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Load->getMemoryVT(),
                                     *Load->getMemOperand()))
    return SDValue();
 
  SDValue Mask = Load->getMask();
 
  // FIXME: Handled masked loads somehow.
  if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
    return SDValue();
 
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV load type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
 
  SDValue VL = Load->getVectorLength();
  VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
                   DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
 
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
  SDValue L = DAG.getLoadVP(NewVT, DL, Load->getChain(), Load->getBasePtr(),
                            DAG.getAllOnesConstant(DL, MaskVT), VL,
                            Load->getPointerInfo(), Load->getBaseAlign(),
                            Load->getMemOperand()->getFlags(), AAMDNodes());
  return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
}
 
// While RVV has alignment restrictions, we should always be able to store as a
// legal equivalently-sized byte-typed vector instead. This method is
// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
// It returns SDValue() if the store is already correctly aligned.
SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
                                                    SelectionDAG &DAG) const {
  auto *Store = cast<VPStoreSDNode>(Op);
  assert(Store && Store->getValue().getValueType().isVector() &&
         "Expected vector store");
 
  if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
                                     Store->getMemoryVT(),
                                     *Store->getMemOperand()))
    return SDValue();
 
  SDValue Mask = Store->getMask();
 
  // FIXME: Handled masked stores somehow.
  if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
    return SDValue();
 
  SDLoc DL(Op);
  SDValue StoredVal = Store->getValue();
  MVT VT = StoredVal.getSimpleValueType();
  unsigned EltSizeBits = VT.getScalarSizeInBits();
  assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
         "Unexpected unaligned RVV store type");
  MVT NewVT =
      MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
  assert(NewVT.isValid() &&
         "Expecting equally-sized RVV vector types to be legal");
 
  SDValue VL = Store->getVectorLength();
  VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
                   DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
 
  StoredVal = DAG.getBitcast(NewVT, StoredVal);
 
  LocationSize Size = LocationSize::precise(NewVT.getStoreSize());
  MachineFunction &MF = DAG.getMachineFunction();
  MachineMemOperand *MMO = MF.getMachineMemOperand(
      Store->getPointerInfo(), Store->getMemOperand()->getFlags(), Size,
      Store->getBaseAlign());
 
  MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
  return DAG.getStoreVP(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
                        DAG.getPOISON(Store->getBasePtr().getValueType()),
                        DAG.getAllOnesConstant(DL, MaskVT), VL, NewVT, MMO,
                        ISD::UNINDEXED);
}
 
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget) {
  assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
 
  int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
 
  // All simm32 constants should be handled by isel.
  // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
  // this check redundant, but small immediates are common so this check
  // should have better compile time.
  if (isInt<32>(Imm))
    return Op;
 
  // We only need to cost the immediate, if constant pool lowering is enabled.
  if (!Subtarget.useConstantPoolForLargeInts())
    return Op;
 
  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, Subtarget);
  if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
    return Op;
 
  // Optimizations below are disabled for opt size. If we're optimizing for
  // size, use a constant pool.
  if (DAG.shouldOptForSize())
    return SDValue();
 
  // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
  // that if it will avoid a constant pool.
  // It will require an extra temporary register though.
  // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
  // low and high 32 bits are the same and bit 31 and 63 are set.
  unsigned ShiftAmt, AddOpc;
  RISCVMatInt::InstSeq SeqLo =
      RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc);
  if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
    return Op;
 
  return SDValue();
}
 
SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
                                             SelectionDAG &DAG) const {
  MVT VT = Op.getSimpleValueType();
  const APFloat &Imm = cast<ConstantFPSDNode>(Op)->getValueAPF();
 
  // Can this constant be selected by a Zfa FLI instruction?
  bool Negate = false;
  int Index = getLegalZfaFPImm(Imm, VT);
 
  // If the constant is negative, try negating.
  if (Index < 0 && Imm.isNegative()) {
    Index = getLegalZfaFPImm(-Imm, VT);
    Negate = true;
  }
 
  // If we couldn't find a FLI lowering, fall back to generic code.
  if (Index < 0)
    return SDValue();
 
  // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
  SDLoc DL(Op);
  SDValue Const =
      DAG.getNode(RISCVISD::FLI, DL, VT,
                  DAG.getTargetConstant(Index, DL, Subtarget.getXLenVT()));
  if (!Negate)
    return Const;
 
  return DAG.getNode(ISD::FNEG, DL, VT, Const);
}
 
static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
                             SelectionDAG &DAG) {
 
  unsigned IsData = Op.getConstantOperandVal(4);
 
  // mips-p8700  we support data prefetch for now.
  if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
    return Op.getOperand(0);
  return Op;
}
 
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
                                 const RISCVSubtarget &Subtarget) {
  SDLoc dl(Op);
  AtomicOrdering FenceOrdering =
      static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
  SyncScope::ID FenceSSID =
      static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
 
  if (Subtarget.hasStdExtZtso()) {
    // The only fence that needs an instruction is a sequentially-consistent
    // cross-thread fence.
    if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
        FenceSSID == SyncScope::System)
      return Op;
 
    // MEMBARRIER is a compiler barrier; it codegens to a no-op.
    return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
  }
 
  // singlethread fences only synchronize with signal handlers on the same
  // thread and thus only need to preserve instruction order, not actually
  // enforce memory ordering.
  if (FenceSSID == SyncScope::SingleThread)
    // MEMBARRIER is a compiler barrier; it codegens to a no-op.
    return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
 
  return Op;
}
 
SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
                                             SelectionDAG &DAG) const {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
  MVT XLenVT = Subtarget.getXLenVT();
  unsigned Check = Op.getConstantOperandVal(1);
  unsigned TDCMask = 0;
  if (Check & fcSNan)
    TDCMask |= RISCV::FPMASK_Signaling_NaN;
  if (Check & fcQNan)
    TDCMask |= RISCV::FPMASK_Quiet_NaN;
  if (Check & fcPosInf)
    TDCMask |= RISCV::FPMASK_Positive_Infinity;
  if (Check & fcNegInf)
    TDCMask |= RISCV::FPMASK_Negative_Infinity;
  if (Check & fcPosNormal)
    TDCMask |= RISCV::FPMASK_Positive_Normal;
  if (Check & fcNegNormal)
    TDCMask |= RISCV::FPMASK_Negative_Normal;
  if (Check & fcPosSubnormal)
    TDCMask |= RISCV::FPMASK_Positive_Subnormal;
  if (Check & fcNegSubnormal)
    TDCMask |= RISCV::FPMASK_Negative_Subnormal;
  if (Check & fcPosZero)
    TDCMask |= RISCV::FPMASK_Positive_Zero;
  if (Check & fcNegZero)
    TDCMask |= RISCV::FPMASK_Negative_Zero;
 
  bool IsOneBitMask = isPowerOf2_32(TDCMask);
 
  SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
 
  if (VT.isVector()) {
    SDValue Op0 = Op.getOperand(0);
    MVT VT0 = Op.getOperand(0).getSimpleValueType();
 
    if (VT.isScalableVector()) {
      MVT DstVT = VT0.changeVectorElementTypeToInteger();
      auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget);
      SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
                                    VL, Op->getFlags());
      if (IsOneBitMask)
        return DAG.getSetCC(DL, VT, FPCLASS,
                            DAG.getConstant(TDCMask, DL, DstVT),
                            ISD::CondCode::SETEQ);
      SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
                                DAG.getConstant(TDCMask, DL, DstVT));
      return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
                          ISD::SETNE);
    }
 
    MVT ContainerVT0 = getContainerForFixedLengthVector(VT0);
    MVT ContainerVT = getContainerForFixedLengthVector(VT);
    MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
    auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget);
    Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget);
 
    SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
                                  Mask, VL, Op->getFlags());
 
    TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
                           DAG.getUNDEF(ContainerDstVT), TDCMaskV, VL);
    if (IsOneBitMask) {
      SDValue VMSEQ =
          DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
                      {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ),
                       DAG.getUNDEF(ContainerVT), Mask, VL});
      return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget);
    }
    SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
                              TDCMaskV, DAG.getUNDEF(ContainerDstVT), Mask, VL);
 
    SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
    SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
                            DAG.getUNDEF(ContainerDstVT), SplatZero, VL);
 
    SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
                                {AND, SplatZero, DAG.getCondCode(ISD::SETNE),
                                 DAG.getUNDEF(ContainerVT), Mask, VL});
    return convertFromScalableVector(VT, VMSNE, DAG, Subtarget);
  }
 
  SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
  SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
  SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT),
                             ISD::CondCode::SETNE);
  return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
}
 
// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
// operations propagate nans.
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
                                      const RISCVSubtarget &Subtarget) {
  SDLoc DL(Op);
  MVT VT = Op.getSimpleValueType();
 
  SDValue X = Op.getOperand(0);
  SDValue Y = Op.getOperand(1);
 
  if (!VT.isVector()) {
    MVT XLenVT = Subtarget.getXLenVT();
 
    // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
    // ensures that when one input is a nan, the other will also be a nan
    // allowing the nan to propagate. If both inputs are nan, this will swap the
    // inputs which is harmless.
 
    SDValue NewY = Y;
    if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(X)) {
      SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
      NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
    }
 
    SDValue NewX = X;
    if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Y)) {
      SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
      NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
    }
 
    unsigned Opc =
        Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
    return DAG.getNode(Opc, DL, VT, NewX, NewY);
  }
 
  // Check no NaNs before converting to fixed vector scalable.
  bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(X);
  bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Y);
 
  MVT ContainerVT = VT;
  if (VT.isFixedLengthVector()) {
    ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
    X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
    Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
  }
 
  auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
 
  SDValue NewY = Y;
  if (!XIsNeverNan) {
    SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
                                    {X, X, DAG.getCondCode(ISD::SETOEQ),
                                     DAG.getUNDEF(ContainerVT), Mask, VL});
    NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
                       DAG.getUNDEF(ContainerVT), VL);
  }
 
  SDValue NewX = X;
  if (!YIsNeverNan) {
    SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
                                    {Y, Y, DAG.getCondCode(ISD::SETOEQ),
                                     DAG.getUNDEF(ContainerVT), Mask, VL});
    NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
                       DAG.getUNDEF(ContainerVT), VL);
  }
 
  unsigned Opc =
      Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
  SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
                            DAG.getUNDEF(ContainerVT), Mask, VL);
  if (VT.isFixedLengthVector())
    Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
  return Res;
}
 
static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
                               const RISCVSubtarget &Subtarget) {
  bool IsFABS = Op.getOpcode() == ISD::FABS;
  assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
         "Wrong opcode for lowering FABS or FNEG.");
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
 
  SDLoc DL(Op);
  SDValue Fmv =
      DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op.getOperand(0));
 
  APInt Mask = IsFABS ? APInt::getSignedMaxValue(16) : APInt::getSignMask(16);
  Mask = Mask.sext(Subtarget.getXLen());
 
  unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
  SDValue Logic =
      DAG.getNode(LogicOpc, DL, XLenVT, Fmv, DAG.getConstant(Mask, DL, XLenVT));
  return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, Logic);
}
 
static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
                              const RISCVSubtarget &Subtarget) {
  assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
 
  MVT XLenVT = Subtarget.getXLenVT();
  MVT VT = Op.getSimpleValueType();
  assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
 
  SDValue Mag = Op.getOperand(0);
  SDValue Sign = Op.getOperand(1);
 
  SDLoc DL(Op);
 
  // Get sign bit into an integer value.
  unsigned SignSize = Sign.getValueSizeInBits();
  SDValue SignAsInt = [&]() {
    if (SignSize == Subtarget.getXLen())
      return DAG.getNode(ISD::BITCAST, DL, XLenVT, Sign);
    switch (SignSize) {
    case 16:
      return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Sign);
    case 32:
      return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, Sign);
    case 64: {
      assert(XLenVT == MVT::i32 && "Unexpected type");
      // Copy the upper word to integer.
      SignSize = 32;
      return DAG.getNode(RISCVISD::SplitF64, DL, {MVT::i32, MVT::i32}, Sign)
          .getValue(1);
    }
    default:
      llvm_unreachable("Unexpected sign size");
    }
  }();
 
  // Get the signbit at the right position for MagAsInt.
  if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
    SignAsInt = DAG.getNode(ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, XLenVT,
                            SignAsInt,
                            DAG.getConstant(std::abs(ShiftAmount), DL, XLenVT));
 
  // Mask the sign bit and any bits above it. The extra bits will be dropped
  // when we convert back to FP.
  SDValue SignMask = DAG.getConstant(
      APInt::getSignMask(16).sext(Subtarget.getXLen()), DL, XLenVT);
  SDValue SignBit = DAG.getNode(ISD::AND, DL, XLenVT, SignAsInt, SignMask);
 
  // Transform Mag value to integer, and clear the sign bit.
  SDValue MagAsInt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Mag);
  SDValue ClearSignMask = DAG.getConstant(
      APInt::getSignedMaxValue(16).sext(Subtarget.getXLen()), DL, XLenVT);
  SDValue ClearedSign =
      DAG.getNode(ISD::AND, DL, XLenVT, MagAsInt, ClearSignMask);
 
  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, XLenVT, ClearedSign, SignBit,
                                   SDNodeFlags::Disjoint);
 
  return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, CopiedSign);
}
 
/// Get a RISC-V target specified VL op for a given SDNode.
static unsigned getRISCVVLOp(SDValue Op) {
#define OP_CASE(NODE)                                                          \
  case ISD::NODE:                                                              \
    return RISCVISD::NODE##_VL;
#define VP_CASE(NODE)                                                          \
  case ISD::VP_##NODE:                                                         \
    return RISCVISD::NODE##_VL;
  // clang-format off
  switch (Op.getOpcode()) {
  default:
    llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
  OP_CASE(ADD)
  OP_CASE(SUB)
  OP_CASE(MUL)
  OP_CASE(MULHS)
  OP_CASE(MULHU)
  OP_CASE(SDIV)
  OP_CASE(SREM)
  OP_CASE(UDIV)
  OP_CASE(UREM)
  OP_CASE(SHL)
  OP_CASE(SRA)
  OP_CASE(SRL)
  OP_CASE(ROTL)
  OP_CASE(ROTR)
  OP_CASE(BSWAP)
  OP_CASE(CTTZ)
  OP_CASE(CTLZ)
  OP_CASE(CTPOP)
  OP_CASE(BITREVERSE)
  OP_CASE(CLMUL)
  OP_CASE(CLMULH)
  OP_CASE(SADDSAT)
  OP_CASE(UADDSAT)
  OP_CASE(SSUBSAT)
  OP_CASE(USUBSAT)
  OP_CASE(AVGFLOORS)
  OP_CASE(AVGFLOORU)
  OP_CASE(AVGCEILS)
  OP_CASE(AVGCEILU)
  OP_CASE(FADD)
  OP_CASE(FSUB)
  OP_CASE(FMUL)
  OP_CASE(FDIV)
  OP_CASE(FNEG)
  OP_CASE(FABS)
  OP_CASE(FCOPYSIGN)
  OP_CASE(FSQRT)
  OP_CASE(SMIN)
  OP_CASE(SMAX)
  OP_CASE(UMIN)
  OP_CASE(UMAX)
  OP_CASE(ABDS)
  OP_CASE(ABDU)
  OP_CASE(STRICT_FADD)
  OP_CASE(STRICT_FSUB)
  OP_CASE(STRICT_FMUL)
  OP_CASE(STRICT_FDIV)
  OP_CASE(STRICT_FSQRT)
  VP_CASE(SDIV)       // VP_SDIV
  VP_CASE(SREM)       // VP_SREM
  VP_CASE(UDIV)       // VP_UDIV
  VP_CASE(UREM)       // VP_UREM
  case ISD::CTLZ_ZERO_POISON:
    return RISCVISD::CTLZ_VL;
  case ISD::CTTZ_ZERO_POISON:
    return RISCVISD::CTTZ_VL;
  case ISD::FMA:
    return RISCVISD::VFMADD_VL;
  case ISD::STRICT_FMA:
    return RISCVISD::STRICT_VFMADD_VL;
  case ISD::AND:
    if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
      return RISCVISD::VMAND_VL;
    return RISCVISD::AND_VL;
  case ISD::OR:
    if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
      return RISCVISD::VMOR_VL;
    return RISCVISD::OR_VL;
  case ISD::XOR:
    if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
      return RISCVISD::VMXOR_VL;
    return RISCVISD::XOR_VL;
  case ISD::ANY_EXTEND:
  case ISD::ZERO_EXTEND:
    return RISCVISD::VZEXT_VL;
  case ISD::SIGN_EXTEND:
    return RISCVISD::VSEXT_VL;
  case ISD::SETCC:
    return RISCVISD::SETCC_VL;
  case ISD::VSELECT:
    return RISCVISD::VMERGE_VL;
  case ISD::VP_MERGE:
    return RISCVISD::VMERGE_VL;
  case ISD::FMINNUM:
  case ISD::FMINIMUMNUM:
    return RISCVISD::VFMIN_VL;
  case ISD::FMAXNUM:
  case ISD::FMAXIMUMNUM:
    return RISCVISD::VFMAX_VL;
  case ISD::LRINT:
  case ISD::LLRINT:
    return RISCVISD::VFCVT_RM_X_F_VL;
  case ISD::MASKED_UDIV:
    return RISCVISD::UDIV_VL;
  case ISD::MASKED_UREM:
    return RISCVISD::UREM_VL;
  case ISD::MASKED_SDIV:
    return RISCVISD::SDIV_VL;
  case ISD::MASKED_SREM:
    return RISCVISD::SREM_VL;
  }
  // clang-format on
#undef OP_CASE
#undef VP_CASE
}
 
static bool isPromotedOpNeedingSplit(SDValue Op,
                                     const RISCVSubtarget &Subtarget,
                                     const TargetLowering &TLI) {
  MVT OpVT = Op.getSimpleValueType();
  if (!OpVT.isVector())
    return false;
  MVT EltVT = OpVT.getVectorElementType();
  if (!(EltVT == MVT::f16 && Subtarget.hasVInstructionsF16Minimal() &&
        !Subtarget.hasVInstructionsF16()) &&
      !(EltVT == MVT::bf16 && Subtarget.hasVInstructionsBF16Minimal() &&
        (!Subtarget.hasVInstructionsBF16() ||
         !llvm::is_contained(ZvfbfaOps, Op.getOpcode()))))
    return false;
  // Need to split when the same width f32 vector type isn't legal.
  return !TLI.isTypeLegal(
      MVT::getVectorVT(MVT::f32, OpVT.getVectorElementCount()));
}
 
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
  auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
  SDLoc DL(Op);
 
  SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
  SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
 
  for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
    if (!Op.getOperand(j).getValueType().isVector()) {
      LoOperands[j] = Op.getOperand(j);
      HiOperands[j] = Op.getOperand(j);
      continue;
    }
    std::tie(LoOperands[j], HiOperands[j]) =
        DAG.SplitVector(Op.getOperand(j), DL);
  }
 
  SDValue LoRes =
      DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
  SDValue HiRes =
      DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
 
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
}
 
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG,
                                      bool IsVP) {
  SDLoc DL(Op);
 
  if (IsVP) {
    auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL);
    auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL);
    auto [EVLLo, EVLHi] =
        DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL);
 
    SDValue ResLo =
        DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                    {Op.getOperand(0), Lo, MaskLo, EVLLo}, Op->getFlags());
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                       {ResLo, Hi, MaskHi, EVLHi}, Op->getFlags());
  }
 
  unsigned Opcode = Op.getOpcode();
  unsigned OpNo = Opcode == ISD::VECREDUCE_SEQ_FADD ? 1 : 0;
 
  auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(OpNo), DL);
  if (Opcode == ISD::VECREDUCE_SEQ_FADD) {
    SDValue ResLo = DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
                                Op.getOperand(0), Lo, Op->getFlags());
    return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), ResLo, Hi,
                       Op->getFlags());
  }
 
  SDValue ResLo =
      DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Lo, Op->getFlags());
  SDValue ResHi =
      DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Hi, Op->getFlags());
  unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
  return DAG.getNode(BaseOpc, DL, Op.getValueType(), ResLo, ResHi,
                     Op->getFlags());
}
 
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
 
  assert(Op->isStrictFPOpcode());
 
  auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op->getValueType(0));
 
  SDVTList LoVTs = DAG.getVTList(LoVT, Op->getValueType(1));
  SDVTList HiVTs = DAG.getVTList(HiVT, Op->getValueType(1));
 
  SDLoc DL(Op);
 
  SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
  SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
 
  for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
    if (!Op.getOperand(j).getValueType().isVector()) {
      LoOperands[j] = Op.getOperand(j);
      HiOperands[j] = Op.getOperand(j);
      continue;
    }
    std::tie(LoOperands[j], HiOperands[j]) =
        DAG.SplitVector(Op.getOperand(j), DL);
  }
 
  SDValue LoRes =
      DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
  HiOperands[0] = LoRes.getValue(1);
  SDValue HiRes =
      DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
 
  SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
                          LoRes.getValue(0), HiRes.getValue(0));
  return DAG.getMergeValues({V, HiRes.getValue(1)}, DL);
}
 
SDValue
RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
                                                   SelectionDAG &DAG) const {
  assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
         "Unexpected bfloat16 load lowering");
 
  SDLoc DL(Op);
  LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
  EVT MemVT = LD->getMemoryVT();
  SDValue Load = DAG.getExtLoad(
      ISD::ZEXTLOAD, DL, Subtarget.getXLenVT(), LD->getChain(),
      LD->getBasePtr(),
      EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
      LD->getMemOperand());
  // Using mask to make bf16 nan-boxing valid when we don't have flh
  // instruction. -65536 would be treat as a small number and thus it can be
  // directly used lui to get the constant.
  SDValue mask = DAG.getSignedConstant(-65536, DL, Subtarget.getXLenVT());
  SDValue OrSixteenOne =
      DAG.getNode(ISD::OR, DL, Load.getValueType(), {Load, mask});
  SDValue ConvertedResult =
      DAG.getNode(RISCVISD::NDS_FMV_BF16_X, DL, MVT::bf16, OrSixteenOne);
  return DAG.getMergeValues({ConvertedResult, Load.getValue(1)}, DL);
}
 
SDValue
RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
                                                    SelectionDAG &DAG) const {
  assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
         "Unexpected bfloat16 store lowering");
 
  StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
  SDLoc DL(Op);
  SDValue FMV = DAG.getNode(RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
                            Subtarget.getXLenVT(), ST->getValue());
  return DAG.getTruncStore(
      ST->getChain(), DL, FMV, ST->getBasePtr(),
      EVT::getIntegerVT(*DAG.getContext(), ST->getMemoryVT().getSizeInBits()),
      ST->getMemOperand());
}
 
static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
                             const RISCVSubtarget &Subtarget);
 
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
                                            SelectionDAG &DAG) const {
  switch (Op.getOpcode()) {
  default:
    reportFatalInternalError(
        "Unimplemented RISCVTargetLowering::LowerOperation Case");
  case ISD::PREFETCH:
    return LowerPREFETCH(Op, Subtarget, DAG);
  case ISD::ATOMIC_FENCE:
    return LowerATOMIC_FENCE(Op, DAG, Subtarget);
  case ISD::GlobalAddress:
    return lowerGlobalAddress(Op, DAG);
  case ISD::BlockAddress:
    return lowerBlockAddress(Op, DAG);
  case ISD::ConstantPool:
    return lowerConstantPool(Op, DAG);
  case ISD::JumpTable:
    return lowerJumpTable(Op, DAG);
  case ISD::GlobalTLSAddress:
    return lowerGlobalTLSAddress(Op, DAG);
  case ISD::Constant:
    return lowerConstant(Op, DAG, Subtarget);
  case ISD::ConstantFP:
    return lowerConstantFP(Op, DAG);
  case ISD::SELECT:
    return lowerSELECT(Op, DAG);
  case ISD::BRCOND:
    return lowerBRCOND(Op, DAG);
  case ISD::VASTART:
    return lowerVASTART(Op, DAG);
  case ISD::FRAMEADDR:
    return lowerFRAMEADDR(Op, DAG);
  case ISD::RETURNADDR:
    return lowerRETURNADDR(Op, DAG);
  case ISD::SHL_PARTS:
    return lowerShiftLeftParts(Op, DAG);
  case ISD::SRA_PARTS:
    return lowerShiftRightParts(Op, DAG, true);
  case ISD::SRL_PARTS:
    return lowerShiftRightParts(Op, DAG, false);
  case ISD::ROTL:
  case ISD::ROTR:
    if (Op.getValueType().isFixedLengthVector()) {
      assert(Subtarget.hasStdExtZvkb());
      return lowerToScalableOp(Op, DAG);
    }
    assert(Subtarget.hasVendorXTHeadBb() &&
           !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
           "Unexpected custom legalization");
    // XTHeadBb only supports rotate by constant.
    if (!isa<ConstantSDNode>(Op.getOperand(1)))
      return