AArch64InstrInfo.cpp

Go to the documentation of this file.

//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the AArch64 implementation of the TargetInstrInfo class.
//
//===----------------------------------------------------------------------===//
 
#include "AArch64InstrInfo.h"
#include "AArch64ExpandImm.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PointerAuth.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "MCTargetDesc/AArch64MCLFIRewriter.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineCombinerPattern.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Module.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/LEB128.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <cassert>
#include <cstdint>
#include <iterator>
#include <utility>
 
using namespace llvm;
 
#define GET_INSTRINFO_CTOR_DTOR
#include "AArch64GenInstrInfo.inc"
 
#define DEBUG_TYPE "AArch64InstrInfo"
 
STATISTIC(NumCopyInstrs, "Number of COPY instructions expanded");
STATISTIC(NumZCRegMoveInstrsGPR, "Number of zero-cycle GPR register move "
                                 "instructions expanded from canonical COPY");
STATISTIC(NumZCRegMoveInstrsFPR, "Number of zero-cycle FPR register move "
                                 "instructions expanded from canonical COPY");
STATISTIC(NumZCZeroingInstrsGPR, "Number of zero-cycle GPR zeroing "
                                 "instructions expanded from canonical COPY");
// NumZCZeroingInstrsFPR is counted at AArch64AsmPrinter
 
static cl::opt<unsigned>
    CBDisplacementBits("aarch64-cb-offset-bits", cl::Hidden, cl::init(9),
                       cl::desc("Restrict range of CB instructions (DEBUG)"));
 
static cl::opt<unsigned> TBZDisplacementBits(
    "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
    cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
 
static cl::opt<unsigned> CBZDisplacementBits(
    "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
    cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
 
static cl::opt<unsigned>
    BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
                        cl::desc("Restrict range of Bcc instructions (DEBUG)"));
 
static cl::opt<unsigned>
    BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26),
                      cl::desc("Restrict range of B instructions (DEBUG)"));
 
static cl::opt<unsigned> GatherOptSearchLimit(
    "aarch64-search-limit", cl::Hidden, cl::init(2048),
    cl::desc("Restrict range of instructions to search for the "
             "machine-combiner gather pattern optimization"));
 
static cl::opt<bool> UseCompactUnwindFrameRecordForOutlinedFunctions(
    "aarch64-outliner-compact-unwind-frame", cl::Hidden, cl::init(true),
    cl::desc("Use a frame record for Mach-O non-leaf outlined functions"));
 
AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
    : AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
                          AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
      RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
 
/// Return the maximum number of bytes of code the specified instruction may be
/// after LFI rewriting. If the instruction is not rewritten, std::nullopt is
/// returned (use default sizing).
///
/// NOTE: the size estimates here must be kept in sync with the rewrites in
/// AArch64MCLFIRewriter.cpp. Sizes may be overestimates of the rewritten
/// instruction sequences.
static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  case AArch64::SVC:
    // SVC expands to 4 instructions.
    return 16;
  case AArch64::BR:
  case AArch64::BLR:
    // Indirect branches/calls expand to 2 instructions (guard + br/blr).
    return 8;
  case AArch64::RET:
    // RET through LR is not rewritten, but RET through another register
    // expands to 2 instructions (guard + ret).
    if (MI.getOperand(0).getReg() != AArch64::LR)
      return 8;
    return 4;
  case AArch64::RETAA:
  case AArch64::RETAB:
    // Authenticated returns expand to 3 instructions (authenticate + guard +
    // ret).
    return 12;
  case AArch64::BRAA:
  case AArch64::BRAAZ:
  case AArch64::BRAB:
  case AArch64::BRABZ:
  case AArch64::BLRAA:
  case AArch64::BLRAAZ:
  case AArch64::BLRAB:
  case AArch64::BLRABZ:
    // Authenticated branches/calls expand to 3 instructions (authenticate +
    // guard + branch).
    return 12;
  case AArch64::AUTIASP:
  case AArch64::AUTIBSP:
  case AArch64::AUTIAZ:
  case AArch64::AUTIBZ:
  case AArch64::XPACLRI:
    // Authenticating LR expands to the instruction plus a deferred LR guard.
    return 8;
  case AArch64::SYSxt:
    // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
    if (MI.getOperand(0).getImm() == 3 && MI.getOperand(1).getImm() == 7 &&
        MI.getOperand(3).getImm() == 1)
      return 8;
    return std::nullopt;
  default:
    break;
  }
 
  // Detect instructions that explicitly define SP or LR.
  bool ModifiesLR = false;
  bool ModifiesSP = false;
  for (const MachineOperand &MO : MI.defs()) {
    if (!MO.isReg())
      continue;
    if (MO.getReg() == AArch64::LR)
      ModifiesLR = true;
    else if (MO.getReg() == AArch64::SP)
      ModifiesSP = true;
  }
 
  // Memory accesses expand to a base-register guard plus the rewritten access
  // (8 bytes), with an extra base-register update for pre/post-index forms (12
  // bytes total). If the access also defines LR, an LR mask is appended (+4
  // bytes). Depending on additional optimizations that the rewriter performs,
  // this may be an overestimate.
  if (MI.mayLoadOrStore()) {
    unsigned Size = isLFIPrePostMemAccess(MI.getOpcode()) ? 12 : 8;
    if (ModifiesLR)
      Size += 4;
    return Size;
  }
 
  // Non memory operations that modify LR or SP expand to 2 instructions.
  if (ModifiesSP || ModifiesLR)
    return 8;
 
  // Default case: instructions that don't cause expansion.
  // - TP accesses in LFI are a single load/store, so no expansion.
  // - All remaining instructions are not rewritten.
  return std::nullopt;
}
 
/// GetInstSize - Return the number of bytes of code the specified
/// instruction may be.  This returns the maximum number of bytes.
unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
  const MachineBasicBlock &MBB = *MI.getParent();
  const MachineFunction *MF = MBB.getParent();
  const Function &F = MF->getFunction();
  const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
 
  {
    auto Op = MI.getOpcode();
    if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
      return getInlineAsmLength(MI.getOperand(0).getSymbolName(), MAI);
  }
 
  // Meta-instructions emit no code.
  if (MI.isMetaInstruction())
    return 0;
 
  // FIXME: We currently only handle pseudoinstructions that don't get expanded
  //        before the assembly printer.
  unsigned NumBytes = 0;
  const MCInstrDesc &Desc = MI.getDesc();
 
  // LFI rewriter expansions that supersede normal sizing.
  const auto &STI = MF->getSubtarget<AArch64Subtarget>();
  if (STI.isLFI())
    if (auto Size = getLFIInstSizeInBytes(MI))
      return *Size;
 
  if (!MI.isBundle() && isTailCallReturnInst(MI)) {
    NumBytes = Desc.getSize() ? Desc.getSize() : 4;
 
    const auto *MFI = MF->getInfo<AArch64FunctionInfo>();
    if (!MFI->shouldSignReturnAddress(*MF))
      return NumBytes;
 
    auto Method = STI.getAuthenticatedLRCheckMethod(*MF);
    NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
    return NumBytes;
  }
 
  // Size should be preferably set in
  // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
  // Specific cases handle instructions of variable sizes
  switch (Desc.getOpcode()) {
  default:
    if (Desc.getSize())
      return Desc.getSize();
 
    // Anything not explicitly designated otherwise (i.e. pseudo-instructions
    // with fixed constant size but not specified in .td file) is a normal
    // 4-byte insn.
    NumBytes = 4;
    break;
  case TargetOpcode::STACKMAP:
    // The upper bound for a stackmap intrinsic is the full length of its shadow
    NumBytes = StackMapOpers(&MI).getNumPatchBytes();
    assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
    break;
  case TargetOpcode::PATCHPOINT:
    // The size of the patchpoint intrinsic is the number of bytes requested
    NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
    assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
    break;
  case TargetOpcode::STATEPOINT:
    NumBytes = StatepointOpers(&MI).getNumPatchBytes();
    assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
    // No patch bytes means a normal call inst is emitted
    if (NumBytes == 0)
      NumBytes = 4;
    break;
  case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
    // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER
    // instructions are expanded to the specified number of NOPs. Otherwise,
    // they are expanded to 36-byte XRay sleds.
    NumBytes =
        F.getFnAttributeAsParsedInteger("patchable-function-entry", 9) * 4;
    break;
  case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
  case TargetOpcode::PATCHABLE_TAIL_CALL:
  case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
    // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
    NumBytes = 36;
    break;
  case TargetOpcode::PATCHABLE_EVENT_CALL:
    // EVENT_CALL XRay sleds are exactly 6 instructions long (no alignment).
    NumBytes = 24;
    break;
 
  case AArch64::SPACE:
    NumBytes = MI.getOperand(1).getImm();
    break;
  case AArch64::MOVaddr:
  case AArch64::MOVaddrJT:
  case AArch64::MOVaddrCP:
  case AArch64::MOVaddrBA:
  case AArch64::MOVaddrTLS:
  case AArch64::MOVaddrEXT: {
    // Use the same logic as the pseudo expansion to count instructions.
    SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
    AArch64_IMM::expandMOVAddr(Desc.getOpcode(),
                               MI.getOperand(1).getTargetFlags(),
                               Subtarget.isTargetMachO(), Insn);
    NumBytes = Insn.size() * 4;
    break;
  }
 
  case AArch64::MOVi32imm:
  case AArch64::MOVi64imm: {
    // Use the same logic as the pseudo expansion to count instructions.
    unsigned BitSize = Desc.getOpcode() == AArch64::MOVi32imm ? 32 : 64;
    SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
    AArch64_IMM::expandMOVImm(MI.getOperand(1).getImm(), BitSize, Insn);
    NumBytes = Insn.size() * 4;
    break;
  }
 
  case TargetOpcode::BUNDLE:
    NumBytes = getInstBundleSize(MI);
    break;
  }
 
  return NumBytes;
}
 
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
                            SmallVectorImpl<MachineOperand> &Cond) {
  // Block ends with fall-through condbranch.
  switch (LastInst->getOpcode()) {
  default:
    llvm_unreachable("Unknown branch instruction?");
  case AArch64::Bcc:
    Target = LastInst->getOperand(1).getMBB();
    Cond.push_back(LastInst->getOperand(0));
    break;
  case AArch64::CBZW:
  case AArch64::CBZX:
  case AArch64::CBNZW:
  case AArch64::CBNZX:
    Target = LastInst->getOperand(1).getMBB();
    Cond.push_back(MachineOperand::CreateImm(-1));
    Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
    Cond.push_back(LastInst->getOperand(0));
    break;
  case AArch64::TBZW:
  case AArch64::TBZX:
  case AArch64::TBNZW:
  case AArch64::TBNZX:
    Target = LastInst->getOperand(2).getMBB();
    Cond.push_back(MachineOperand::CreateImm(-1));
    Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
    Cond.push_back(LastInst->getOperand(0));
    Cond.push_back(LastInst->getOperand(1));
    break;
  case AArch64::CBWPri:
  case AArch64::CBXPri:
  case AArch64::CBWPrr:
  case AArch64::CBXPrr:
    Target = LastInst->getOperand(3).getMBB();
    Cond.push_back(MachineOperand::CreateImm(-1));
    Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
    Cond.push_back(LastInst->getOperand(0));
    Cond.push_back(LastInst->getOperand(1));
    Cond.push_back(LastInst->getOperand(2));
    break;
  case AArch64::CBBAssertExt:
  case AArch64::CBHAssertExt:
    Target = LastInst->getOperand(3).getMBB();
    Cond.push_back(MachineOperand::CreateImm(-1));                    // -1
    Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode())); // Opc
    Cond.push_back(LastInst->getOperand(0));                          // Cond
    Cond.push_back(LastInst->getOperand(1));                          // Op0
    Cond.push_back(LastInst->getOperand(2));                          // Op1
    Cond.push_back(LastInst->getOperand(4));                          // Ext0
    Cond.push_back(LastInst->getOperand(5));                          // Ext1
    break;
  }
}
 
static unsigned getBranchDisplacementBits(unsigned Opc) {
  switch (Opc) {
  default:
    llvm_unreachable("unexpected opcode!");
  case AArch64::B:
    return BDisplacementBits;
  case AArch64::TBNZW:
  case AArch64::TBZW:
  case AArch64::TBNZX:
  case AArch64::TBZX:
    return TBZDisplacementBits;
  case AArch64::CBNZW:
  case AArch64::CBZW:
  case AArch64::CBNZX:
  case AArch64::CBZX:
    return CBZDisplacementBits;
  case AArch64::Bcc:
    return BCCDisplacementBits;
  case AArch64::CBWPri:
  case AArch64::CBXPri:
  case AArch64::CBBAssertExt:
  case AArch64::CBHAssertExt:
  case AArch64::CBWPrr:
  case AArch64::CBXPrr:
    return CBDisplacementBits;
  }
}
 
bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
                                             int64_t BrOffset) const {
  unsigned Bits = getBranchDisplacementBits(BranchOp);
  assert(Bits >= 3 && "max branch displacement must be enough to jump"
                      "over conditional branch expansion");
  return isIntN(Bits, BrOffset / 4);
}
 
MachineBasicBlock *
AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
  switch (MI.getOpcode()) {
  default:
    llvm_unreachable("unexpected opcode!");
  case AArch64::B:
    return MI.getOperand(0).getMBB();
  case AArch64::TBZW:
  case AArch64::TBNZW:
  case AArch64::TBZX:
  case AArch64::TBNZX:
    return MI.getOperand(2).getMBB();
  case AArch64::CBZW:
  case AArch64::CBNZW:
  case AArch64::CBZX:
  case AArch64::CBNZX:
  case AArch64::Bcc:
    return MI.getOperand(1).getMBB();
  case AArch64::CBWPri:
  case AArch64::CBXPri:
  case AArch64::CBBAssertExt:
  case AArch64::CBHAssertExt:
  case AArch64::CBWPrr:
  case AArch64::CBXPrr:
    return MI.getOperand(3).getMBB();
  }
}
 
void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
                                            MachineBasicBlock &NewDestBB,
                                            MachineBasicBlock &RestoreBB,
                                            const DebugLoc &DL,
                                            int64_t BrOffset,
                                            RegScavenger *RS) const {
  assert(RS && "RegScavenger required for long branching");
  assert(MBB.empty() &&
         "new block should be inserted for expanding unconditional branch");
  assert(MBB.pred_size() == 1);
  assert(RestoreBB.empty() &&
         "restore block should be inserted for restoring clobbered registers");
 
  auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) {
    // Offsets outside of the signed 33-bit range are not supported for ADRP +
    // ADD.
    if (!isInt<33>(BrOffset))
      report_fatal_error(
          "Branch offsets outside of the signed 33-bit range not supported");
 
    BuildMI(MBB, MBB.end(), DL, get(AArch64::ADRP), Reg)
        .addSym(DestBB.getSymbol(), AArch64II::MO_PAGE);
    BuildMI(MBB, MBB.end(), DL, get(AArch64::ADDXri), Reg)
        .addReg(Reg)
        .addSym(DestBB.getSymbol(), AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
        .addImm(0);
    BuildMI(MBB, MBB.end(), DL, get(AArch64::BR)).addReg(Reg);
  };
 
  RS->enterBasicBlockEnd(MBB);
  // If X16 is unused, we can rely on the linker to insert a range extension
  // thunk if NewDestBB is out of range of a single B instruction.
  constexpr Register Reg = AArch64::X16;
  if (!RS->isRegUsed(Reg)) {
    insertUnconditionalBranch(MBB, &NewDestBB, DL);
    RS->setRegUsed(Reg);
    return;
  }
 
  // In a cold block without BTI, insert the indirect branch if a register is
  // free. Skip this if BTI is enabled to avoid inserting a BTI at the target,
  // prioritizing a dynamic cost in cold code over a static cost in hot code.
  AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>();
  bool HasBTI = AFI && AFI->branchTargetEnforcement();
  if (MBB.getSectionID() == MBBSectionID::ColdSectionID && !HasBTI) {
    Register Scavenged = RS->FindUnusedReg(&AArch64::GPR64RegClass);
    if (Scavenged != AArch64::NoRegister) {
      buildIndirectBranch(Scavenged, NewDestBB);
      RS->setRegUsed(Scavenged);
      return;
    }
  }
 
  // Note: Spilling X16 briefly moves the stack pointer, making it incompatible
  // with red zones.
  if (!AFI || AFI->hasRedZone().value_or(true))
    report_fatal_error(
        "Unable to insert indirect branch inside function that has red zone");
 
  // Otherwise, spill X16 and defer range extension to the linker.
  BuildMI(MBB, MBB.end(), DL, get(AArch64::STRXpre))
      .addReg(AArch64::SP, RegState::Define)
      .addReg(Reg)
      .addReg(AArch64::SP)
      .addImm(-16);
 
  BuildMI(MBB, MBB.end(), DL, get(AArch64::B)).addMBB(&RestoreBB);
 
  BuildMI(RestoreBB, RestoreBB.end(), DL, get(AArch64::LDRXpost))
      .addReg(AArch64::SP, RegState::Define)
      .addReg(Reg, RegState::Define)
      .addReg(AArch64::SP)
      .addImm(16);
}
 
// Branch analysis.
bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
                                     MachineBasicBlock *&TBB,
                                     MachineBasicBlock *&FBB,
                                     SmallVectorImpl<MachineOperand> &Cond,
                                     bool AllowModify) const {
  // If the block has no terminators, it just falls into the block after it.
  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
  if (I == MBB.end())
    return false;
 
  // Skip over SpeculationBarrierEndBB terminators
  if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
      I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
    --I;
  }
 
  if (!isUnpredicatedTerminator(*I))
    return false;
 
  // Get the last instruction in the block.
  MachineInstr *LastInst = &*I;
 
  // If there is only one terminator instruction, process it.
  unsigned LastOpc = LastInst->getOpcode();
  if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
    if (isUncondBranchOpcode(LastOpc)) {
      TBB = LastInst->getOperand(0).getMBB();
      return false;
    }
    if (isCondBranchOpcode(LastOpc)) {
      // Block ends with fall-through condbranch.
      parseCondBranch(LastInst, TBB, Cond);
      return false;
    }
    return true; // Can't handle indirect branch.
  }
 
  // Get the instruction before it if it is a terminator.
  MachineInstr *SecondLastInst = &*I;
  unsigned SecondLastOpc = SecondLastInst->getOpcode();
 
  // If AllowModify is true and the block ends with two or more unconditional
  // branches, delete all but the first unconditional branch.
  if (AllowModify && isUncondBranchOpcode(LastOpc)) {
    while (isUncondBranchOpcode(SecondLastOpc)) {
      LastInst->eraseFromParent();
      LastInst = SecondLastInst;
      LastOpc = LastInst->getOpcode();
      if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
        // Return now the only terminator is an unconditional branch.
        TBB = LastInst->getOperand(0).getMBB();
        return false;
      }
      SecondLastInst = &*I;
      SecondLastOpc = SecondLastInst->getOpcode();
    }
  }
 
  // If we're allowed to modify and the block ends in a unconditional branch
  // which could simply fallthrough, remove the branch.  (Note: This case only
  // matters when we can't understand the whole sequence, otherwise it's also
  // handled by BranchFolding.cpp.)
  if (AllowModify && isUncondBranchOpcode(LastOpc) &&
      MBB.isLayoutSuccessor(getBranchDestBlock(*LastInst))) {
    LastInst->eraseFromParent();
    LastInst = SecondLastInst;
    LastOpc = LastInst->getOpcode();
    if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
      assert(!isUncondBranchOpcode(LastOpc) &&
             "unreachable unconditional branches removed above");
 
      if (isCondBranchOpcode(LastOpc)) {
        // Block ends with fall-through condbranch.
        parseCondBranch(LastInst, TBB, Cond);
        return false;
      }
      return true; // Can't handle indirect branch.
    }
    SecondLastInst = &*I;
    SecondLastOpc = SecondLastInst->getOpcode();
  }
 
  // If there are three terminators, we don't know what sort of block this is.
  if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I))
    return true;
 
  // If the block ends with a B and a Bcc, handle it.
  if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
    parseCondBranch(SecondLastInst, TBB, Cond);
    FBB = LastInst->getOperand(0).getMBB();
    return false;
  }
 
  // If the block ends with two unconditional branches, handle it.  The second
  // one is not executed, so remove it.
  if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
    TBB = SecondLastInst->getOperand(0).getMBB();
    I = LastInst;
    if (AllowModify)
      I->eraseFromParent();
    return false;
  }
 
  // ...likewise if it ends with an indirect branch followed by an unconditional
  // branch.
  if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
    I = LastInst;
    if (AllowModify)
      I->eraseFromParent();
    return true;
  }
 
  // Otherwise, can't handle this.
  return true;
}
 
bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
                                              MachineBranchPredicate &MBP,
                                              bool AllowModify) const {
  // Use analyzeBranch to validate the branch pattern.
  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
  SmallVector<MachineOperand, 4> Cond;
  if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
    return true;
 
  // analyzeBranch returns success with empty Cond for unconditional branches.
  if (Cond.empty())
    return true;
 
  MBP.TrueDest = TBB;
  assert(MBP.TrueDest && "expected!");
  MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
 
  MBP.ConditionDef = nullptr;
  MBP.SingleUseCondition = false;
 
  // Find the conditional branch. After analyzeBranch succeeds with non-empty
  // Cond, there's exactly one conditional branch - either last (fallthrough)
  // or second-to-last (followed by unconditional B).
  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
  if (I == MBB.end())
    return true;
 
  if (isUncondBranchOpcode(I->getOpcode())) {
    if (I == MBB.begin())
      return true;
    --I;
  }
 
  MachineInstr *CondBranch = &*I;
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
 
  switch (CondBranch->getOpcode()) {
  default:
    return true;
 
  case AArch64::Bcc:
    // Bcc takes the NZCV flag as the operand to branch on, walk up the
    // instruction stream to find the last instruction to define NZCV.
    for (MachineInstr &MI : llvm::drop_begin(llvm::reverse(MBB))) {
      if (MI.modifiesRegister(AArch64::NZCV, /*TRI=*/nullptr)) {
        MBP.ConditionDef = &MI;
        break;
      }
    }
    return false;
 
  case AArch64::CBZW:
  case AArch64::CBZX:
  case AArch64::CBNZW:
  case AArch64::CBNZX: {
    MBP.LHS = CondBranch->getOperand(0);
    MBP.RHS = MachineOperand::CreateImm(0);
    unsigned Opc = CondBranch->getOpcode();
    MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
                        ? MachineBranchPredicate::PRED_NE
                        : MachineBranchPredicate::PRED_EQ;
    Register CondReg = MBP.LHS.getReg();
    if (CondReg.isVirtual())
      MBP.ConditionDef = MRI.getVRegDef(CondReg);
    return false;
  }
 
  case AArch64::TBZW:
  case AArch64::TBZX:
  case AArch64::TBNZW:
  case AArch64::TBNZX: {
    Register CondReg = CondBranch->getOperand(0).getReg();
    if (CondReg.isVirtual())
      MBP.ConditionDef = MRI.getVRegDef(CondReg);
    return false;
  }
  }
}
 
bool AArch64InstrInfo::reverseBranchCondition(
    SmallVectorImpl<MachineOperand> &Cond) const {
  if (Cond[0].getImm() != -1) {
    // Regular Bcc
    AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
    Cond[0].setImm(AArch64CC::getInvertedCondCode(CC));
  } else {
    // Folded compare-and-branch
    switch (Cond[1].getImm()) {
    default:
      llvm_unreachable("Unknown conditional branch!");
    case AArch64::CBZW:
      Cond[1].setImm(AArch64::CBNZW);
      break;
    case AArch64::CBNZW:
      Cond[1].setImm(AArch64::CBZW);
      break;
    case AArch64::CBZX:
      Cond[1].setImm(AArch64::CBNZX);
      break;
    case AArch64::CBNZX:
      Cond[1].setImm(AArch64::CBZX);
      break;
    case AArch64::TBZW:
      Cond[1].setImm(AArch64::TBNZW);
      break;
    case AArch64::TBNZW:
      Cond[1].setImm(AArch64::TBZW);
      break;
    case AArch64::TBZX:
      Cond[1].setImm(AArch64::TBNZX);
      break;
    case AArch64::TBNZX:
      Cond[1].setImm(AArch64::TBZX);
      break;
 
    // Cond is { -1, Opcode, CC, Op0, Op1, ... }
    case AArch64::CBWPri:
    case AArch64::CBXPri:
    case AArch64::CBBAssertExt:
    case AArch64::CBHAssertExt:
    case AArch64::CBWPrr:
    case AArch64::CBXPrr: {
      // Pseudos using standard 4bit Arm condition codes
      AArch64CC::CondCode CC =
          static_cast<AArch64CC::CondCode>(Cond[2].getImm());
      Cond[2].setImm(AArch64CC::getInvertedCondCode(CC));
    }
    }
  }
 
  return false;
}
 
unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
                                        int *BytesRemoved) const {
  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
  if (I == MBB.end())
    return 0;
 
  if (!isUncondBranchOpcode(I->getOpcode()) &&
      !isCondBranchOpcode(I->getOpcode()))
    return 0;
 
  // Remove the branch.
  I->eraseFromParent();
 
  I = MBB.end();
 
  if (I == MBB.begin()) {
    if (BytesRemoved)
      *BytesRemoved = 4;
    return 1;
  }
  --I;
  if (!isCondBranchOpcode(I->getOpcode())) {
    if (BytesRemoved)
      *BytesRemoved = 4;
    return 1;
  }
 
  // Remove the branch.
  I->eraseFromParent();
  if (BytesRemoved)
    *BytesRemoved = 8;
 
  return 2;
}
 
void AArch64InstrInfo::instantiateCondBranch(
    MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
    ArrayRef<MachineOperand> Cond) const {
  if (Cond[0].getImm() != -1) {
    // Regular Bcc
    BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
  } else {
    // Folded compare-and-branch
    // Note that we use addOperand instead of addReg to keep the flags.
 
    // cbz, cbnz
    const MachineInstrBuilder MIB =
        BuildMI(&MBB, DL, get(Cond[1].getImm())).add(Cond[2]);
 
    // tbz/tbnz
    if (Cond.size() > 3)
      MIB.add(Cond[3]);
 
    // cb
    if (Cond.size() > 4)
      MIB.add(Cond[4]);
 
    MIB.addMBB(TBB);
 
    // cb[b,h]
    if (Cond.size() > 5) {
      MIB.addImm(Cond[5].getImm());
      MIB.addImm(Cond[6].getImm());
    }
  }
}
 
unsigned AArch64InstrInfo::insertBranch(
    MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
    ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
  // Shouldn't be a fall through.
  assert(TBB && "insertBranch must not be told to insert a fallthrough");
 
  if (!FBB) {
    if (Cond.empty()) // Unconditional branch?
      BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
    else
      instantiateCondBranch(MBB, DL, TBB, Cond);
 
    if (BytesAdded)
      *BytesAdded = 4;
 
    return 1;
  }
 
  // Two-way conditional branch.
  instantiateCondBranch(MBB, DL, TBB, Cond);
  BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
 
  if (BytesAdded)
    *BytesAdded = 8;
 
  return 2;
}
 
bool llvm::optimizeTerminators(MachineBasicBlock *MBB,
                               const TargetInstrInfo &TII) {
  for (MachineInstr &MI : MBB->terminators()) {
    unsigned Opc = MI.getOpcode();
    switch (Opc) {
    case AArch64::CBZW:
    case AArch64::CBZX:
    case AArch64::TBZW:
    case AArch64::TBZX:
      // CBZ/TBZ with WZR/XZR -> unconditional B
      if (MI.getOperand(0).getReg() == AArch64::WZR ||
          MI.getOperand(0).getReg() == AArch64::XZR) {
        DEBUG_WITH_TYPE("optimizeTerminators",
                        dbgs() << "Removing always taken branch: " << MI);
        MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
        SmallVector<MachineBasicBlock *> Succs(MBB->successors());
        for (auto *S : Succs)
          if (S != Target)
            MBB->removeSuccessor(S);
        DebugLoc DL = MI.getDebugLoc();
        while (MBB->rbegin() != &MI)
          MBB->rbegin()->eraseFromParent();
        MI.eraseFromParent();
        BuildMI(MBB, DL, TII.get(AArch64::B)).addMBB(Target);
        return true;
      }
      break;
    case AArch64::CBNZW:
    case AArch64::CBNZX:
    case AArch64::TBNZW:
    case AArch64::TBNZX:
      // CBNZ/TBNZ with WZR/XZR -> never taken, remove branch and successor
      if (MI.getOperand(0).getReg() == AArch64::WZR ||
          MI.getOperand(0).getReg() == AArch64::XZR) {
        DEBUG_WITH_TYPE("optimizeTerminators",
                        dbgs() << "Removing never taken branch: " << MI);
        MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
        MI.getParent()->removeSuccessor(Target);
        MI.eraseFromParent();
        return true;
      }
      break;
    }
  }
  return false;
}
 
// Find the original register that VReg is copied from.
static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
  while (Register::isVirtualRegister(VReg)) {
    const MachineInstr *DefMI = MRI.getVRegDef(VReg);
    if (!DefMI || !DefMI->isFullCopy())
      return VReg;
    VReg = DefMI->getOperand(1).getReg();
  }
  return VReg;
}
 
// Determine if VReg is defined by an instruction that can be folded into a
// csel instruction. If so, return the folded opcode, and the replacement
// register.
static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
                                unsigned *NewReg = nullptr) {
  VReg = removeCopies(MRI, VReg);
  if (!Register::isVirtualRegister(VReg))
    return 0;
 
  bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
  const MachineInstr *DefMI = MRI.getVRegDef(VReg);
  if (!DefMI)
    return 0;
  unsigned Opc = 0;
  unsigned SrcReg = 0;
  switch (DefMI->getOpcode()) {
  case AArch64::SUBREG_TO_REG:
    // Check for the following way to define an 64-bit immediate:
    //   %0:gpr32 = MOVi32imm 1
    //   %1:gpr64 = SUBREG_TO_REG %0:gpr32, %subreg.sub_32
    if (!DefMI->getOperand(1).isReg())
      return 0;
    if (!DefMI->getOperand(2).isImm() ||
        DefMI->getOperand(2).getImm() != AArch64::sub_32)
      return 0;
    DefMI = MRI.getVRegDef(DefMI->getOperand(1).getReg());
    if (DefMI->getOpcode() != AArch64::MOVi32imm)
      return 0;
    if (!DefMI->getOperand(1).isImm() || DefMI->getOperand(1).getImm() != 1)
      return 0;
    assert(Is64Bit);
    SrcReg = AArch64::XZR;
    Opc = AArch64::CSINCXr;
    break;
 
  case AArch64::MOVi32imm:
  case AArch64::MOVi64imm:
    if (!DefMI->getOperand(1).isImm() || DefMI->getOperand(1).getImm() != 1)
      return 0;
    SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
    Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
    break;
 
  case AArch64::ADDSXri:
  case AArch64::ADDSWri:
    // if NZCV is used, do not fold.
    if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr,
                                         true) == -1)
      return 0;
    // fall-through to ADDXri and ADDWri.
    [[fallthrough]];
  case AArch64::ADDXri:
  case AArch64::ADDWri:
    // add x, 1 -> csinc.
    if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
        DefMI->getOperand(3).getImm() != 0)
      return 0;
    SrcReg = DefMI->getOperand(1).getReg();
    Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
    break;
 
  case AArch64::ORNXrr:
  case AArch64::ORNWrr: {
    // not x -> csinv, represented as orn dst, xzr, src.
    unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
    if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
      return 0;
    SrcReg = DefMI->getOperand(2).getReg();
    Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
    break;
  }
 
  case AArch64::SUBSXrr:
  case AArch64::SUBSWrr:
    // if NZCV is used, do not fold.
    if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr,
                                         true) == -1)
      return 0;
    // fall-through to SUBXrr and SUBWrr.
    [[fallthrough]];
  case AArch64::SUBXrr:
  case AArch64::SUBWrr: {
    // neg x -> csneg, represented as sub dst, xzr, src.
    unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
    if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
      return 0;
    SrcReg = DefMI->getOperand(2).getReg();
    Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
    break;
  }
  default:
    return 0;
  }
  assert(Opc && SrcReg && "Missing parameters");
 
  if (NewReg)
    *NewReg = SrcReg;
  return Opc;
}
 
bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                       ArrayRef<MachineOperand> Cond,
                                       Register DstReg, Register TrueReg,
                                       Register FalseReg, int &CondCycles,
                                       int &TrueCycles,
                                       int &FalseCycles) const {
  // Check register classes.
  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
  const TargetRegisterClass *RC =
      RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
  if (!RC)
    return false;
 
  // Also need to check the dest regclass, in case we're trying to optimize
  // something like:
  // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
  if (!RI.getCommonSubClass(RC, MRI.getRegClass(DstReg)))
    return false;
 
  // Expanding cbz/tbz requires an extra cycle of latency on the condition.
  unsigned ExtraCondLat = Cond.size() != 1;
 
  // GPRs are handled by csel.
  // FIXME: Fold in x+1, -x, and ~x when applicable.
  if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
      AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
    // Single-cycle csel, csinc, csinv, and csneg.
    CondCycles = 1 + ExtraCondLat;
    TrueCycles = FalseCycles = 1;
    if (canFoldIntoCSel(MRI, TrueReg))
      TrueCycles = 0;
    else if (canFoldIntoCSel(MRI, FalseReg))
      FalseCycles = 0;
    return true;
  }
 
  // Scalar floating point is handled by fcsel.
  // FIXME: Form fabs, fmin, and fmax when applicable.
  if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
      AArch64::FPR32RegClass.hasSubClassEq(RC)) {
    CondCycles = 5 + ExtraCondLat;
    TrueCycles = FalseCycles = 2;
    return true;
  }
 
  // Can't do vectors.
  return false;
}
 
void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator I,
                                    const DebugLoc &DL, Register DstReg,
                                    ArrayRef<MachineOperand> Cond,
                                    Register TrueReg, Register FalseReg) const {
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
 
  // Parse the condition code, see parseCondBranch() above.
  AArch64CC::CondCode CC;
  switch (Cond.size()) {
  default:
    llvm_unreachable("Unknown condition opcode in Cond");
  case 1: // b.cc
    CC = AArch64CC::CondCode(Cond[0].getImm());
    break;
  case 3: { // cbz/cbnz
    // We must insert a compare against 0.
    bool Is64Bit;
    switch (Cond[1].getImm()) {
    default:
      llvm_unreachable("Unknown branch opcode in Cond");
    case AArch64::CBZW:
      Is64Bit = false;
      CC = AArch64CC::EQ;
      break;
    case AArch64::CBZX:
      Is64Bit = true;
      CC = AArch64CC::EQ;
      break;
    case AArch64::CBNZW:
      Is64Bit = false;
      CC = AArch64CC::NE;
      break;
    case AArch64::CBNZX:
      Is64Bit = true;
      CC = AArch64CC::NE;
      break;
    }
    Register SrcReg = Cond[2].getReg();
    if (Is64Bit) {
      // cmp reg, #0 is actually subs xzr, reg, #0.
      MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
      BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
          .addReg(SrcReg)
          .addImm(0)
          .addImm(0);
    } else {
      MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
      BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
          .addReg(SrcReg)
          .addImm(0)
          .addImm(0);
    }
    break;
  }
  case 4: { // tbz/tbnz
    // We must insert a tst instruction.
    switch (Cond[1].getImm()) {
    default:
      llvm_unreachable("Unknown branch opcode in Cond");
    case AArch64::TBZW:
    case AArch64::TBZX:
      CC = AArch64CC::EQ;
      break;
    case AArch64::TBNZW:
    case AArch64::TBNZX:
      CC = AArch64CC::NE;
      break;
    }
    // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
    if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
      BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
          .addReg(Cond[2].getReg())
          .addImm(
              AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
    else
      BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
          .addReg(Cond[2].getReg())
          .addImm(
              AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
    break;
  }
  case 5: { // cb
    // We must insert a cmp, that is a subs
    //            0       1   2    3    4
    // Cond is { -1, Opcode, CC, Op0, Op1 }
 
    unsigned SubsOpc, SubsDestReg;
    bool IsImm = false;
    CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
    switch (Cond[1].getImm()) {
    default:
      llvm_unreachable("Unknown branch opcode in Cond");
    case AArch64::CBWPri:
      SubsOpc = AArch64::SUBSWri;
      SubsDestReg = AArch64::WZR;
      IsImm = true;
      break;
    case AArch64::CBXPri:
      SubsOpc = AArch64::SUBSXri;
      SubsDestReg = AArch64::XZR;
      IsImm = true;
      break;
    case AArch64::CBWPrr:
      SubsOpc = AArch64::SUBSWrr;
      SubsDestReg = AArch64::WZR;
      IsImm = false;
      break;
    case AArch64::CBXPrr:
      SubsOpc = AArch64::SUBSXrr;
      SubsDestReg = AArch64::XZR;
      IsImm = false;
      break;
    }
 
    if (IsImm)
      BuildMI(MBB, I, DL, get(SubsOpc), SubsDestReg)
          .addReg(Cond[3].getReg())
          .addImm(Cond[4].getImm())
          .addImm(0);
    else
      BuildMI(MBB, I, DL, get(SubsOpc), SubsDestReg)
          .addReg(Cond[3].getReg())
          .addReg(Cond[4].getReg());
  } break;
  case 7: { // cb[b,h]
    // We must insert a cmp, that is a subs, but also zero- or sign-extensions
    // that have been folded. For the first operand we codegen an explicit
    // extension, for the second operand we fold the extension into cmp.
    //            0       1   2    3    4    5      6
    // Cond is { -1, Opcode, CC, Op0, Op1, Ext0, Ext1 }
 
    // We need a new register for the now explicitly extended register
    Register Reg = Cond[4].getReg();
    if (Cond[5].getImm() != AArch64_AM::InvalidShiftExtend) {
      unsigned ExtOpc;
      unsigned ExtBits;
      AArch64_AM::ShiftExtendType ExtendType =
          AArch64_AM::getExtendType(Cond[5].getImm());
      switch (ExtendType) {
      default:
        llvm_unreachable("Unknown shift-extend for CB instruction");
      case AArch64_AM::SXTB:
        assert(
            Cond[1].getImm() == AArch64::CBBAssertExt &&
            "Unexpected compare-and-branch instruction for SXTB shift-extend");
        ExtOpc = AArch64::SBFMWri;
        ExtBits = AArch64_AM::encodeLogicalImmediate(0xff, 32);
        break;
      case AArch64_AM::SXTH:
        assert(
            Cond[1].getImm() == AArch64::CBHAssertExt &&
            "Unexpected compare-and-branch instruction for SXTH shift-extend");
        ExtOpc = AArch64::SBFMWri;
        ExtBits = AArch64_AM::encodeLogicalImmediate(0xffff, 32);
        break;
      case AArch64_AM::UXTB:
        assert(
            Cond[1].getImm() == AArch64::CBBAssertExt &&
            "Unexpected compare-and-branch instruction for UXTB shift-extend");
        ExtOpc = AArch64::ANDWri;
        ExtBits = AArch64_AM::encodeLogicalImmediate(0xff, 32);
        break;
      case AArch64_AM::UXTH:
        assert(
            Cond[1].getImm() == AArch64::CBHAssertExt &&
            "Unexpected compare-and-branch instruction for UXTH shift-extend");
        ExtOpc = AArch64::ANDWri;
        ExtBits = AArch64_AM::encodeLogicalImmediate(0xffff, 32);
        break;
      }
 
      // Build the explicit extension of the first operand
      Reg = MRI.createVirtualRegister(&AArch64::GPR32spRegClass);
      MachineInstrBuilder MBBI =
          BuildMI(MBB, I, DL, get(ExtOpc), Reg).addReg(Cond[4].getReg());
      if (ExtOpc != AArch64::ANDWri)
        MBBI.addImm(0);
      MBBI.addImm(ExtBits);
    }
 
    // Now, subs with an extended second operand
    if (Cond[6].getImm() != AArch64_AM::InvalidShiftExtend) {
      AArch64_AM::ShiftExtendType ExtendType =
          AArch64_AM::getExtendType(Cond[6].getImm());
      MRI.constrainRegClass(Reg, MRI.getRegClass(Cond[3].getReg()));
      MRI.constrainRegClass(Cond[3].getReg(), &AArch64::GPR32spRegClass);
      BuildMI(MBB, I, DL, get(AArch64::SUBSWrx), AArch64::WZR)
          .addReg(Cond[3].getReg())
          .addReg(Reg)
          .addImm(AArch64_AM::getArithExtendImm(ExtendType, 0));
    } // If no extension is needed, just a regular subs
    else {
      MRI.constrainRegClass(Reg, MRI.getRegClass(Cond[3].getReg()));
      MRI.constrainRegClass(Cond[3].getReg(), &AArch64::GPR32spRegClass);
      BuildMI(MBB, I, DL, get(AArch64::SUBSWrr), AArch64::WZR)
          .addReg(Cond[3].getReg())
          .addReg(Reg);
    }
 
    CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
  } break;
  }
 
  unsigned Opc = 0;
  const TargetRegisterClass *RC = nullptr;
  bool TryFold = false;
  if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
    RC = &AArch64::GPR64RegClass;
    Opc = AArch64::CSELXr;
    TryFold = true;
  } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
    RC = &AArch64::GPR32RegClass;
    Opc = AArch64::CSELWr;
    TryFold = true;
  } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
    RC = &AArch64::FPR64RegClass;
    Opc = AArch64::FCSELDrrr;
  } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
    RC = &AArch64::FPR32RegClass;
    Opc = AArch64::FCSELSrrr;
  }
  assert(RC && "Unsupported regclass");
 
  // Try folding simple instructions into the csel.
  if (TryFold) {
    unsigned NewReg = 0;
    unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewReg);
    if (FoldedOpc) {
      // The folded opcodes csinc, csinc and csneg apply the operation to
      // FalseReg, so we need to invert the condition.
      CC = AArch64CC::getInvertedCondCode(CC);
      TrueReg = FalseReg;
    } else
      FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewReg);
 
    // Fold the operation. Leave any dead instructions for DCE to clean up.
    if (FoldedOpc) {
      FalseReg = NewReg;
      Opc = FoldedOpc;
      // Extend the live range of NewReg.
      MRI.clearKillFlags(NewReg);
    }
  }
 
  // Pull all virtual register into the appropriate class.
  MRI.constrainRegClass(TrueReg, RC);
  // FalseReg might be WZR or XZR if the folded operand is a literal 1.
  assert(
      (FalseReg.isVirtual() || FalseReg == AArch64::WZR ||
       FalseReg == AArch64::XZR) &&
      "FalseReg was folded into a non-virtual register other than WZR or XZR");
  if (FalseReg.isVirtual())
    MRI.constrainRegClass(FalseReg, RC);
 
  // Insert the csel.
  BuildMI(MBB, I, DL, get(Opc), DstReg)
      .addReg(TrueReg)
      .addReg(FalseReg)
      .addImm(CC);
}
 
// Return true if Imm can be loaded into a register by a "cheap" sequence of
// instructions. For now, "cheap" means at most two instructions.
static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize) {
  if (BitSize == 32)
    return true;
 
  assert(BitSize == 64 && "Only bit sizes of 32 or 64 allowed");
  uint64_t Imm = static_cast<uint64_t>(MI.getOperand(1).getImm());
  SmallVector<AArch64_IMM::ImmInsnModel, 4> Is;
  AArch64_IMM::expandMOVImm(Imm, BitSize, Is);
 
  return Is.size() <= 2;
}
 
// Check if a COPY instruction is cheap.
static bool isCheapCopy(const MachineInstr &MI, const AArch64RegisterInfo &RI) {
  assert(MI.isCopy() && "Expected COPY instruction");
  const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
 
  // Cross-bank copies (e.g., between GPR and FPR) are expensive on AArch64,
  // typically requiring an FMOV instruction with a 2-6 cycle latency.
  auto GetRegClass = [&](Register Reg) -> const TargetRegisterClass * {
    if (Reg.isVirtual())
      return MRI.getRegClass(Reg);
    if (Reg.isPhysical())
      return RI.getMinimalPhysRegClass(Reg);
    return nullptr;
  };
  const TargetRegisterClass *DstRC = GetRegClass(MI.getOperand(0).getReg());
  const TargetRegisterClass *SrcRC = GetRegClass(MI.getOperand(1).getReg());
  if (DstRC && SrcRC && !RI.getCommonSubClass(DstRC, SrcRC))
    return false;
 
  return MI.isAsCheapAsAMove();
}
 
// FIXME: this implementation should be micro-architecture dependent, so a
// micro-architecture target hook should be introduced here in future.
bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
  if (Subtarget.hasExynosCheapAsMoveHandling()) {
    if (isExynosCheapAsMove(MI))
      return true;
    return MI.isAsCheapAsAMove();
  }
 
  switch (MI.getOpcode()) {
  default:
    return MI.isAsCheapAsAMove();
 
  case TargetOpcode::COPY:
    return isCheapCopy(MI, RI);
 
  case AArch64::ADDWrs:
  case AArch64::ADDXrs:
  case AArch64::SUBWrs:
  case AArch64::SUBXrs:
    return Subtarget.hasALULSLFast() && MI.getOperand(3).getImm() <= 4;
 
  // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
  // ORRXri, it is as cheap as MOV.
  // Likewise if it can be expanded to MOVZ/MOVN/MOVK.
  case AArch64::MOVi32imm:
    return isCheapImmediate(MI, 32);
  case AArch64::MOVi64imm:
    return isCheapImmediate(MI, 64);
  }
}
 
bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    return false;
 
  case AArch64::ADDWrs:
  case AArch64::ADDXrs:
  case AArch64::ADDSWrs:
  case AArch64::ADDSXrs: {
    unsigned Imm = MI.getOperand(3).getImm();
    unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
    if (ShiftVal == 0)
      return true;
    return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
  }
 
  case AArch64::ADDWrx:
  case AArch64::ADDXrx:
  case AArch64::ADDXrx64:
  case AArch64::ADDSWrx:
  case AArch64::ADDSXrx:
  case AArch64::ADDSXrx64: {
    unsigned Imm = MI.getOperand(3).getImm();
    switch (AArch64_AM::getArithExtendType(Imm)) {
    default:
      return false;
    case AArch64_AM::UXTB:
    case AArch64_AM::UXTH:
    case AArch64_AM::UXTW:
    case AArch64_AM::UXTX:
      return AArch64_AM::getArithShiftValue(Imm) <= 4;
    }
  }
 
  case AArch64::SUBWrs:
  case AArch64::SUBSWrs: {
    unsigned Imm = MI.getOperand(3).getImm();
    unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
    return ShiftVal == 0 ||
           (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
  }
 
  case AArch64::SUBXrs:
  case AArch64::SUBSXrs: {
    unsigned Imm = MI.getOperand(3).getImm();
    unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
    return ShiftVal == 0 ||
           (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
  }
 
  case AArch64::SUBWrx:
  case AArch64::SUBXrx:
  case AArch64::SUBXrx64:
  case AArch64::SUBSWrx:
  case AArch64::SUBSXrx:
  case AArch64::SUBSXrx64: {
    unsigned Imm = MI.getOperand(3).getImm();
    switch (AArch64_AM::getArithExtendType(Imm)) {
    default:
      return false;
    case AArch64_AM::UXTB:
    case AArch64_AM::UXTH:
    case AArch64_AM::UXTW:
    case AArch64_AM::UXTX:
      return AArch64_AM::getArithShiftValue(Imm) == 0;
    }
  }
 
  case AArch64::LDRBBroW:
  case AArch64::LDRBBroX:
  case AArch64::LDRBroW:
  case AArch64::LDRBroX:
  case AArch64::LDRDroW:
  case AArch64::LDRDroX:
  case AArch64::LDRHHroW:
  case AArch64::LDRHHroX:
  case AArch64::LDRHroW:
  case AArch64::LDRHroX:
  case AArch64::LDRQroW:
  case AArch64::LDRQroX:
  case AArch64::LDRSBWroW:
  case AArch64::LDRSBWroX:
  case AArch64::LDRSBXroW:
  case AArch64::LDRSBXroX:
  case AArch64::LDRSHWroW:
  case AArch64::LDRSHWroX:
  case AArch64::LDRSHXroW:
  case AArch64::LDRSHXroX:
  case AArch64::LDRSWroW:
  case AArch64::LDRSWroX:
  case AArch64::LDRSroW:
  case AArch64::LDRSroX:
  case AArch64::LDRWroW:
  case AArch64::LDRWroX:
  case AArch64::LDRXroW:
  case AArch64::LDRXroX:
  case AArch64::PRFMroW:
  case AArch64::PRFMroX:
  case AArch64::STRBBroW:
  case AArch64::STRBBroX:
  case AArch64::STRBroW:
  case AArch64::STRBroX:
  case AArch64::STRDroW:
  case AArch64::STRDroX:
  case AArch64::STRHHroW:
  case AArch64::STRHHroX:
  case AArch64::STRHroW:
  case AArch64::STRHroX:
  case AArch64::STRQroW:
  case AArch64::STRQroX:
  case AArch64::STRSroW:
  case AArch64::STRSroX:
  case AArch64::STRWroW:
  case AArch64::STRWroX:
  case AArch64::STRXroW:
  case AArch64::STRXroX: {
    unsigned IsSigned = MI.getOperand(3).getImm();
    return !IsSigned;
  }
  }
}
 
bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
  unsigned Opc = MI.getOpcode();
  switch (Opc) {
    default:
      return false;
    case AArch64::SEH_StackAlloc:
    case AArch64::SEH_SaveFPLR:
    case AArch64::SEH_SaveFPLR_X:
    case AArch64::SEH_SaveReg:
    case AArch64::SEH_SaveReg_X:
    case AArch64::SEH_SaveRegP:
    case AArch64::SEH_SaveRegP_X:
    case AArch64::SEH_SaveFReg:
    case AArch64::SEH_SaveFReg_X:
    case AArch64::SEH_SaveFRegP:
    case AArch64::SEH_SaveFRegP_X:
    case AArch64::SEH_SetFP:
    case AArch64::SEH_AddFP:
    case AArch64::SEH_Nop:
    case AArch64::SEH_PrologEnd:
    case AArch64::SEH_EpilogStart:
    case AArch64::SEH_EpilogEnd:
    case AArch64::SEH_PACSignLR:
    case AArch64::SEH_SaveAnyRegI:
    case AArch64::SEH_SaveAnyRegIP:
    case AArch64::SEH_SaveAnyRegQP:
    case AArch64::SEH_SaveAnyRegQPX:
    case AArch64::SEH_AllocZ:
    case AArch64::SEH_SaveZReg:
    case AArch64::SEH_SavePReg:
      return true;
  }
}
 
bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
                                             Register &SrcReg, Register &DstReg,
                                             unsigned &SubIdx) const {
  switch (MI.getOpcode()) {
  default:
    return false;
  case AArch64::SBFMXri: // aka sxtw
  case AArch64::UBFMXri: // aka uxtw
    // Check for the 32 -> 64 bit extension case, these instructions can do
    // much more.
    if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
      return false;
    // This is a signed or unsigned 32 -> 64 bit extension.
    SrcReg = MI.getOperand(1).getReg();
    DstReg = MI.getOperand(0).getReg();
    SubIdx = AArch64::sub_32;
    return true;
  }
}
 
bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
    const MachineInstr &MIa, const MachineInstr &MIb) const {
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
  int64_t OffsetA = 0, OffsetB = 0;
  TypeSize WidthA(0, false), WidthB(0, false);
  bool OffsetAIsScalable = false, OffsetBIsScalable = false;
 
  assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
  assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
 
  if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
      MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
    return false;
 
  // Retrieve the base, offset from the base and width. Width
  // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8).  If
  // base are identical, and the offset of a lower memory access +
  // the width doesn't overlap the offset of a higher memory access,
  // then the memory accesses are different.
  // If OffsetAIsScalable and OffsetBIsScalable are both true, they
  // are assumed to have the same scale (vscale).
  if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, OffsetAIsScalable,
                                   WidthA, TRI) &&
      getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, OffsetBIsScalable,
                                   WidthB, TRI)) {
    if (BaseOpA->isIdenticalTo(*BaseOpB) &&
        OffsetAIsScalable == OffsetBIsScalable) {
      int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
      int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
      TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
      if (LowWidth.isScalable() == OffsetAIsScalable &&
          LowOffset + (int)LowWidth.getKnownMinValue() <= HighOffset)
        return true;
    }
  }
  return false;
}
 
bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
                                            const MachineBasicBlock *MBB,
                                            const MachineFunction &MF) const {
  if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
    return true;
 
  // Do not move an instruction that can be recognized as a branch target.
  if (hasBTISemantics(MI))
    return true;
 
  switch (MI.getOpcode()) {
  case AArch64::HINT:
    // CSDB hints are scheduling barriers.
    if (MI.getOperand(0).getImm() == 0x14)
      return true;
    break;
  case AArch64::DSB:
  case AArch64::ISB:
    // DSB and ISB also are scheduling barriers.
    return true;
  case AArch64::MSRpstatesvcrImm1:
    // SMSTART and SMSTOP are also scheduling barriers.
    return true;
  default:;
  }
  if (isSEHInstruction(MI))
    return true;
  auto Next = std::next(MI.getIterator());
  return Next != MBB->end() && Next->isCFIInstruction();
}
 
/// analyzeCompare - For a comparison instruction, return the source registers
/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
/// Return true if the comparison instruction can be analyzed.
bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
                                      Register &SrcReg2, int64_t &CmpMask,
                                      int64_t &CmpValue) const {
  // The first operand can be a frame index where we'd normally expect a
  // register.
  // FIXME: Pass subregisters out of analyzeCompare
  assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
  if (!MI.getOperand(1).isReg() || MI.getOperand(1).getSubReg())
    return false;
 
  switch (MI.getOpcode()) {
  default:
    break;
  case AArch64::PTEST_PP:
  case AArch64::PTEST_PP_ANY:
  case AArch64::PTEST_PP_FIRST:
    SrcReg = MI.getOperand(0).getReg();
    SrcReg2 = MI.getOperand(1).getReg();
    if (MI.getOperand(2).getSubReg())
      return false;
 
    // Not sure about the mask and value for now...
    CmpMask = ~0;
    CmpValue = 0;
    return true;
  case AArch64::SUBSWrr:
  case AArch64::SUBSWrs:
  case AArch64::SUBSWrx:
  case AArch64::SUBSXrr:
  case AArch64::SUBSXrs:
  case AArch64::SUBSXrx:
  case AArch64::ADDSWrr:
  case AArch64::ADDSWrs:
  case AArch64::ADDSWrx:
  case AArch64::ADDSXrr:
  case AArch64::ADDSXrs:
  case AArch64::ADDSXrx:
    // Replace SUBSWrr with SUBWrr if NZCV is not used.
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = MI.getOperand(2).getReg();
 
    // FIXME: Pass subregisters out of analyzeCompare
    if (MI.getOperand(2).getSubReg())
      return false;
 
    CmpMask = ~0;
    CmpValue = 0;
    return true;
  case AArch64::SUBSWri:
  case AArch64::ADDSWri:
  case AArch64::SUBSXri:
  case AArch64::ADDSXri:
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = 0;
    CmpMask = ~0;
    CmpValue = MI.getOperand(2).getImm();
    return true;
  case AArch64::ANDSWri:
  case AArch64::ANDSXri:
    // ANDS does not use the same encoding scheme as the others xxxS
    // instructions.
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = 0;
    CmpMask = ~0;
    CmpValue = AArch64_AM::decodeLogicalImmediate(
                   MI.getOperand(2).getImm(),
                   MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
    return true;
  }
 
  return false;
}
 
static bool UpdateOperandRegClass(MachineInstr &Instr) {
  MachineBasicBlock *MBB = Instr.getParent();
  assert(MBB && "Can't get MachineBasicBlock here");
  MachineFunction *MF = MBB->getParent();
  assert(MF && "Can't get MachineFunction here");
  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
  MachineRegisterInfo *MRI = &MF->getRegInfo();
 
  for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
       ++OpIdx) {
    MachineOperand &MO = Instr.getOperand(OpIdx);
    const TargetRegisterClass *OpRegCstraints =
        Instr.getRegClassConstraint(OpIdx, TII, TRI);
 
    // If there's no constraint, there's nothing to do.
    if (!OpRegCstraints)
      continue;
    // If the operand is a frame index, there's nothing to do here.
    // A frame index operand will resolve correctly during PEI.
    if (MO.isFI())
      continue;
 
    assert(MO.isReg() &&
           "Operand has register constraints without being a register!");
 
    Register Reg = MO.getReg();
    if (Reg.isPhysical()) {
      if (!OpRegCstraints->contains(Reg))
        return false;
    } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
               !MRI->constrainRegClass(Reg, OpRegCstraints))
      return false;
  }
 
  return true;
}
 
/// Return the opcode that does not set flags when possible - otherwise
/// return the original opcode. The caller is responsible to do the actual
/// substitution and legality checking.
static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
  // Don't convert all compare instructions, because for some the zero register
  // encoding becomes the sp register.
  bool MIDefinesZeroReg = false;
  if (MI.definesRegister(AArch64::WZR, /*TRI=*/nullptr) ||
      MI.definesRegister(AArch64::XZR, /*TRI=*/nullptr))
    MIDefinesZeroReg = true;
 
  switch (MI.getOpcode()) {
  default:
    return MI.getOpcode();
  case AArch64::ADDSWrr:
    return AArch64::ADDWrr;
  case AArch64::ADDSWri:
    return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
  case AArch64::ADDSWrs:
    return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
  case AArch64::ADDSWrx:
    return AArch64::ADDWrx;
  case AArch64::ADDSXrr:
    return AArch64::ADDXrr;
  case AArch64::ADDSXri:
    return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
  case AArch64::ADDSXrs:
    return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
  case AArch64::ADDSXrx:
    return AArch64::ADDXrx;
  case AArch64::SUBSWrr:
    return AArch64::SUBWrr;
  case AArch64::SUBSWri:
    return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
  case AArch64::SUBSWrs:
    return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
  case AArch64::SUBSWrx:
    return AArch64::SUBWrx;
  case AArch64::SUBSXrr:
    return AArch64::SUBXrr;
  case AArch64::SUBSXri:
    return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
  case AArch64::SUBSXrs:
    return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
  case AArch64::SUBSXrx:
    return AArch64::SUBXrx;
  }
}
 
enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
 
/// True when condition flags are accessed (either by writing or reading)
/// on the instruction trace starting at From and ending at To.
///
/// Note: If From and To are from different blocks it's assumed CC are accessed
///       on the path.
static bool areCFlagsAccessedBetweenInstrs(
    MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
    const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
  // Early exit if To is at the beginning of the BB.
  if (To == To->getParent()->begin())
    return true;
 
  // Check whether the instructions are in the same basic block
  // If not, assume the condition flags might get modified somewhere.
  if (To->getParent() != From->getParent())
    return true;
 
  // From must be above To.
  assert(std::any_of(
      ++To.getReverse(), To->getParent()->rend(),
      [From](MachineInstr &MI) { return MI.getIterator() == From; }));
 
  // We iterate backward starting at \p To until we hit \p From.
  for (const MachineInstr &Instr :
       instructionsWithoutDebug(++To.getReverse(), From.getReverse())) {
    if (((AccessToCheck & AK_Write) &&
         Instr.modifiesRegister(AArch64::NZCV, TRI)) ||
        ((AccessToCheck & AK_Read) && Instr.readsRegister(AArch64::NZCV, TRI)))
      return true;
  }
  return false;
}
 
std::optional<unsigned>
AArch64InstrInfo::canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
                                      MachineInstr *Pred,
                                      const MachineRegisterInfo *MRI) const {
  unsigned MaskOpcode = Mask->getOpcode();
  unsigned PredOpcode = Pred->getOpcode();
  bool PredIsPTestLike = isPTestLikeOpcode(PredOpcode);
  bool PredIsWhileLike = isWhileOpcode(PredOpcode);
 
  if (PredIsWhileLike) {
    // For PTEST(PG, PG), PTEST is redundant when PG is the result of a WHILEcc
    // instruction and the condition is "any" since WHILcc does an implicit
    // PTEST(ALL, PG) check and PG is always a subset of ALL.
    if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
      return PredOpcode;
 
    // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
    // redundant since WHILE performs an implicit PTEST with an all active
    // mask.
    if (isPTrueOpcode(MaskOpcode) && Mask->getOperand(1).getImm() == 31 &&
        getElementSizeForOpcode(MaskOpcode) ==
            getElementSizeForOpcode(PredOpcode))
      return PredOpcode;
 
    // For PTEST_FIRST(PTRUE_ALL, WHILE), the PTEST_FIRST is redundant since
    // WHILEcc performs an implicit PTEST with an all active mask, setting
    // the N flag as the PTEST_FIRST would.
    if (PTest->getOpcode() == AArch64::PTEST_PP_FIRST &&
        isPTrueOpcode(MaskOpcode) && Mask->getOperand(1).getImm() == 31)
      return PredOpcode;
 
    return {};
  }
 
  if (PredIsPTestLike) {
    // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
    // instruction that sets the flags as PTEST would and the condition is
    // "any" since PG is always a subset of the governing predicate of the
    // ptest-like instruction.
    if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
      return PredOpcode;
 
    auto PTestLikeMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
 
    // If the PTEST like instruction's general predicate is not `Mask`, attempt
    // to look through a copy and try again. This is because some instructions
    // take a predicate whose register class is a subset of its result class.
    if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
        PTestLikeMask->getOperand(1).getReg().isVirtual())
      PTestLikeMask =
          MRI->getUniqueVRegDef(PTestLikeMask->getOperand(1).getReg());
 
    // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
    // the element size matches and either the PTEST_LIKE instruction uses
    // the same all active mask or the condition is "any".
    if (isPTrueOpcode(MaskOpcode) && Mask->getOperand(1).getImm() == 31 &&
        getElementSizeForOpcode(MaskOpcode) ==
            getElementSizeForOpcode(PredOpcode)) {
      if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY)
        return PredOpcode;
    }
 
    // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
    // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
    // on 8-bit predicates like the PTEST.  Otherwise, for instructions like
    // compare that also support 16/32/64-bit predicates, the implicit PTEST
    // performed by the compare could consider fewer lanes for these element
    // sizes.
    //
    // For example, consider
    //
    //   ptrue p0.b                    ; P0=1111-1111-1111-1111
    //   index z0.s, #0, #1            ; Z0=<0,1,2,3>
    //   index z1.s, #1, #1            ; Z1=<1,2,3,4>
    //   cmphi p1.s, p0/z, z1.s, z0.s  ; P1=0001-0001-0001-0001
    //                                 ;       ^ last active
    //   ptest p0, p1.b                ; P1=0001-0001-0001-0001
    //                                 ;     ^ last active
    //
    // where the compare generates a canonical all active 32-bit predicate
    // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
    // active flag, whereas the PTEST instruction with the same mask doesn't.
    // For PTEST_ANY this doesn't apply as the flags in this case would be
    // identical regardless of element size.
    uint64_t PredElementSize = getElementSizeForOpcode(PredOpcode);
    if (Mask == PTestLikeMask && (PredElementSize == AArch64::ElementSizeB ||
                                  PTest->getOpcode() == AArch64::PTEST_PP_ANY))
      return PredOpcode;
 
    return {};
  }
 
  // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
  // opcode so the PTEST becomes redundant.
  switch (PredOpcode) {
  case AArch64::AND_PPzPP:
  case AArch64::BIC_PPzPP:
  case AArch64::EOR_PPzPP:
  case AArch64::NAND_PPzPP:
  case AArch64::NOR_PPzPP:
  case AArch64::ORN_PPzPP:
  case AArch64::ORR_PPzPP:
  case AArch64::BRKA_PPzP:
  case AArch64::BRKPA_PPzPP:
  case AArch64::BRKB_PPzP:
  case AArch64::BRKPB_PPzPP:
  case AArch64::RDFFR_PPz: {
    // Check to see if our mask is the same. If not the resulting flag bits
    // may be different and we can't remove the ptest.
    auto *PredMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
    if (Mask != PredMask)
      return {};
    break;
  }
  case AArch64::BRKN_PPzP: {
    // BRKN uses an all active implicit mask to set flags unlike the other
    // flag-setting instructions.
    // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
    if ((MaskOpcode != AArch64::PTRUE_B) ||
        (Mask->getOperand(1).getImm() != 31))
      return {};
    break;
  }
  case AArch64::PTRUE_B:
    // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
    break;
  default:
    // Bail out if we don't recognize the input
    return {};
  }
 
  return convertToFlagSettingOpc(PredOpcode);
}
 
/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
/// operation which could set the flags in an identical manner
bool AArch64InstrInfo::optimizePTestInstr(
    MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
    const MachineRegisterInfo *MRI) const {
  auto *Mask = MRI->getUniqueVRegDef(MaskReg);
  auto *Pred = MRI->getUniqueVRegDef(PredReg);
 
  if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) {
    // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies
    // before the branch to extract each subregister.
    auto Op = Pred->getOperand(1);
    if (Op.isReg() && Op.getReg().isVirtual() &&
        Op.getSubReg() == AArch64::psub0)
      Pred = MRI->getUniqueVRegDef(Op.getReg());
  }
 
  unsigned PredOpcode = Pred->getOpcode();
  auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI);
  if (!NewOp)
    return false;
 
  const TargetRegisterInfo *TRI = &getRegisterInfo();
 
  // If another instruction between Pred and PTest accesses flags, don't remove
  // the ptest or update the earlier instruction to modify them.
  if (areCFlagsAccessedBetweenInstrs(Pred, PTest, TRI))
    return false;
 
  // If we pass all the checks, it's safe to remove the PTEST and use the flags
  // as they are prior to PTEST. Sometimes this requires the tested PTEST
  // operand to be replaced with an equivalent instruction that also sets the
  // flags.
  PTest->eraseFromParent();
  if (*NewOp != PredOpcode) {
    Pred->setDesc(get(*NewOp));
    bool succeeded = UpdateOperandRegClass(*Pred);
    (void)succeeded;
    assert(succeeded && "Operands have incompatible register classes!");
    Pred->addRegisterDefined(AArch64::NZCV, TRI);
  }
 
  // Ensure that the flags def is live.
  if (Pred->registerDefIsDead(AArch64::NZCV, TRI)) {
    unsigned i = 0, e = Pred->getNumOperands();
    for (; i != e; ++i) {
      MachineOperand &MO = Pred->getOperand(i);
      if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
        MO.setIsDead(false);
        break;
      }
    }
  }
  return true;
}
 
/// Try to optimize a compare instruction. A compare instruction is an
/// instruction which produces AArch64::NZCV. It can be truly compare
/// instruction
/// when there are no uses of its destination register.
///
/// The following steps are tried in order:
/// 1. Convert CmpInstr into an unconditional version.
/// 2. Remove CmpInstr if above there is an instruction producing a needed
///    condition code or an instruction which can be converted into such an
///    instruction.
///    Only comparison with zero is supported.
bool AArch64InstrInfo::optimizeCompareInstr(
    MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
    int64_t CmpValue, const MachineRegisterInfo *MRI) const {
  assert(CmpInstr.getParent());
  assert(MRI);
 
  // Replace SUBSWrr with SUBWrr if NZCV is not used.
  int DeadNZCVIdx =
      CmpInstr.findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, true);
  if (DeadNZCVIdx != -1) {
    if (CmpInstr.definesRegister(AArch64::WZR, /*TRI=*/nullptr) ||
        CmpInstr.definesRegister(AArch64::XZR, /*TRI=*/nullptr)) {
      CmpInstr.eraseFromParent();
      return true;
    }
    unsigned Opc = CmpInstr.getOpcode();
    unsigned NewOpc = convertToNonFlagSettingOpc(CmpInstr);
    if (NewOpc == Opc)
      return false;
    const MCInstrDesc &MCID = get(NewOpc);
    CmpInstr.setDesc(MCID);
    CmpInstr.removeOperand(DeadNZCVIdx);
    bool succeeded = UpdateOperandRegClass(CmpInstr);
    (void)succeeded;
    assert(succeeded && "Some operands reg class are incompatible!");
    return true;
  }
 
  if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
      CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY ||
      CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST)
    return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2, MRI);
 
  if (SrcReg2 != 0)
    return false;
 
  // CmpInstr is a Compare instruction if destination register is not used.
  if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
    return false;
 
  if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *MRI))
    return true;
  return (CmpValue == 0 || CmpValue == 1) &&
         removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *MRI);
}
 
/// Get opcode of S version of Instr.
/// If Instr is S version its opcode is returned.
/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
/// or we are not interested in it.
static unsigned sForm(MachineInstr &Instr) {
  switch (Instr.getOpcode()) {
  default:
    return AArch64::INSTRUCTION_LIST_END;
 
  case AArch64::ADDSWrr:
  case AArch64::ADDSWri:
  case AArch64::ADDSXrr:
  case AArch64::ADDSXri:
  case AArch64::ADDSWrx:
  case AArch64::ADDSXrx:
  case AArch64::ADDSWrs:
  case AArch64::ADDSXrs:
  case AArch64::SUBSWrr:
  case AArch64::SUBSWri:
  case AArch64::SUBSWrx:
  case AArch64::SUBSWrs:
  case AArch64::SUBSXrr:
  case AArch64::SUBSXri:
  case AArch64::SUBSXrx:
  case AArch64::SUBSXrs:
  case AArch64::ANDSWri:
  case AArch64::ANDSWrr:
  case AArch64::ANDSWrs:
  case AArch64::ANDSXri:
  case AArch64::ANDSXrr:
  case AArch64::ANDSXrs:
  case AArch64::BICSWrr:
  case AArch64::BICSXrr:
  case AArch64::BICSWrs:
  case AArch64::BICSXrs:
  case AArch64::ADCSWr:
  case AArch64::ADCSXr:
  case AArch64::SBCSWr:
  case AArch64::SBCSXr:
    return Instr.getOpcode();
 
  case AArch64::ADDWrr:
    return AArch64::ADDSWrr;
  case AArch64::ADDWri:
    return AArch64::ADDSWri;
  case AArch64::ADDXrr:
    return AArch64::ADDSXrr;
  case AArch64::ADDXri:
    return AArch64::ADDSXri;
  case AArch64::ADDWrx:
    return AArch64::ADDSWrx;
  case AArch64::ADDXrx:
    return AArch64::ADDSXrx;
  case AArch64::ADDWrs:
    return AArch64::ADDSWrs;
  case AArch64::ADDXrs:
    return AArch64::ADDSXrs;
  case AArch64::ADCWr:
    return AArch64::ADCSWr;
  case AArch64::ADCXr:
    return AArch64::ADCSXr;
  case AArch64::SUBWrr:
    return AArch64::SUBSWrr;
  case AArch64::SUBWri:
    return AArch64::SUBSWri;
  case AArch64::SUBXrr:
    return AArch64::SUBSXrr;
  case AArch64::SUBXri:
    return AArch64::SUBSXri;
  case AArch64::SUBWrx:
    return AArch64::SUBSWrx;
  case AArch64::SUBXrx:
    return AArch64::SUBSXrx;
  case AArch64::SUBWrs:
    return AArch64::SUBSWrs;
  case AArch64::SUBXrs:
    return AArch64::SUBSXrs;
  case AArch64::SBCWr:
    return AArch64::SBCSWr;
  case AArch64::SBCXr:
    return AArch64::SBCSXr;
  case AArch64::ANDWri:
    return AArch64::ANDSWri;
  case AArch64::ANDXri:
    return AArch64::ANDSXri;
  case AArch64::ANDWrr:
    return AArch64::ANDSWrr;
  case AArch64::ANDWrs:
    return AArch64::ANDSWrs;
  case AArch64::ANDXrr:
    return AArch64::ANDSXrr;
  case AArch64::ANDXrs:
    return AArch64::ANDSXrs;
  case AArch64::BICWrr:
    return AArch64::BICSWrr;
  case AArch64::BICXrr:
    return AArch64::BICSXrr;
  case AArch64::BICWrs:
    return AArch64::BICSWrs;
  case AArch64::BICXrs:
    return AArch64::BICSXrs;
  }
}
 
/// Check if AArch64::NZCV should be alive in successors of MBB.
static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB) {
  for (auto *BB : MBB->successors())
    if (BB->isLiveIn(AArch64::NZCV))
      return true;
  return false;
}
 
/// \returns The condition code operand index for \p Instr if it is a branch
/// or select and -1 otherwise.
int AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(
    const MachineInstr &Instr) {
  switch (Instr.getOpcode()) {
  default:
    return -1;
 
  case AArch64::Bcc: {
    int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV, /*TRI=*/nullptr);
    assert(Idx >= 2);
    return Idx - 2;
  }
 
  case AArch64::CSINVWr:
  case AArch64::CSINVXr:
  case AArch64::CSINCWr:
  case AArch64::CSINCXr:
  case AArch64::CSELWr:
  case AArch64::CSELXr:
  case AArch64::CSNEGWr:
  case AArch64::CSNEGXr:
  case AArch64::FCSELSrrr:
  case AArch64::FCSELDrrr: {
    int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV, /*TRI=*/nullptr);
    assert(Idx >= 1);
    return Idx - 1;
  }
  }
}
 
/// Find a condition code used by the instruction.
/// Returns AArch64CC::Invalid if either the instruction does not use condition
/// codes or we don't optimize CmpInstr in the presence of such instructions.
static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
  int CCIdx =
      AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(Instr);
  return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
                          Instr.getOperand(CCIdx).getImm())
                    : AArch64CC::Invalid;
}
 
static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
  assert(CC != AArch64CC::Invalid);
  UsedNZCV UsedFlags;
  switch (CC) {
  default:
    break;
 
  case AArch64CC::EQ: // Z set
  case AArch64CC::NE: // Z clear
    UsedFlags.Z = true;
    break;
 
  case AArch64CC::HI: // Z clear and C set
  case AArch64CC::LS: // Z set   or  C clear
    UsedFlags.Z = true;
    [[fallthrough]];
  case AArch64CC::HS: // C set
  case AArch64CC::LO: // C clear
    UsedFlags.C = true;
    break;
 
  case AArch64CC::MI: // N set
  case AArch64CC::PL: // N clear
    UsedFlags.N = true;
    break;
 
  case AArch64CC::VS: // V set
  case AArch64CC::VC: // V clear
    UsedFlags.V = true;
    break;
 
  case AArch64CC::GT: // Z clear, N and V the same
  case AArch64CC::LE: // Z set,   N and V differ
    UsedFlags.Z = true;
    [[fallthrough]];
  case AArch64CC::GE: // N and V the same
  case AArch64CC::LT: // N and V differ
    UsedFlags.N = true;
    UsedFlags.V = true;
    break;
  }
  return UsedFlags;
}
 
/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
/// \returns std::nullopt otherwise.
///
/// Collect instructions using that flags in \p CCUseInstrs if provided.
std::optional<UsedNZCV>
llvm::examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
                       const TargetRegisterInfo &TRI,
                       SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
  MachineBasicBlock *CmpParent = CmpInstr.getParent();
  if (MI.getParent() != CmpParent)
    return std::nullopt;
 
  if (areCFlagsAliveInSuccessors(CmpParent))
    return std::nullopt;
 
  UsedNZCV NZCVUsedAfterCmp;
  for (MachineInstr &Instr : instructionsWithoutDebug(
           std::next(CmpInstr.getIterator()), CmpParent->instr_end())) {
    if (Instr.readsRegister(AArch64::NZCV, &TRI)) {
      AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
      if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
        return std::nullopt;
      NZCVUsedAfterCmp |= getUsedNZCV(CC);
      if (CCUseInstrs)
        CCUseInstrs->push_back(&Instr);
    }
    if (Instr.modifiesRegister(AArch64::NZCV, &TRI))
      break;
  }
  return NZCVUsedAfterCmp;
}
 
static bool isADDSRegImm(unsigned Opcode) {
  return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
}
 
static bool isSUBSRegImm(unsigned Opcode) {
  return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
}
 
static bool isANDOpcode(MachineInstr &MI) {
  unsigned Opc = sForm(MI);
  switch (Opc) {
  case AArch64::ANDSWri:
  case AArch64::ANDSWrr:
  case AArch64::ANDSWrs:
  case AArch64::ANDSXri:
  case AArch64::ANDSXrr:
  case AArch64::ANDSXrs:
  case AArch64::BICSWrr:
  case AArch64::BICSXrr:
  case AArch64::BICSWrs:
  case AArch64::BICSXrs:
    return true;
  default:
    return false;
  }
}
 
/// Check if CmpInstr can be substituted by MI.
///
/// CmpInstr can be substituted:
/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
/// - and, MI and CmpInstr are from the same MachineBB
/// - and, condition flags are not alive in successors of the CmpInstr parent
/// - and, if MI opcode is the S form there must be no defs of flags between
///        MI and CmpInstr
///        or if MI opcode is not the S form there must be neither defs of flags
///        nor uses of flags between MI and CmpInstr.
/// - and, C is not used after CmpInstr; CmpInstr's C is from adds/subs #0 on
///        SrcReg and can differ from MI (e.g. carry out of ADCS/SBCS).
/// - and, V is not used after CmpInstr unless MI is AND/BIC (V cleared) or MI
///        has NoSWrap (overflow is poison and the fold is still safe).
static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr,
                                       const TargetRegisterInfo &TRI) {
  // MI is an opcode sForm maps (add/sub/adc/sbc/and/bic and their S forms).
  assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
 
  const unsigned CmpOpcode = CmpInstr.getOpcode();
  if (!isADDSRegImm(CmpOpcode) && !isSUBSRegImm(CmpOpcode))
    return false;
 
  assert((CmpInstr.getOperand(2).isImm() &&
          CmpInstr.getOperand(2).getImm() == 0) &&
         "Caller guarantees that CmpInstr compares with constant 0");
 
  std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
  if (!NZVCUsed || NZVCUsed->C)
    return false;
 
  // CmpInstr is ADDS/SUBS with immediate 0 on SrcReg (compare SrcReg to zero).
  // After the fold, users see NZCV from MI (or its S form), not from CmpInstr.
  // N/Z match CmpInstr for the value in SrcReg; C/V need not match in general
  // (e.g. ADCS vs adds #0), so we require C unused after CmpInstr and gate V
  // as below. NoSWrap makes signed overflow poison; AND/BIC clear V.
  if (NZVCUsed->V && !MI.getFlag(MachineInstr::NoSWrap) && !isANDOpcode(MI))
    return false;
 
  AccessKind AccessToCheck = AK_Write;
  if (sForm(MI) != MI.getOpcode())
    AccessToCheck = AK_All;
  return !areCFlagsAccessedBetweenInstrs(&MI, &CmpInstr, &TRI, AccessToCheck);
}
 
/// Substitute an instruction comparing to zero with another instruction
/// which produces needed condition flags.
///
/// Return true on success.
bool AArch64InstrInfo::substituteCmpToZero(
    MachineInstr &CmpInstr, unsigned SrcReg,
    const MachineRegisterInfo &MRI) const {
  // Get the unique definition of SrcReg.
  MachineInstr *MI = MRI.getUniqueVRegDef(SrcReg);
  if (!MI)
    return false;
 
  const TargetRegisterInfo &TRI = getRegisterInfo();
 
  unsigned NewOpc = sForm(*MI);
  if (NewOpc == AArch64::INSTRUCTION_LIST_END)
    return false;
 
  if (!canInstrSubstituteCmpInstr(*MI, CmpInstr, TRI))
    return false;
 
  // Update the instruction to set NZCV.
  MI->setDesc(get(NewOpc));
  CmpInstr.eraseFromParent();
  bool succeeded = UpdateOperandRegClass(*MI);
  (void)succeeded;
  assert(succeeded && "Some operands reg class are incompatible!");
  MI->addRegisterDefined(AArch64::NZCV, &TRI);
  return true;
}
 
/// \returns True if \p CmpInstr can be removed.
///
/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
/// codes used in \p CCUseInstrs must be inverted.
static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr,
                                 int CmpValue, const TargetRegisterInfo &TRI,
                                 SmallVectorImpl<MachineInstr *> &CCUseInstrs,
                                 bool &IsInvertCC) {
  assert((CmpValue == 0 || CmpValue == 1) &&
         "Only comparisons to 0 or 1 considered for removal!");
 
  // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
  unsigned MIOpc = MI.getOpcode();
  if (MIOpc == AArch64::CSINCWr) {
    if (MI.getOperand(1).getReg() != AArch64::WZR ||
        MI.getOperand(2).getReg() != AArch64::WZR)
      return false;
  } else if (MIOpc == AArch64::CSINCXr) {
    if (MI.getOperand(1).getReg() != AArch64::XZR ||
        MI.getOperand(2).getReg() != AArch64::XZR)
      return false;
  } else {
    return false;
  }
  AArch64CC::CondCode MICC = findCondCodeUsedByInstr(MI);
  if (MICC == AArch64CC::Invalid)
    return false;
 
  // NZCV needs to be defined
  if (MI.findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, true) != -1)
    return false;
 
  // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
  const unsigned CmpOpcode = CmpInstr.getOpcode();
  bool IsSubsRegImm = isSUBSRegImm(CmpOpcode);
  if (CmpValue && !IsSubsRegImm)
    return false;
  if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(CmpOpcode))
    return false;
 
  // MI conditions allowed: eq, ne, mi, pl
  UsedNZCV MIUsedNZCV = getUsedNZCV(MICC);
  if (MIUsedNZCV.C || MIUsedNZCV.V)
    return false;
 
  std::optional<UsedNZCV> NZCVUsedAfterCmp =
      examineCFlagsUse(MI, CmpInstr, TRI, &CCUseInstrs);
  // Condition flags are not used in CmpInstr basic block successors and only
  // Z or N flags allowed to be used after CmpInstr within its basic block
  if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
    return false;
  // Z or N flag used after CmpInstr must correspond to the flag used in MI
  if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
      (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
    return false;
  // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
  if (MIUsedNZCV.N && !CmpValue)
    return false;
 
  // There must be no defs of flags between MI and CmpInstr
  if (areCFlagsAccessedBetweenInstrs(&MI, &CmpInstr, &TRI, AK_Write))
    return false;
 
  // Condition code is inverted in the following cases:
  // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
  // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
  IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
               (!CmpValue && MICC == AArch64CC::NE);
  return true;
}
 
/// Remove comparison in csinc-cmp sequence
///
/// Examples:
/// 1. \code
///   csinc w9, wzr, wzr, ne
///   cmp   w9, #0
///   b.eq
///    \endcode
/// to
///    \code
///   csinc w9, wzr, wzr, ne
///   b.ne
///    \endcode
///
/// 2. \code
///   csinc x2, xzr, xzr, mi
///   cmp   x2, #1
///   b.pl
///    \endcode
/// to
///    \code
///   csinc x2, xzr, xzr, mi
///   b.pl
///    \endcode
///
/// \param  CmpInstr comparison instruction
/// \return True when comparison removed
bool AArch64InstrInfo::removeCmpToZeroOrOne(
    MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
    const MachineRegisterInfo &MRI) const {
  MachineInstr *MI = MRI.getUniqueVRegDef(SrcReg);
  if (!MI)
    return false;
  const TargetRegisterInfo &TRI = getRegisterInfo();
  SmallVector<MachineInstr *, 4> CCUseInstrs;
  bool IsInvertCC = false;
  if (!canCmpInstrBeRemoved(*MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
                            IsInvertCC))
    return false;
  // Make transformation
  CmpInstr.eraseFromParent();
  if (IsInvertCC) {
    // Invert condition codes in CmpInstr CC users
    for (MachineInstr *CCUseInstr : CCUseInstrs) {
      int Idx = findCondCodeUseOperandIdxForBranchOrSelect(*CCUseInstr);
      assert(Idx >= 0 && "Unexpected instruction using CC.");
      MachineOperand &CCOperand = CCUseInstr->getOperand(Idx);
      AArch64CC::CondCode CCUse = AArch64CC::getInvertedCondCode(
          static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
      CCOperand.setImm(CCUse);
    }
  }
  return true;
}
 
bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
  if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
      MI.getOpcode() != AArch64::CATCHRET &&
      MI.getOpcode() != AArch64::STACK_GUARD_UNMIX)
    return false;
 
  MachineBasicBlock &MBB = *MI.getParent();
  auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
  auto TRI = Subtarget.getRegisterInfo();
  DebugLoc DL = MI.getDebugLoc();
 
  if (MI.getOpcode() == AArch64::STACK_GUARD_UNMIX) {
    // Expand STACK_GUARD_UNMIX to: sub Rd, fp, Rs
    // This computes FP - stored_mixed_value to unmix the cookie
    Register DstReg = MI.getOperand(0).getReg();
    Register SrcReg = MI.getOperand(1).getReg();
 
    BuildMI(MBB, MI, DL, get(AArch64::SUBXrr), DstReg)
        .addReg(AArch64::FP)
        .addReg(SrcReg);
 
    MBB.erase(MI);
    return true;
  }
 
  if (MI.getOpcode() == AArch64::CATCHRET) {
    // Skip to the first instruction before the epilog.
    const TargetInstrInfo *TII =
      MBB.getParent()->getSubtarget().getInstrInfo();
    MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
    auto MBBI = MachineBasicBlock::iterator(MI);
    MachineBasicBlock::iterator FirstEpilogSEH = std::prev(MBBI);
    while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) &&
           FirstEpilogSEH != MBB.begin())
      FirstEpilogSEH = std::prev(FirstEpilogSEH);
    if (FirstEpilogSEH != MBB.begin())
      FirstEpilogSEH = std::next(FirstEpilogSEH);
    BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP))
        .addReg(AArch64::X0, RegState::Define)
        .addMBB(TargetMBB);
    BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri))
        .addReg(AArch64::X0, RegState::Define)
        .addReg(AArch64::X0)
        .addMBB(TargetMBB)
        .addImm(0);
    TargetMBB->setMachineBlockAddressTaken();
    return true;
  }
 
  Register Reg = MI.getOperand(0).getReg();
  Module &M = *MBB.getParent()->getFunction().getParent();
  if (M.getStackProtectorGuard() == "sysreg") {
    const AArch64SysReg::SysReg *SrcReg =
        AArch64SysReg::lookupSysRegByName(M.getStackProtectorGuardReg());
    if (!SrcReg)
      report_fatal_error("Unknown SysReg for Stack Protector Guard Register");
 
    // mrs xN, sysreg
    BuildMI(MBB, MI, DL, get(AArch64::MRS))
        .addDef(Reg, RegState::Renamable)
        .addImm(SrcReg->Encoding);
    int Offset = M.getStackProtectorGuardOffset();
    if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
      // ldr xN, [xN, #offset]
      BuildMI(MBB, MI, DL, get(AArch64::LDRXui))
          .addDef(Reg)
          .addUse(Reg, RegState::Kill)
          .addImm(Offset / 8);
    } else if (Offset >= -256 && Offset <= 255) {
      // ldur xN, [xN, #offset]
      BuildMI(MBB, MI, DL, get(AArch64::LDURXi))
          .addDef(Reg)
          .addUse(Reg, RegState::Kill)
          .addImm(Offset);
    } else if (Offset >= -4095 && Offset <= 4095) {
      if (Offset > 0) {
        // add xN, xN, #offset
        BuildMI(MBB, MI, DL, get(AArch64::ADDXri))
            .addDef(Reg)
            .addUse(Reg, RegState::Kill)
            .addImm(Offset)
            .addImm(0);
      } else {
        // sub xN, xN, #offset
        BuildMI(MBB, MI, DL, get(AArch64::SUBXri))
            .addDef(Reg)
            .addUse(Reg, RegState::Kill)
            .addImm(-Offset)
            .addImm(0);
      }
      // ldr xN, [xN]
      BuildMI(MBB, MI, DL, get(AArch64::LDRXui))
          .addDef(Reg)
          .addUse(Reg, RegState::Kill)
          .addImm(0);
    } else {
      // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
      // than 23760.
      // It might be nice to use AArch64::MOVi32imm here, which would get
      // expanded in PreSched2 after PostRA, but our lone scratch Reg already
      // contains the MRS result. findScratchNonCalleeSaveRegister() in
      // AArch64FrameLowering might help us find such a scratch register
      // though. If we failed to find a scratch register, we could emit a
      // stream of add instructions to build up the immediate. Or, we could try
      // to insert a AArch64::MOVi32imm before register allocation so that we
      // didn't need to scavenge for a scratch register.
      report_fatal_error("Unable to encode Stack Protector Guard Offset");
    }
    MBB.erase(MI);
    return true;
  }
 
  const GlobalValue *GV =
      cast<GlobalValue>((*MI.memoperands_begin())->getValue());
  const TargetMachine &TM = MBB.getParent()->getTarget();
  unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
  const unsigned char MO_NC = AArch64II::MO_NC;
 
  unsigned GuardWidth = M.getStackProtectorGuardValueWidth().value_or(
      Subtarget.isTargetILP32() ? 4 : 8);
  if (GuardWidth != 4 && GuardWidth != 8)
    report_fatal_error("Unsupported stack protector value width");
  if ((OpFlags & AArch64II::MO_GOT) != 0) {
    BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
        .addGlobalAddress(GV, 0, OpFlags);
    if (GuardWidth == 4) {
      unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
      BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
          .addDef(Reg32, RegState::Dead)
          .addUse(Reg, RegState::Kill)
          .addImm(0)
          .addMemOperand(*MI.memoperands_begin())
          .addDef(Reg, RegState::Implicit);
    } else {
      BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
          .addReg(Reg, RegState::Kill)
          .addImm(0)
          .addMemOperand(*MI.memoperands_begin());
    }
  } else if (TM.getCodeModel() == CodeModel::Large) {
    BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
        .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC)
        .addImm(0);
    BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
        .addReg(Reg, RegState::Kill)
        .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC)
        .addImm(16);
    BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
        .addReg(Reg, RegState::Kill)
        .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC)
        .addImm(32);
    BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
        .addReg(Reg, RegState::Kill)
        .addGlobalAddress(GV, 0, AArch64II::MO_G3)
        .addImm(48);
    if (GuardWidth == 4) {
      unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
      BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
          .addDef(Reg32, RegState::Dead)
          .addUse(Reg, RegState::Kill)
          .addImm(0)
          .addMemOperand(*MI.memoperands_begin())
          .addDef(Reg, RegState::Implicit);
    } else {
      BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
          .addReg(Reg, RegState::Kill)
          .addImm(0)
          .addMemOperand(*MI.memoperands_begin());
    }
  } else {
    BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
        .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
    unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
    if (GuardWidth == 4) {
      unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
      BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
          .addDef(Reg32, RegState::Dead)
          .addUse(Reg, RegState::Kill)
          .addGlobalAddress(GV, 0, LoFlags)
          .addMemOperand(*MI.memoperands_begin())
          .addDef(Reg, RegState::Implicit);
    } else {
      BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
          .addReg(Reg, RegState::Kill)
          .addGlobalAddress(GV, 0, LoFlags)
          .addMemOperand(*MI.memoperands_begin());
    }
  }
  // To match MSVC. Unlike x86_64 which uses xor instruction to mix the cookie,
  // we use sub instruction to mix the cookie on aarch64.
  // The mixing happens here in expandPostRAPseudo (after RA) to ensure we use
  // the final frame pointer value.
  if (Subtarget.getTargetTriple().isOSMSVCRT())
    BuildMI(MBB, MI, DL, get(AArch64::SUBXrr), Reg)
        .addReg(AArch64::FP)
        .addReg(Reg, RegState::Kill);
 
  MBB.erase(MI);
 
  return true;
}
 
// Return true if this instruction simply sets its single destination register
// to zero. This is equivalent to a register rename of the zero-register.
bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    break;
  case AArch64::MOVZWi:
  case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
    if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) {
      assert(MI.getDesc().getNumOperands() == 3 &&
             MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
      return true;
    }
    break;
  case AArch64::ANDWri: // and Rd, Rzr, #imm
    return MI.getOperand(1).getReg() == AArch64::WZR;
  case AArch64::ANDXri:
    return MI.getOperand(1).getReg() == AArch64::XZR;
  case TargetOpcode::COPY:
    return MI.getOperand(1).getReg() == AArch64::WZR;
  }
  return false;
}
 
// Return true if this instruction simply renames a general register without
// modifying bits.
bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    break;
  case TargetOpcode::COPY: {
    // GPR32 copies will by lowered to ORRXrs
    Register DstReg = MI.getOperand(0).getReg();
    return (AArch64::GPR32RegClass.contains(DstReg) ||
            AArch64::GPR64RegClass.contains(DstReg));
  }
  case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
    if (MI.getOperand(1).getReg() == AArch64::XZR) {
      assert(MI.getDesc().getNumOperands() == 4 &&
             MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
      return true;
    }
    break;
  case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
    if (MI.getOperand(2).getImm() == 0) {
      assert(MI.getDesc().getNumOperands() == 4 &&
             MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
      return true;
    }
    break;
  }
  return false;
}
 
// Return true if this instruction simply renames a general register without
// modifying bits.
bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    break;
  case TargetOpcode::COPY: {
    Register DstReg = MI.getOperand(0).getReg();
    return AArch64::FPR128RegClass.contains(DstReg);
  }
  case AArch64::ORRv16i8:
    if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
      assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
             "invalid ORRv16i8 operands");
      return true;
    }
    break;
  }
  return false;
}
 
static bool isFrameLoadOpcode(int Opcode) {
  switch (Opcode) {
  default:
    return false;
  case AArch64::LDRWui:
  case AArch64::LDRXui:
  case AArch64::LDRBui:
  case AArch64::LDRHui:
  case AArch64::LDRSui:
  case AArch64::LDRDui:
  case AArch64::LDRQui:
  case AArch64::LDR_PXI:
    return true;
  }
}
 
Register AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
                                               int &FrameIndex) const {
  if (!isFrameLoadOpcode(MI.getOpcode()))
    return Register();
 
  if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
      MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
    FrameIndex = MI.getOperand(1).getIndex();
    return MI.getOperand(0).getReg();
  }
  return Register();
}
 
static bool isFrameStoreOpcode(int Opcode) {
  switch (Opcode) {
  default:
    return false;
  case AArch64::STRWui:
  case AArch64::STRXui:
  case AArch64::STRBui:
  case AArch64::STRHui:
  case AArch64::STRSui:
  case AArch64::STRDui:
  case AArch64::STRQui:
  case AArch64::STR_PXI:
    return true;
  }
}
 
Register AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
                                              int &FrameIndex) const {
  if (!isFrameStoreOpcode(MI.getOpcode()))
    return Register();
 
  if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
      MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
    FrameIndex = MI.getOperand(1).getIndex();
    return MI.getOperand(0).getReg();
  }
  return Register();
}
 
Register AArch64InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
                                                    int &FrameIndex) const {
  if (!isFrameStoreOpcode(MI.getOpcode()))
    return Register();
 
  if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
    return Reg;
 
  SmallVector<const MachineMemOperand *, 1> Accesses;
  if (hasStoreToStackSlot(MI, Accesses)) {
    if (Accesses.size() > 1)
      return Register();
 
    FrameIndex =
        cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
            ->getFrameIndex();
    return MI.getOperand(0).getReg();
  }
  return Register();
}
 
Register AArch64InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
                                                     int &FrameIndex) const {
  if (!isFrameLoadOpcode(MI.getOpcode()))
    return Register();
 
  if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
    return Reg;
 
  SmallVector<const MachineMemOperand *, 1> Accesses;
  if (hasLoadFromStackSlot(MI, Accesses)) {
    if (Accesses.size() > 1)
      return Register();
 
    FrameIndex =
        cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
            ->getFrameIndex();
    return MI.getOperand(0).getReg();
  }
  return Register();
}
 
/// Check all MachineMemOperands for a hint to suppress pairing.
bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
  return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
    return MMO->getFlags() & MOSuppressPair;
  });
}
 
/// Set a flag on the first MachineMemOperand to suppress pairing.
void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
  if (MI.memoperands_empty())
    return;
  (*MI.memoperands_begin())->setFlags(MOSuppressPair);
}
 
/// Check all MachineMemOperands for a hint that the load/store is strided.
bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
  return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
    return MMO->getFlags() & MOStridedAccess;
  });
}
 
bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
  switch (Opc) {
  default:
    return false;
  case AArch64::STURSi:
  case AArch64::STRSpre:
  case AArch64::STURDi:
  case AArch64::STRDpre:
  case AArch64::STURQi:
  case AArch64::STRQpre:
  case AArch64::STURBBi:
  case AArch64::STURHHi:
  case AArch64::STURWi:
  case AArch64::STRWpre:
  case AArch64::STURXi:
  case AArch64::STRXpre:
  case AArch64::LDURSi:
  case AArch64::LDRSpre:
  case AArch64::LDURDi:
  case AArch64::LDRDpre:
  case AArch64::LDURQi:
  case AArch64::LDRQpre:
  case AArch64::LDURWi:
  case AArch64::LDRWpre:
  case AArch64::LDURXi:
  case AArch64::LDRXpre:
  case AArch64::LDRSWpre:
  case AArch64::LDURSWi:
  case AArch64::LDURHHi:
  case AArch64::LDURBBi:
  case AArch64::LDURSBWi:
  case AArch64::LDURSHWi:
    return true;
  }
}
 
std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
  switch (Opc) {
  default: return {};
  case AArch64::PRFMui: return AArch64::PRFUMi;
  case AArch64::LDRXui: return AArch64::LDURXi;
  case AArch64::LDRWui: return AArch64::LDURWi;
  case AArch64::LDRBui: return AArch64::LDURBi;
  case AArch64::LDRHui: return AArch64::LDURHi;
  case AArch64::LDRSui: return AArch64::LDURSi;
  case AArch64::LDRDui: return AArch64::LDURDi;
  case AArch64::LDRQui: return AArch64::LDURQi;
  case AArch64::LDRBBui: return AArch64::LDURBBi;
  case AArch64::LDRHHui: return AArch64::LDURHHi;
  case AArch64::LDRSBXui: return AArch64::LDURSBXi;
  case AArch64::LDRSBWui: return AArch64::LDURSBWi;
  case AArch64::LDRSHXui: return AArch64::LDURSHXi;
  case AArch64::LDRSHWui: return AArch64::LDURSHWi;
  case AArch64::LDRSWui: return AArch64::LDURSWi;
  case AArch64::STRXui: return AArch64::STURXi;
  case AArch64::STRWui: return AArch64::STURWi;
  case AArch64::STRBui: return AArch64::STURBi;
  case AArch64::STRHui: return AArch64::STURHi;
  case AArch64::STRSui: return AArch64::STURSi;
  case AArch64::STRDui: return AArch64::STURDi;
  case AArch64::STRQui: return AArch64::STURQi;
  case AArch64::STRBBui: return AArch64::STURBBi;
  case AArch64::STRHHui: return AArch64::STURHHi;
  }
}
 
unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
  switch (Opc) {
  default:
    llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
  case AArch64::ADDG:
  case AArch64::LDAPURBi:
  case AArch64::LDAPURHi:
  case AArch64::LDAPURi:
  case AArch64::LDAPURSBWi:
  case AArch64::LDAPURSBXi:
  case AArch64::LDAPURSHWi:
  case AArch64::LDAPURSHXi:
  case AArch64::LDAPURSWi:
  case AArch64::LDAPURXi:
  case AArch64::LDR_PPXI:
  case AArch64::LDR_PXI:
  case AArch64::LDR_ZXI:
  case AArch64::LDR_ZZXI:
  case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
  case AArch64::LDR_ZZZXI:
  case AArch64::LDR_ZZZZXI:
  case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
  case AArch64::LDRBBui:
  case AArch64::LDRBui:
  case AArch64::LDRDui:
  case AArch64::LDRHHui:
  case AArch64::LDRHui:
  case AArch64::LDRQui:
  case AArch64::LDRSBWui:
  case AArch64::LDRSBXui:
  case AArch64::LDRSHWui:
  case AArch64::LDRSHXui:
  case AArch64::LDRSui:
  case AArch64::LDRSWui:
  case AArch64::LDRWui:
  case AArch64::LDRXui:
  case AArch64::LDURBBi:
  case AArch64::LDURBi:
  case AArch64::LDURDi:
  case AArch64::LDURHHi:
  case AArch64::LDURHi:
  case AArch64::LDURQi:
  case AArch64::LDURSBWi:
  case AArch64::LDURSBXi:
  case AArch64::LDURSHWi:
  case AArch64::LDURSHXi:
  case AArch64::LDURSi:
  case AArch64::LDURSWi:
  case AArch64::LDURWi:
  case AArch64::LDURXi:
  case AArch64::PRFMui:
  case AArch64::PRFUMi:
  case AArch64::ST2Gi:
  case AArch64::STGi:
  case AArch64::STLURBi:
  case AArch64::STLURHi:
  case AArch64::STLURWi:
  case AArch64::STLURXi:
  case AArch64::StoreSwiftAsyncContext:
  case AArch64::STR_PPXI:
  case AArch64::STR_PXI:
  case AArch64::STR_ZXI:
  case AArch64::STR_ZZXI:
  case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
  case AArch64::STR_ZZZXI:
  case AArch64::STR_ZZZZXI:
  case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
  case AArch64::STRBBui:
  case AArch64::STRBui:
  case AArch64::STRDui:
  case AArch64::STRHHui:
  case AArch64::STRHui:
  case AArch64::STRQui:
  case AArch64::STRSui:
  case AArch64::STRWui:
  case AArch64::STRXui:
  case AArch64::STURBBi:
  case AArch64::STURBi:
  case AArch64::STURDi:
  case AArch64::STURHHi:
  case AArch64::STURHi:
  case AArch64::STURQi:
  case AArch64::STURSi:
  case AArch64::STURWi:
  case AArch64::STURXi:
  case AArch64::STZ2Gi:
  case AArch64::STZGi:
  case AArch64::TAGPstack:
    return 2;
  case AArch64::LD1B_D_IMM:
  case AArch64::LD1B_H_IMM:
  case AArch64::LD1B_IMM:
  case AArch64::LD1B_S_IMM:
  case AArch64::LD1D_IMM:
  case AArch64::LD1H_D_IMM:
  case AArch64::LD1H_IMM:
  case AArch64::LD1H_S_IMM:
  case AArch64::LD1RB_D_IMM:
  case AArch64::LD1RB_H_IMM:
  case AArch64::LD1RB_IMM:
  case AArch64::LD1RB_S_IMM:
  case AArch64::LD1RD_IMM:
  case AArch64::LD1RH_D_IMM:
  case AArch64::LD1RH_IMM:
  case AArch64::LD1RH_S_IMM:
  case AArch64::LD1RSB_D_IMM:
  case AArch64::LD1RSB_H_IMM:
  case AArch64::LD1RSB_S_IMM:
  case AArch64::LD1RSH_D_IMM:
  case AArch64::LD1RSH_S_IMM:
  case AArch64::LD1RSW_IMM:
  case AArch64::LD1RW_D_IMM:
  case AArch64::LD1RW_IMM:
  case AArch64::LD1SB_D_IMM:
  case AArch64::LD1SB_H_IMM:
  case AArch64::LD1SB_S_IMM:
  case AArch64::LD1SH_D_IMM:
  case AArch64::LD1SH_S_IMM:
  case AArch64::LD1SW_D_IMM:
  case AArch64::LD1W_D_IMM:
  case AArch64::LD1W_IMM:
  case AArch64::LD2B_IMM:
  case AArch64::LD2D_IMM:
  case AArch64::LD2H_IMM:
  case AArch64::LD2W_IMM:
  case AArch64::LD3B_IMM:
  case AArch64::LD3D_IMM:
  case AArch64::LD3H_IMM:
  case AArch64::LD3W_IMM:
  case AArch64::LD4B_IMM:
  case AArch64::LD4D_IMM:
  case AArch64::LD4H_IMM:
  case AArch64::LD4W_IMM:
  case AArch64::LDG:
  case AArch64::LDNF1B_D_IMM:
  case AArch64::LDNF1B_H_IMM:
  case AArch64::LDNF1B_IMM:
  case AArch64::LDNF1B_S_IMM:
  case AArch64::LDNF1D_IMM:
  case AArch64::LDNF1H_D_IMM:
  case AArch64::LDNF1H_IMM:
  case AArch64::LDNF1H_S_IMM:
  case AArch64::LDNF1SB_D_IMM:
  case AArch64::LDNF1SB_H_IMM:
  case AArch64::LDNF1SB_S_IMM:
  case AArch64::LDNF1SH_D_IMM:
  case AArch64::LDNF1SH_S_IMM:
  case AArch64::LDNF1SW_D_IMM:
  case AArch64::LDNF1W_D_IMM:
  case AArch64::LDNF1W_IMM:
  case AArch64::LDNPDi:
  case AArch64::LDNPQi:
  case AArch64::LDNPSi:
  case AArch64::LDNPWi:
  case AArch64::LDNPXi:
  case AArch64::LDNT1B_ZRI:
  case AArch64::LDNT1D_ZRI:
  case AArch64::LDNT1H_ZRI:
  case AArch64::LDNT1W_ZRI:
  case AArch64::LDPDi:
  case AArch64::LDPQi:
  case AArch64::LDPSi:
  case AArch64::LDPWi:
  case AArch64::LDPXi:
  case AArch64::LDRBBpost:
  case AArch64::LDRBBpre:
  case AArch64::LDRBpost:
  case AArch64::LDRBpre:
  case AArch64::LDRDpost:
  case AArch64::LDRDpre:
  case AArch64::LDRHHpost:
  case AArch64::LDRHHpre:
  case AArch64::LDRHpost:
  case AArch64::LDRHpre:
  case AArch64::LDRQpost:
  case AArch64::LDRQpre:
  case AArch64::LDRSpost:
  case AArch64::LDRSpre:
  case AArch64::LDRWpost:
  case AArch64::LDRWpre:
  case AArch64::LDRXpost:
  case AArch64::LDRXpre:
  case AArch64::ST1B_D_IMM:
  case AArch64::ST1B_H_IMM:
  case AArch64::ST1B_IMM:
  case AArch64::ST1B_S_IMM:
  case AArch64::ST1D_IMM:
  case AArch64::ST1H_D_IMM:
  case AArch64::ST1H_IMM:
  case AArch64::ST1H_S_IMM:
  case AArch64::ST1W_D_IMM:
  case AArch64::ST1W_IMM:
  case AArch64::ST2B_IMM:
  case AArch64::ST2D_IMM:
  case AArch64::ST2H_IMM:
  case AArch64::ST2W_IMM:
  case AArch64::ST3B_IMM:
  case AArch64::ST3D_IMM:
  case AArch64::ST3H_IMM:
  case AArch64::ST3W_IMM:
  case AArch64::ST4B_IMM:
  case AArch64::ST4D_IMM:
  case AArch64::ST4H_IMM:
  case AArch64::ST4W_IMM:
  case AArch64::STGPi:
  case AArch64::STGPreIndex:
  case AArch64::STZGPreIndex:
  case AArch64::ST2GPreIndex:
  case AArch64::STZ2GPreIndex:
  case AArch64::STGPostIndex:
  case AArch64::STZGPostIndex:
  case AArch64::ST2GPostIndex:
  case AArch64::STZ2GPostIndex:
  case AArch64::STNPDi:
  case AArch64::STNPQi:
  case AArch64::STNPSi:
  case AArch64::STNPWi:
  case AArch64::STNPXi:
  case AArch64::STNT1B_ZRI:
  case AArch64::STNT1D_ZRI:
  case AArch64::STNT1H_ZRI:
  case AArch64::STNT1W_ZRI:
  case AArch64::STPDi:
  case AArch64::STPQi:
  case AArch64::STPSi:
  case AArch64::STPWi:
  case AArch64::STPXi:
  case AArch64::STRBBpost:
  case AArch64::STRBBpre:
  case AArch64::STRBpost:
  case AArch64::STRBpre:
  case AArch64::STRDpost:
  case AArch64::STRDpre:
  case AArch64::STRHHpost:
  case AArch64::STRHHpre:
  case AArch64::STRHpost:
  case AArch64::STRHpre:
  case AArch64::STRQpost:
  case AArch64::STRQpre:
  case AArch64::STRSpost:
  case AArch64::STRSpre:
  case AArch64::STRWpost:
  case AArch64::STRWpre:
  case AArch64::STRXpost:
  case AArch64::STRXpre:
  case AArch64::LD1B_2Z_IMM:
  case AArch64::LD1B_2Z_STRIDED_IMM:
  case AArch64::LD1H_2Z_IMM:
  case AArch64::LD1H_2Z_STRIDED_IMM:
  case AArch64::LD1W_2Z_IMM:
  case AArch64::LD1W_2Z_STRIDED_IMM:
  case AArch64::LD1D_2Z_IMM:
  case AArch64::LD1D_2Z_STRIDED_IMM:
  case AArch64::LD1B_4Z_IMM:
  case AArch64::LD1B_4Z_STRIDED_IMM:
  case AArch64::LD1H_4Z_IMM:
  case AArch64::LD1H_4Z_STRIDED_IMM:
  case AArch64::LD1W_4Z_IMM:
  case AArch64::LD1W_4Z_STRIDED_IMM:
  case AArch64::LD1D_4Z_IMM:
  case AArch64::LD1D_4Z_STRIDED_IMM:
  case AArch64::LD1B_2Z_IMM_PSEUDO:
  case AArch64::LD1H_2Z_IMM_PSEUDO:
  case AArch64::LD1W_2Z_IMM_PSEUDO:
  case AArch64::LD1D_2Z_IMM_PSEUDO:
  case AArch64::LD1B_4Z_IMM_PSEUDO:
  case AArch64::LD1H_4Z_IMM_PSEUDO:
  case AArch64::LD1W_4Z_IMM_PSEUDO:
  case AArch64::LD1D_4Z_IMM_PSEUDO:
  case AArch64::ST1B_2Z_IMM:
  case AArch64::ST1B_2Z_STRIDED_IMM:
  case AArch64::ST1H_2Z_IMM:
  case AArch64::ST1H_2Z_STRIDED_IMM:
  case AArch64::ST1W_2Z_IMM:
  case AArch64::ST1W_2Z_STRIDED_IMM:
  case AArch64::ST1D_2Z_IMM:
  case AArch64::ST1D_2Z_STRIDED_IMM:
  case AArch64::LDNT1B_2Z_IMM_PSEUDO:
  case AArch64::LDNT1B_2Z_IMM:
  case AArch64::LDNT1B_2Z_STRIDED_IMM:
  case AArch64::LDNT1H_2Z_IMM_PSEUDO:
  case AArch64::LDNT1H_2Z_IMM:
  case AArch64::LDNT1H_2Z_STRIDED_IMM:
  case AArch64::LDNT1W_2Z_IMM_PSEUDO:
  case AArch64::LDNT1W_2Z_IMM:
  case AArch64::LDNT1W_2Z_STRIDED_IMM:
  case AArch64::LDNT1D_2Z_IMM_PSEUDO:
  case AArch64::LDNT1D_2Z_IMM:
  case AArch64::LDNT1D_2Z_STRIDED_IMM:
  case AArch64::STNT1B_2Z_IMM:
  case AArch64::STNT1B_2Z_STRIDED_IMM:
  case AArch64::STNT1H_2Z_IMM:
  case AArch64::STNT1H_2Z_STRIDED_IMM:
  case AArch64::STNT1W_2Z_IMM:
  case AArch64::STNT1W_2Z_STRIDED_IMM:
  case AArch64::STNT1D_2Z_IMM:
  case AArch64::STNT1D_2Z_STRIDED_IMM:
  case AArch64::ST1B_2Z_IMM_PSEUDO:
  case AArch64::ST1H_2Z_IMM_PSEUDO:
  case AArch64::ST1W_2Z_IMM_PSEUDO:
  case AArch64::ST1D_2Z_IMM_PSEUDO:
  case AArch64::STNT1B_2Z_IMM_PSEUDO:
  case AArch64::STNT1H_2Z_IMM_PSEUDO:
  case AArch64::STNT1W_2Z_IMM_PSEUDO:
  case AArch64::STNT1D_2Z_IMM_PSEUDO:
  case AArch64::ST1B_4Z_IMM:
  case AArch64::ST1B_4Z_STRIDED_IMM:
  case AArch64::ST1H_4Z_IMM:
  case AArch64::ST1H_4Z_STRIDED_IMM:
  case AArch64::ST1W_4Z_IMM:
  case AArch64::ST1W_4Z_STRIDED_IMM:
  case AArch64::ST1D_4Z_IMM:
  case AArch64::ST1D_4Z_STRIDED_IMM:
  case AArch64::LDNT1B_4Z_IMM_PSEUDO:
  case AArch64::LDNT1B_4Z_IMM:
  case AArch64::LDNT1B_4Z_STRIDED_IMM:
  case AArch64::LDNT1H_4Z_IMM_PSEUDO:
  case AArch64::LDNT1H_4Z_IMM:
  case AArch64::LDNT1H_4Z_STRIDED_IMM:
  case AArch64::LDNT1W_4Z_IMM_PSEUDO:
  case AArch64::LDNT1W_4Z_IMM:
  case AArch64::LDNT1W_4Z_STRIDED_IMM:
  case AArch64::LDNT1D_4Z_IMM_PSEUDO:
  case AArch64::LDNT1D_4Z_IMM:
  case AArch64::LDNT1D_4Z_STRIDED_IMM:
  case AArch64::STNT1B_4Z_IMM:
  case AArch64::STNT1B_4Z_STRIDED_IMM:
  case AArch64::STNT1H_4Z_IMM:
  case AArch64::STNT1H_4Z_STRIDED_IMM:
  case AArch64::STNT1W_4Z_IMM:
  case AArch64::STNT1W_4Z_STRIDED_IMM:
  case AArch64::STNT1D_4Z_IMM:
  case AArch64::STNT1D_4Z_STRIDED_IMM:
  case AArch64::ST1B_4Z_IMM_PSEUDO:
  case AArch64::ST1H_4Z_IMM_PSEUDO:
  case AArch64::ST1W_4Z_IMM_PSEUDO:
  case AArch64::ST1D_4Z_IMM_PSEUDO:
  case AArch64::STNT1B_4Z_IMM_PSEUDO:
  case AArch64::STNT1H_4Z_IMM_PSEUDO:
  case AArch64::STNT1W_4Z_IMM_PSEUDO:
  case AArch64::STNT1D_4Z_IMM_PSEUDO:
    return 3;
  case AArch64::LDPDpost:
  case AArch64::LDPDpre:
  case AArch64::LDPQpost:
  case AArch64::LDPQpre:
  case AArch64::LDPSpost:
  case AArch64::LDPSpre:
  case AArch64::LDPWpost:
  case AArch64::LDPWpre:
  case AArch64::LDPXpost:
  case AArch64::LDPXpre:
  case AArch64::STGPpre:
  case AArch64::STGPpost:
  case AArch64::STPDpost:
  case AArch64::STPDpre:
  case AArch64::STPQpost:
  case AArch64::STPQpre:
  case AArch64::STPSpost:
  case AArch64::STPSpre:
  case AArch64::STPWpost:
  case AArch64::STPWpre:
  case AArch64::STPXpost:
  case AArch64::STPXpre:
    return 4;
  }
}
 
bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    return false;
  // Scaled instructions.
  case AArch64::STRSui:
  case AArch64::STRDui:
  case AArch64::STRQui:
  case AArch64::STRXui:
  case AArch64::STRWui:
  case AArch64::LDRSui:
  case AArch64::LDRDui:
  case AArch64::LDRQui:
  case AArch64::LDRXui:
  case AArch64::LDRWui:
  case AArch64::LDRSWui:
  // Unscaled instructions.
  case AArch64::STURSi:
  case AArch64::STRSpre:
  case AArch64::STURDi:
  case AArch64::STRDpre:
  case AArch64::STURQi:
  case AArch64::STRQpre:
  case AArch64::STURWi:
  case AArch64::STRWpre:
  case AArch64::STURXi:
  case AArch64::STRXpre:
  case AArch64::LDURSi:
  case AArch64::LDRSpre:
  case AArch64::LDURDi:
  case AArch64::LDRDpre:
  case AArch64::LDURQi:
  case AArch64::LDRQpre:
  case AArch64::LDURWi:
  case AArch64::LDRWpre:
  case AArch64::LDURXi:
  case AArch64::LDRXpre:
  case AArch64::LDURSWi:
  case AArch64::LDRSWpre:
  // SVE instructions.
  case AArch64::LDR_ZXI:
  case AArch64::STR_ZXI:
    return true;
  }
}
 
bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    assert((!MI.isCall() || !MI.isReturn()) &&
           "Unexpected instruction - was a new tail call opcode introduced?");
    return false;
  case AArch64::TCRETURNdi:
  case AArch64::TCRETURNri:
  case AArch64::TCRETURNrix16x17:
  case AArch64::TCRETURNrix17:
  case AArch64::TCRETURNrinotx16:
  case AArch64::TCRETURNriALL:
  case AArch64::AUTH_TCRETURN:
  case AArch64::AUTH_TCRETURN_BTI:
    return true;
  }
}
 
unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
  switch (Opc) {
  default:
    llvm_unreachable("Opcode has no flag setting equivalent!");
  // 32-bit cases:
  case AArch64::ADDWri:
    return AArch64::ADDSWri;
  case AArch64::ADDWrr:
    return AArch64::ADDSWrr;
  case AArch64::ADDWrs:
    return AArch64::ADDSWrs;
  case AArch64::ADDWrx:
    return AArch64::ADDSWrx;
  case AArch64::ANDWri:
    return AArch64::ANDSWri;
  case AArch64::ANDWrr:
    return AArch64::ANDSWrr;
  case AArch64::ANDWrs:
    return AArch64::ANDSWrs;
  case AArch64::BICWrr:
    return AArch64::BICSWrr;
  case AArch64::BICWrs:
    return AArch64::BICSWrs;
  case AArch64::SUBWri:
    return AArch64::SUBSWri;
  case AArch64::SUBWrr:
    return AArch64::SUBSWrr;
  case AArch64::SUBWrs:
    return AArch64::SUBSWrs;
  case AArch64::SUBWrx:
    return AArch64::SUBSWrx;
  // 64-bit cases:
  case AArch64::ADDXri:
    return AArch64::ADDSXri;
  case AArch64::ADDXrr:
    return AArch64::ADDSXrr;
  case AArch64::ADDXrs:
    return AArch64::ADDSXrs;
  case AArch64::ADDXrx:
    return AArch64::ADDSXrx;
  case AArch64::ANDXri:
    return AArch64::ANDSXri;
  case AArch64::ANDXrr:
    return AArch64::ANDSXrr;
  case AArch64::ANDXrs:
    return AArch64::ANDSXrs;
  case AArch64::BICXrr:
    return AArch64::BICSXrr;
  case AArch64::BICXrs:
    return AArch64::BICSXrs;
  case AArch64::SUBXri:
    return AArch64::SUBSXri;
  case AArch64::SUBXrr:
    return AArch64::SUBSXrr;
  case AArch64::SUBXrs:
    return AArch64::SUBSXrs;
  case AArch64::SUBXrx:
    return AArch64::SUBSXrx;
  // SVE instructions:
  case AArch64::AND_PPzPP:
    return AArch64::ANDS_PPzPP;
  case AArch64::BIC_PPzPP:
    return AArch64::BICS_PPzPP;
  case AArch64::EOR_PPzPP:
    return AArch64::EORS_PPzPP;
  case AArch64::NAND_PPzPP:
    return AArch64::NANDS_PPzPP;
  case AArch64::NOR_PPzPP:
    return AArch64::NORS_PPzPP;
  case AArch64::ORN_PPzPP:
    return AArch64::ORNS_PPzPP;
  case AArch64::ORR_PPzPP:
    return AArch64::ORRS_PPzPP;
  case AArch64::BRKA_PPzP:
    return AArch64::BRKAS_PPzP;
  case AArch64::BRKPA_PPzPP:
    return AArch64::BRKPAS_PPzPP;
  case AArch64::BRKB_PPzP:
    return AArch64::BRKBS_PPzP;
  case AArch64::BRKPB_PPzPP:
    return AArch64::BRKPBS_PPzPP;
  case AArch64::BRKN_PPzP:
    return AArch64::BRKNS_PPzP;
  case AArch64::RDFFR_PPz:
    return AArch64::RDFFRS_PPz;
  case AArch64::PTRUE_B:
    return AArch64::PTRUES_B;
  }
}
 
// Is this a candidate for ld/st merging or pairing?  For example, we don't
// touch volatiles or load/stores that have a hint to avoid pair formation.
bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
 
  bool IsPreLdSt = isPreLdSt(MI);
 
  // If this is a volatile load/store, don't mess with it.
  if (MI.hasOrderedMemoryRef())
    return false;
 
  // Make sure this is a reg/fi+imm (as opposed to an address reloc).
  // For Pre-inc LD/ST, the operand is shifted by one.
  assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
          MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
         "Expected a reg or frame index operand.");
 
  // For Pre-indexed addressing quadword instructions, the third operand is the
  // immediate value.
  bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(3).isImm();
 
  if (!MI.getOperand(2).isImm() && !IsImmPreLdSt)
    return false;
 
  // Can't merge/pair if the instruction modifies the base register.
  // e.g., ldr x0, [x0]
  // This case will never occur with an FI base.
  // However, if the instruction is an LDR<S,D,Q,W,X,SW>pre or
  // STR<S,D,Q,W,X>pre, it can be merged.
  // For example:
  //   ldr q0, [x11, #32]!
  //   ldr q1, [x11, #16]
  //   to
  //   ldp q0, q1, [x11, #32]!
  if (MI.getOperand(1).isReg() && !IsPreLdSt) {
    Register BaseReg = MI.getOperand(1).getReg();
    const TargetRegisterInfo *TRI = &getRegisterInfo();
    if (MI.modifiesRegister(BaseReg, TRI))
      return false;
  }
 
  // Pairing SVE fills/spills is only valid for little-endian targets that
  // implement VLS 128.
  switch (MI.getOpcode()) {
  default:
    break;
  case AArch64::LDR_ZXI:
  case AArch64::STR_ZXI:
    if (!Subtarget.isLittleEndian() ||
        Subtarget.getSVEVectorSizeInBits() != 128)
      return false;
  }
 
  // Check if this load/store has a hint to avoid pair formation.
  // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
  if (isLdStPairSuppressed(MI))
    return false;
 
  // Do not pair any callee-save store/reload instructions in the
  // prologue/epilogue if the CFI information encoded the operations as separate
  // instructions, as that will cause the size of the actual prologue to mismatch
  // with the prologue size recorded in the Windows CFI.
  const MCAsmInfo &MAI = MI.getMF()->getTarget().getMCAsmInfo();
  bool NeedsWinCFI =
      MAI.usesWindowsCFI() && MI.getMF()->getFunction().needsUnwindTableEntry();
  if (NeedsWinCFI && (MI.getFlag(MachineInstr::FrameSetup) ||
                      MI.getFlag(MachineInstr::FrameDestroy)))
    return false;
 
  // On some CPUs quad load/store pairs are slower than two single load/stores.
  if (Subtarget.isPaired128Slow()) {
    switch (MI.getOpcode()) {
    default:
      break;
    case AArch64::LDURQi:
    case AArch64::STURQi:
    case AArch64::LDRQui:
    case AArch64::STRQui:
      return false;
    }
  }
 
  return true;
}
 
bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
    const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
    int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
    const TargetRegisterInfo *TRI) const {
  if (!LdSt.mayLoadOrStore())
    return false;
 
  const MachineOperand *BaseOp;
  TypeSize WidthN(0, false);
  if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
                                    WidthN, TRI))
    return false;
  // The maximum vscale is 16 under AArch64, return the maximal extent for the
  // vector.
  Width = LocationSize::precise(WidthN);
  BaseOps.push_back(BaseOp);
  return true;
}
 
std::optional<ExtAddrMode>
AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
                                          const TargetRegisterInfo *TRI) const {
  const MachineOperand *Base; // Filled with the base operand of MI.
  int64_t Offset;             // Filled with the offset of MI.
  bool OffsetIsScalable;
  if (!getMemOperandWithOffset(MemI, Base, Offset, OffsetIsScalable, TRI))
    return std::nullopt;
 
  if (!Base->isReg())
    return std::nullopt;
  ExtAddrMode AM;
  AM.BaseReg = Base->getReg();
  AM.Displacement = Offset;
  AM.ScaledReg = 0;
  AM.Scale = 0;
  return AM;
}
 
bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
                                           Register Reg,
                                           const MachineInstr &AddrI,
                                           ExtAddrMode &AM) const {
  // Filter out instructions into which we cannot fold.
  unsigned NumBytes;
  int64_t OffsetScale = 1;
  switch (MemI.getOpcode()) {
  default:
    return false;
 
  case AArch64::LDURQi:
  case AArch64::STURQi:
    NumBytes = 16;
    break;
 
  case AArch64::LDURDi:
  case AArch64::STURDi:
  case AArch64::LDURXi:
  case AArch64::STURXi:
    NumBytes = 8;
    break;
 
  case AArch64::LDURWi:
  case AArch64::LDURSWi:
  case AArch64::STURWi:
    NumBytes = 4;
    break;
 
  case AArch64::LDURHi:
  case AArch64::STURHi:
  case AArch64::LDURHHi:
  case AArch64::STURHHi:
  case AArch64::LDURSHXi:
  case AArch64::LDURSHWi:
    NumBytes = 2;
    break;
 
  case AArch64::LDRBroX:
  case AArch64::LDRBBroX:
  case AArch64::LDRSBXroX:
  case AArch64::LDRSBWroX:
  case AArch64::STRBroX:
  case AArch64::STRBBroX:
  case AArch64::LDURBi:
  case AArch64::LDURBBi:
  case AArch64::LDURSBXi:
  case AArch64::LDURSBWi:
  case AArch64::STURBi:
  case AArch64::STURBBi:
  case AArch64::LDRBui:
  case AArch64::LDRBBui:
  case AArch64::LDRSBXui:
  case AArch64::LDRSBWui:
  case AArch64::STRBui:
  case AArch64::STRBBui:
    NumBytes = 1;
    break;
 
  case AArch64::LDRQroX:
  case AArch64::STRQroX:
  case AArch64::LDRQui:
  case AArch64::STRQui:
    NumBytes = 16;
    OffsetScale = 16;
    break;
 
  case AArch64::LDRDroX:
  case AArch64::STRDroX:
  case AArch64::LDRXroX:
  case AArch64::STRXroX:
  case AArch64::LDRDui:
  case AArch64::STRDui:
  case AArch64::LDRXui:
  case AArch64::STRXui:
    NumBytes = 8;
    OffsetScale = 8;
    break;
 
  case AArch64::LDRWroX:
  case AArch64::LDRSWroX:
  case AArch64::STRWroX:
  case AArch64::LDRWui:
  case AArch64::LDRSWui:
  case AArch64::STRWui:
    NumBytes = 4;
    OffsetScale = 4;
    break;
 
  case AArch64::LDRHroX:
  case AArch64::STRHroX:
  case AArch64::LDRHHroX:
  case AArch64::STRHHroX:
  case AArch64::LDRSHXroX:
  case AArch64::LDRSHWroX:
  case AArch64::LDRHui:
  case AArch64::STRHui:
  case AArch64::LDRHHui:
  case AArch64::STRHHui:
  case AArch64::LDRSHXui:
  case AArch64::LDRSHWui:
    NumBytes = 2;
    OffsetScale = 2;
    break;
  }
 
  // Check the fold operand is not the loaded/stored value.
  const MachineOperand &BaseRegOp = MemI.getOperand(0);
  if (BaseRegOp.isReg() && BaseRegOp.getReg() == Reg)
    return false;
 
  // Handle memory instructions with a [Reg, Reg] addressing mode.
  if (MemI.getOperand(2).isReg()) {
    // Bail if the addressing mode already includes extension of the offset
    // register.
    if (MemI.getOperand(3).getImm())
      return false;
 
    // Check if we actually have a scaled offset.
    if (MemI.getOperand(4).getImm() == 0)
      OffsetScale = 1;
 
    // If the address instructions is folded into the base register, then the
    // addressing mode must not have a scale. Then we can swap the base and the
    // scaled registers.
    if (MemI.getOperand(1).getReg() == Reg && OffsetScale != 1)
      return false;
 
    switch (AddrI.getOpcode()) {
    default:
      return false;
 
    case AArch64::SBFMXri:
      // sxtw Xa, Wm
      // ldr Xd, [Xn, Xa, lsl #N]
      // ->
      // ldr Xd, [Xn, Wm, sxtw #N]
      if (AddrI.getOperand(2).getImm() != 0 ||
          AddrI.getOperand(3).getImm() != 31)
        return false;
 
      AM.BaseReg = MemI.getOperand(1).getReg();
      if (AM.BaseReg == Reg)
        AM.BaseReg = MemI.getOperand(2).getReg();
      AM.ScaledReg = AddrI.getOperand(1).getReg();
      AM.Scale = OffsetScale;
      AM.Displacement = 0;
      AM.Form = ExtAddrMode::Formula::SExtScaledReg;
      return true;
 
    case TargetOpcode::SUBREG_TO_REG: {
      // mov Wa, Wm
      // ldr Xd, [Xn, Xa, lsl #N]
      // ->
      // ldr Xd, [Xn, Wm, uxtw #N]
 
      // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG.
      if (AddrI.getOperand(2).getImm() != AArch64::sub_32)
        return false;
 
      const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo();
      Register OffsetReg = AddrI.getOperand(1).getReg();
      if (!OffsetReg.isVirtual() || !MRI.hasOneNonDBGUse(OffsetReg))
        return false;
 
      const MachineInstr &DefMI = *MRI.getVRegDef(OffsetReg);
      if (DefMI.getOpcode() != AArch64::ORRWrs ||
          DefMI.getOperand(1).getReg() != AArch64::WZR ||
          DefMI.getOperand(3).getImm() != 0)
        return false;
 
      AM.BaseReg = MemI.getOperand(1).getReg();
      if (AM.BaseReg == Reg)
        AM.BaseReg = MemI.getOperand(2).getReg();
      AM.ScaledReg = DefMI.getOperand(2).getReg();
      AM.Scale = OffsetScale;
      AM.Displacement = 0;
      AM.Form = ExtAddrMode::Formula::ZExtScaledReg;
      return true;
    }
    }
  }
 
  // Handle memory instructions with a [Reg, #Imm] addressing mode.
 
  // Check we are not breaking a potential conversion to an LDP.
  auto validateOffsetForLDP = [](unsigned NumBytes, int64_t OldOffset,
                                 int64_t NewOffset) -> bool {
    int64_t MinOffset, MaxOffset;
    switch (NumBytes) {
    default:
      return true;
    case 4:
      MinOffset = -256;
      MaxOffset = 252;
      break;
    case 8:
      MinOffset = -512;
      MaxOffset = 504;
      break;
    case 16:
      MinOffset = -1024;
      MaxOffset = 1008;
      break;
    }
    return OldOffset < MinOffset || OldOffset > MaxOffset ||
           (NewOffset >= MinOffset && NewOffset <= MaxOffset);
  };
  auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool {
    int64_t OldOffset = MemI.getOperand(2).getImm() * OffsetScale;
    int64_t NewOffset = OldOffset + Disp;
    if (!isLegalAddressingMode(NumBytes, NewOffset, /* Scale */ 0))
      return false;
    // If the old offset would fit into an LDP, but the new offset wouldn't,
    // bail out.
    if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
      return false;
    AM.BaseReg = AddrI.getOperand(1).getReg();
    AM.ScaledReg = 0;
    AM.Scale = 0;
    AM.Displacement = NewOffset;
    AM.Form = ExtAddrMode::Formula::Basic;
    return true;
  };
 
  auto canFoldAddRegIntoAddrMode =
      [&](int64_t Scale,
          ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
    if (MemI.getOperand(2).getImm() != 0)
      return false;
    if ((unsigned)Scale != Scale)
      return false;
    if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
      return false;
    AM.BaseReg = AddrI.getOperand(1).getReg();
    AM.ScaledReg = AddrI.getOperand(2).getReg();
    AM.Scale = Scale;
    AM.Displacement = 0;
    AM.Form = Form;
    return true;
  };
 
  auto avoidSlowSTRQ = [&](const MachineInstr &MemI) {
    unsigned Opcode = MemI.getOpcode();
    return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
           Subtarget.isSTRQroSlow();
  };
 
  int64_t Disp = 0;
  const bool OptSize = MemI.getMF()->getFunction().hasOptSize();
  switch (AddrI.getOpcode()) {
  default:
    return false;
 
  case AArch64::ADDXri:
    // add Xa, Xn, #N
    // ldr Xd, [Xa, #M]
    // ->
    // ldr Xd, [Xn, #N'+M]
    Disp = AddrI.getOperand(2).getImm() << AddrI.getOperand(3).getImm();
    return canFoldAddSubImmIntoAddrMode(Disp);
 
  case AArch64::SUBXri:
    // sub Xa, Xn, #N
    // ldr Xd, [Xa, #M]
    // ->
    // ldr Xd, [Xn, #N'+M]
    Disp = AddrI.getOperand(2).getImm() << AddrI.getOperand(3).getImm();
    return canFoldAddSubImmIntoAddrMode(-Disp);
 
  case AArch64::ADDXrs: {
    // add Xa, Xn, Xm, lsl #N
    // ldr Xd, [Xa]
    // ->
    // ldr Xd, [Xn, Xm, lsl #N]
 
    // Don't fold the add if the result would be slower, unless optimising for
    // size.
    unsigned Shift = static_cast<unsigned>(AddrI.getOperand(3).getImm());
    if (AArch64_AM::getShiftType(Shift) != AArch64_AM::ShiftExtendType::LSL)
      return false;
    Shift = AArch64_AM::getShiftValue(Shift);
    if (!OptSize) {
      if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
        return false;
      if (avoidSlowSTRQ(MemI))
        return false;
    }
    return canFoldAddRegIntoAddrMode(1ULL << Shift);
  }
 
  case AArch64::ADDXrr:
    // add Xa, Xn, Xm
    // ldr Xd, [Xa]
    // ->
    // ldr Xd, [Xn, Xm, lsl #0]
 
    // Don't fold the add if the result would be slower, unless optimising for
    // size.
    if (!OptSize && avoidSlowSTRQ(MemI))
      return false;
    return canFoldAddRegIntoAddrMode(1);
 
  case AArch64::ADDXrx:
    // add Xa, Xn, Wm, {s,u}xtw #N
    // ldr Xd, [Xa]
    // ->
    // ldr Xd, [Xn, Wm, {s,u}xtw #N]
 
    // Don't fold the add if the result would be slower, unless optimising for
    // size.
    if (!OptSize && avoidSlowSTRQ(MemI))
      return false;
 
    // Can fold only sign-/zero-extend of a word.
    unsigned Imm = static_cast<unsigned>(AddrI.getOperand(3).getImm());
    AArch64_AM::ShiftExtendType Extend = AArch64_AM::getArithExtendType(Imm);
    if (Extend != AArch64_AM::UXTW && Extend != AArch64_AM::SXTW)
      return false;
 
    return canFoldAddRegIntoAddrMode(
        1ULL << AArch64_AM::getArithShiftValue(Imm),
        (Extend == AArch64_AM::SXTW) ? ExtAddrMode::Formula::SExtScaledReg
                                     : ExtAddrMode::Formula::ZExtScaledReg);
  }
}
 
// Given an opcode for an instruction with a [Reg, #Imm] addressing mode,
// return the opcode of an instruction performing the same operation, but using
// the [Reg, Reg] addressing mode.
static unsigned regOffsetOpcode(unsigned Opcode) {
  switch (Opcode) {
  default:
    llvm_unreachable("Address folding not implemented for instruction");
 
  case AArch64::LDURQi:
  case AArch64::LDRQui:
    return AArch64::LDRQroX;
  case AArch64::STURQi:
  case AArch64::STRQui:
    return AArch64::STRQroX;
  case AArch64::LDURDi:
  case AArch64::LDRDui:
    return AArch64::LDRDroX;
  case AArch64::STURDi:
  case AArch64::STRDui:
    return AArch64::STRDroX;
  case AArch64::LDURXi:
  case AArch64::LDRXui:
    return AArch64::LDRXroX;
  case AArch64::STURXi:
  case AArch64::STRXui:
    return AArch64::STRXroX;
  case AArch64::LDURWi:
  case AArch64::LDRWui:
    return AArch64::LDRWroX;
  case AArch64::LDURSWi:
  case AArch64::LDRSWui:
    return AArch64::LDRSWroX;
  case AArch64::STURWi:
  case AArch64::STRWui:
    return AArch64::STRWroX;
  case AArch64::LDURHi:
  case AArch64::LDRHui:
    return AArch64::LDRHroX;
  case AArch64::STURHi:
  case AArch64::STRHui:
    return AArch64::STRHroX;
  case AArch64::LDURHHi:
  case AArch64::LDRHHui:
    return AArch64::LDRHHroX;
  case AArch64::STURHHi:
  case AArch64::STRHHui:
    return AArch64::STRHHroX;
  case AArch64::LDURSHXi:
  case AArch64::LDRSHXui:
    return AArch64::LDRSHXroX;
  case AArch64::LDURSHWi:
  case AArch64::LDRSHWui:
    return AArch64::LDRSHWroX;
  case AArch64::LDURBi:
  case AArch64::LDRBui:
    return AArch64::LDRBroX;
  case AArch64::LDURBBi:
  case AArch64::LDRBBui:
    return AArch64::LDRBBroX;
  case AArch64::LDURSBXi:
  case AArch64::LDRSBXui:
    return AArch64::LDRSBXroX;
  case AArch64::LDURSBWi:
  case AArch64::LDRSBWui:
    return AArch64::LDRSBWroX;
  case AArch64::STURBi:
  case AArch64::STRBui:
    return AArch64::STRBroX;
  case AArch64::STURBBi:
  case AArch64::STRBBui:
    return AArch64::STRBBroX;
  }
}
 
// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
// the opcode of an instruction performing the same operation, but using the
// [Reg, #Imm] addressing mode with scaled offset.
unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale) {
  switch (Opcode) {
  default:
    llvm_unreachable("Address folding not implemented for instruction");
 
  case AArch64::LDURQi:
    Scale = 16;
    return AArch64::LDRQui;
  case AArch64::STURQi:
    Scale = 16;
    return AArch64::STRQui;
  case AArch64::LDURDi:
    Scale = 8;
    return AArch64::LDRDui;
  case AArch64::STURDi:
    Scale = 8;
    return AArch64::STRDui;
  case AArch64::LDURXi:
    Scale = 8;
    return AArch64::LDRXui;
  case AArch64::STURXi:
    Scale = 8;
    return AArch64::STRXui;
  case AArch64::LDURWi:
    Scale = 4;
    return AArch64::LDRWui;
  case AArch64::LDURSWi:
    Scale = 4;
    return AArch64::LDRSWui;
  case AArch64::STURWi:
    Scale = 4;
    return AArch64::STRWui;
  case AArch64::LDURHi:
    Scale = 2;
    return AArch64::LDRHui;
  case AArch64::STURHi:
    Scale = 2;
    return AArch64::STRHui;
  case AArch64::LDURHHi:
    Scale = 2;
    return AArch64::LDRHHui;
  case AArch64::STURHHi:
    Scale = 2;
    return AArch64::STRHHui;
  case AArch64::LDURSHXi:
    Scale = 2;
    return AArch64::LDRSHXui;
  case AArch64::LDURSHWi:
    Scale = 2;
    return AArch64::LDRSHWui;
  case AArch64::LDURBi:
    Scale = 1;
    return AArch64::LDRBui;
  case AArch64::LDURBBi:
    Scale = 1;
    return AArch64::LDRBBui;
  case AArch64::LDURSBXi:
    Scale = 1;
    return AArch64::LDRSBXui;
  case AArch64::LDURSBWi:
    Scale = 1;
    return AArch64::LDRSBWui;
  case AArch64::STURBi:
    Scale = 1;
    return AArch64::STRBui;
  case AArch64::STURBBi:
    Scale = 1;
    return AArch64::STRBBui;
  case AArch64::LDRQui:
  case AArch64::STRQui:
    Scale = 16;
    return Opcode;
  case AArch64::LDRDui:
  case AArch64::STRDui:
  case AArch64::LDRXui:
  case AArch64::STRXui:
    Scale = 8;
    return Opcode;
  case AArch64::LDRWui:
  case AArch64::LDRSWui:
  case AArch64::STRWui:
    Scale = 4;
    return Opcode;
  case AArch64::LDRHui:
  case AArch64::STRHui:
  case AArch64::LDRHHui:
  case AArch64::STRHHui:
  case AArch64::LDRSHXui:
  case AArch64::LDRSHWui:
    Scale = 2;
    return Opcode;
  case AArch64::LDRBui:
  case AArch64::LDRBBui:
  case AArch64::LDRSBXui:
  case AArch64::LDRSBWui:
  case AArch64::STRBui:
  case AArch64::STRBBui:
    Scale = 1;
    return Opcode;
  }
}
 
// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
// the opcode of an instruction performing the same operation, but using the
// [Reg, #Imm] addressing mode with unscaled offset.
unsigned unscaledOffsetOpcode(unsigned Opcode) {
  switch (Opcode) {
  default:
    llvm_unreachable("Address folding not implemented for instruction");
 
  case AArch64::LDURQi:
  case AArch64::STURQi:
  case AArch64::LDURDi:
  case AArch64::STURDi:
  case AArch64::LDURXi:
  case AArch64::STURXi:
  case AArch64::LDURWi:
  case AArch64::LDURSWi:
  case AArch64::STURWi:
  case AArch64::LDURHi:
  case AArch64::STURHi:
  case AArch64::LDURHHi:
  case AArch64::STURHHi:
  case AArch64::LDURSHXi:
  case AArch64::LDURSHWi:
  case AArch64::LDURBi:
  case AArch64::STURBi:
  case AArch64::LDURBBi:
  case AArch64::STURBBi:
  case AArch64::LDURSBWi:
  case AArch64::LDURSBXi:
    return Opcode;
  case AArch64::LDRQui:
    return AArch64::LDURQi;
  case AArch64::STRQui:
    return AArch64::STURQi;
  case AArch64::LDRDui:
    return AArch64::LDURDi;
  case AArch64::STRDui:
    return AArch64::STURDi;
  case AArch64::LDRXui:
    return AArch64::LDURXi;
  case AArch64::STRXui:
    return AArch64::STURXi;
  case AArch64::LDRWui:
    return AArch64::LDURWi;
  case AArch64::LDRSWui:
    return AArch64::LDURSWi;
  case AArch64::STRWui:
    return AArch64::STURWi;
  case AArch64::LDRHui:
    return AArch64::LDURHi;
  case AArch64::STRHui:
    return AArch64::STURHi;
  case AArch64::LDRHHui:
    return AArch64::LDURHHi;
  case AArch64::STRHHui:
    return AArch64::STURHHi;
  case AArch64::LDRSHXui:
    return AArch64::LDURSHXi;
  case AArch64::LDRSHWui:
    return AArch64::LDURSHWi;
  case AArch64::LDRBBui:
    return AArch64::LDURBBi;
  case AArch64::LDRBui:
    return AArch64::LDURBi;
  case AArch64::STRBBui:
    return AArch64::STURBBi;
  case AArch64::STRBui:
    return AArch64::STURBi;
  case AArch64::LDRSBWui:
    return AArch64::LDURSBWi;
  case AArch64::LDRSBXui:
    return AArch64::LDURSBXi;
  }
}
 
// Given the opcode of a memory load/store instruction, return the opcode of an
// instruction performing the same operation, but using
// the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
// offset register.
static unsigned offsetExtendOpcode(unsigned Opcode) {
  switch (Opcode) {
  default:
    llvm_unreachable("Address folding not implemented for instruction");
 
  case AArch64::LDRQroX:
  case AArch64::LDURQi:
  case AArch64::LDRQui:
    return AArch64::LDRQroW;
  case AArch64::STRQroX:
  case AArch64::STURQi:
  case AArch64::STRQui:
    return AArch64::STRQroW;
  case AArch64::LDRDroX:
  case AArch64::LDURDi:
  case AArch64::LDRDui:
    return AArch64::LDRDroW;
  case AArch64::STRDroX:
  case AArch64::STURDi:
  case AArch64::STRDui:
    return AArch64::STRDroW;
  case AArch64::LDRXroX:
  case AArch64::LDURXi:
  case AArch64::LDRXui:
    return AArch64::LDRXroW;
  case AArch64::STRXroX:
  case AArch64::STURXi:
  case AArch64::STRXui:
    return AArch64::STRXroW;
  case AArch64::LDRWroX:
  case AArch64::LDURWi:
  case AArch64::LDRWui:
    return AArch64::LDRWroW;
  case AArch64::LDRSWroX:
  case AArch64::LDURSWi:
  case AArch64::LDRSWui:
    return AArch64::LDRSWroW;
  case AArch64::STRWroX:
  case AArch64::STURWi:
  case AArch64::STRWui:
    return AArch64::STRWroW;
  case AArch64::LDRHroX:
  case AArch64::LDURHi:
  case AArch64::LDRHui:
    return AArch64::LDRHroW;
  case AArch64::STRHroX:
  case AArch64::STURHi:
  case AArch64::STRHui:
    return AArch64::STRHroW;
  case AArch64::LDRHHroX:
  case AArch64::LDURHHi:
  case AArch64::LDRHHui:
    return AArch64::LDRHHroW;
  case AArch64::STRHHroX:
  case AArch64::STURHHi:
  case AArch64::STRHHui:
    return AArch64::STRHHroW;
  case AArch64::LDRSHXroX:
  case AArch64::LDURSHXi:
  case AArch64::LDRSHXui:
    return AArch64::LDRSHXroW;
  case AArch64::LDRSHWroX:
  case AArch64::LDURSHWi:
  case AArch64::LDRSHWui:
    return AArch64::LDRSHWroW;
  case AArch64::LDRBroX:
  case AArch64::LDURBi:
  case AArch64::LDRBui:
    return AArch64::LDRBroW;
  case AArch64::LDRBBroX:
  case AArch64::LDURBBi:
  case AArch64::LDRBBui:
    return AArch64::LDRBBroW;
  case AArch64::LDRSBXroX:
  case AArch64::LDURSBXi:
  case AArch64::LDRSBXui:
    return AArch64::LDRSBXroW;
  case AArch64::LDRSBWroX:
  case AArch64::LDURSBWi:
  case AArch64::LDRSBWui:
    return AArch64::LDRSBWroW;
  case AArch64::STRBroX:
  case AArch64::STURBi:
  case AArch64::STRBui:
    return AArch64::STRBroW;
  case AArch64::STRBBroX:
  case AArch64::STURBBi:
  case AArch64::STRBBui:
    return AArch64::STRBBroW;
  }
}
 
MachineInstr *AArch64InstrInfo::emitLdStWithAddr(MachineInstr &MemI,
                                                 const ExtAddrMode &AM) const {
 
  const DebugLoc &DL = MemI.getDebugLoc();
  MachineBasicBlock &MBB = *MemI.getParent();
  MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo();
 
  if (AM.Form == ExtAddrMode::Formula::Basic) {
    if (AM.ScaledReg) {
      // The new instruction will be in the form `ldr Rt, [Xn, Xm, lsl #imm]`.
      unsigned Opcode = regOffsetOpcode(MemI.getOpcode());
      MRI.constrainRegClass(AM.BaseReg, &AArch64::GPR64spRegClass);
      auto B = BuildMI(MBB, MemI, DL, get(Opcode))
                   .addReg(MemI.getOperand(0).getReg(),
                           getDefRegState(MemI.mayLoad()))
                   .addReg(AM.BaseReg)
                   .addReg(AM.ScaledReg)
                   .addImm(0)
                   .addImm(AM.Scale > 1)
                   .setMemRefs(MemI.memoperands())
                   .setMIFlags(MemI.getFlags());
      return B.getInstr();
    }
 
    assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
           "Addressing mode not supported for folding");
 
    // The new instruction will be in the form `ld[u]r Rt, [Xn, #imm]`.
    unsigned Scale = 1;
    unsigned Opcode = MemI.getOpcode();
    if (isInt<9>(AM.Displacement))
      Opcode = unscaledOffsetOpcode(Opcode);
    else
      Opcode = scaledOffsetOpcode(Opcode, Scale);
 
    auto B =
        BuildMI(MBB, MemI, DL, get(Opcode))
            .addReg(MemI.getOperand(0).getReg(), getDefRegState(MemI.mayLoad()))
            .addReg(AM.BaseReg)
            .addImm(AM.Displacement / Scale)
            .setMemRefs(MemI.memoperands())
            .setMIFlags(MemI.getFlags());
    return B.getInstr();
  }
 
  if (AM.Form == ExtAddrMode::Formula::SExtScaledReg ||
      AM.Form == ExtAddrMode::Formula::ZExtScaledReg) {
    // The new instruction will be in the form `ldr Rt, [Xn, Wm, {s,u}xtw #N]`.
    assert(AM.ScaledReg && !AM.Displacement &&
           "Address offset can be a register or an immediate, but not both");
    unsigned Opcode = offsetExtendOpcode(MemI.getOpcode());
    MRI.constrainRegClass(AM.BaseReg, &AArch64::GPR64spRegClass);
    // Make sure the offset register is in the correct register class.
    Register OffsetReg = AM.ScaledReg;
    const TargetRegisterClass *RC = MRI.getRegClass(OffsetReg);
    if (RC->hasSuperClassEq(&AArch64::GPR64RegClass)) {
      OffsetReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
      BuildMI(MBB, MemI, DL, get(TargetOpcode::COPY), OffsetReg)
          .addReg(AM.ScaledReg, {}, AArch64::sub_32);
    }
    auto B =
        BuildMI(MBB, MemI, DL, get(Opcode))
            .addReg(MemI.getOperand(0).getReg(), getDefRegState(MemI.mayLoad()))
            .addReg(AM.BaseReg)
            .addReg(OffsetReg)
            .addImm(AM.Form == ExtAddrMode::Formula::SExtScaledReg)
            .addImm(AM.Scale != 1)
            .setMemRefs(MemI.memoperands())
            .setMIFlags(MemI.getFlags());
 
    return B.getInstr();
  }
 
  llvm_unreachable(
      "Function must not be called with an addressing mode it can't handle");
}
 
/// Return true if the opcode is a post-index ld/st instruction, which really
/// loads from base+0.
static bool isPostIndexLdStOpcode(unsigned Opcode) {
  switch (Opcode) {
  default:
    return false;
  case AArch64::LD1Fourv16b_POST:
  case AArch64::LD1Fourv1d_POST:
  case AArch64::LD1Fourv2d_POST:
  case AArch64::LD1Fourv2s_POST:
  case AArch64::LD1Fourv4h_POST:
  case AArch64::LD1Fourv4s_POST:
  case AArch64::LD1Fourv8b_POST:
  case AArch64::LD1Fourv8h_POST:
  case AArch64::LD1Onev16b_POST:
  case AArch64::LD1Onev1d_POST:
  case AArch64::LD1Onev2d_POST:
  case AArch64::LD1Onev2s_POST:
  case AArch64::LD1Onev4h_POST:
  case AArch64::LD1Onev4s_POST:
  case AArch64::LD1Onev8b_POST:
  case AArch64::LD1Onev8h_POST:
  case AArch64::LD1Rv16b_POST:
  case AArch64::LD1Rv1d_POST:
  case AArch64::LD1Rv2d_POST:
  case AArch64::LD1Rv2s_POST:
  case AArch64::LD1Rv4h_POST:
  case AArch64::LD1Rv4s_POST:
  case AArch64::LD1Rv8b_POST:
  case AArch64::LD1Rv8h_POST:
  case AArch64::LD1Threev16b_POST:
  case AArch64::LD1Threev1d_POST:
  case AArch64::LD1Threev2d_POST:
  case AArch64::LD1Threev2s_POST:
  case AArch64::LD1Threev4h_POST:
  case AArch64::LD1Threev4s_POST:
  case AArch64::LD1Threev8b_POST:
  case AArch64::LD1Threev8h_POST:
  case AArch64::LD1Twov16b_POST:
  case AArch64::LD1Twov1d_POST:
  case AArch64::LD1Twov2d_POST:
  case AArch64::LD1Twov2s_POST:
  case AArch64::LD1Twov4h_POST:
  case AArch64::LD1Twov4s_POST:
  case AArch64::LD1Twov8b_POST:
  case AArch64::LD1Twov8h_POST:
  case AArch64::LD1i16_POST:
  case AArch64::LD1i32_POST:
  case AArch64::LD1i64_POST:
  case AArch64::LD1i8_POST:
  case AArch64::LD2Rv16b_POST:
  case AArch64::LD2Rv1d_POST:
  case AArch64::LD2Rv2d_POST:
  case AArch64::LD2Rv2s_POST:
  case AArch64::LD2Rv4h_POST:
  case AArch64::LD2Rv4s_POST:
  case AArch64::LD2Rv8b_POST:
  case AArch64::LD2Rv8h_POST:
  case AArch64::LD2Twov16b_POST:
  case AArch64::LD2Twov2d_POST:
  case AArch64::LD2Twov2s_POST:
  case AArch64::LD2Twov4h_POST:
  case AArch64::LD2Twov4s_POST:
  case AArch64::LD2Twov8b_POST:
  case AArch64::LD2Twov8h_POST:
  case AArch64::LD2i16_POST:
  case AArch64::LD2i32_POST:
  case AArch64::LD2i64_POST:
  case AArch64::LD2i8_POST:
  case AArch64::LD3Rv16b_POST:
  case AArch64::LD3Rv1d_POST:
  case AArch64::LD3Rv2d_POST:
  case AArch64::LD3Rv2s_POST:
  case AArch64::LD3Rv4h_POST:
  case AArch64::LD3Rv4s_POST:
  case AArch64::LD3Rv8b_POST:
  case AArch64::LD3Rv8h_POST:
  case AArch64::LD3Threev16b_POST:
  case AArch64::LD3Threev2d_POST:
  case AArch64::LD3Threev2s_POST:
  case AArch64::LD3Threev4h_POST:
  case AArch64::LD3Threev4s_POST:
  case AArch64::LD3Threev8b_POST:
  case AArch64::LD3Threev8h_POST:
  case AArch64::LD3i16_POST:
  case AArch64::LD3i32_POST:
  case AArch64::LD3i64_POST:
  case AArch64::LD3i8_POST:
  case AArch64::LD4Fourv16b_POST:
  case AArch64::LD4Fourv2d_POST:
  case AArch64::LD4Fourv2s_POST:
  case AArch64::LD4Fourv4h_POST:
  case AArch64::LD4Fourv4s_POST:
  case AArch64::LD4Fourv8b_POST:
  case AArch64::LD4Fourv8h_POST:
  case AArch64::LD4Rv16b_POST:
  case AArch64::LD4Rv1d_POST:
  case AArch64::LD4Rv2d_POST:
  case AArch64::LD4Rv2s_POST:
  case AArch64::LD4Rv4h_POST:
  case AArch64::LD4Rv4s_POST:
  case AArch64::LD4Rv8b_POST:
  case AArch64::LD4Rv8h_POST:
  case AArch64::LD4i16_POST:
  case AArch64::LD4i32_POST:
  case AArch64::LD4i64_POST:
  case AArch64::LD4i8_POST:
  case AArch64::LDAPRWpost:
  case AArch64::LDAPRXpost:
  case AArch64::LDIAPPWpost:
  case AArch64::LDIAPPXpost:
  case AArch64::LDPDpost:
  case AArch64::LDPQpost:
  case AArch64::LDPSWpost:
  case AArch64::LDPSpost:
  case AArch64::LDPWpost:
  case AArch64::LDPXpost:
  case AArch64::LDRBBpost:
  case AArch64::LDRBpost:
  case AArch64::LDRDpost:
  case AArch64::LDRHHpost:
  case AArch64::LDRHpost:
  case AArch64::LDRQpost:
  case AArch64::LDRSBWpost:
  case AArch64::LDRSBXpost:
  case AArch64::LDRSHWpost:
  case AArch64::LDRSHXpost:
  case AArch64::LDRSWpost:
  case AArch64::LDRSpost:
  case AArch64::LDRWpost:
  case AArch64::LDRXpost:
  case AArch64::ST1Fourv16b_POST:
  case AArch64::ST1Fourv1d_POST:
  case AArch64::ST1Fourv2d_POST:
  case AArch64::ST1Fourv2s_POST:
  case AArch64::ST1Fourv4h_POST:
  case AArch64::ST1Fourv4s_POST:
  case AArch64::ST1Fourv8b_POST:
  case AArch64::ST1Fourv8h_POST:
  case AArch64::ST1Onev16b_POST:
  case AArch64::ST1Onev1d_POST:
  case AArch64::ST1Onev2d_POST:
  case AArch64::ST1Onev2s_POST:
  case AArch64::ST1Onev4h_POST:
  case AArch64::ST1Onev4s_POST:
  case AArch64::ST1Onev8b_POST:
  case AArch64::ST1Onev8h_POST:
  case AArch64::ST1Threev16b_POST:
  case AArch64::ST1Threev1d_POST:
  case AArch64::ST1Threev2d_POST:
  case AArch64::ST1Threev2s_POST:
  case AArch64::ST1Threev4h_POST:
  case AArch64::ST1Threev4s_POST:
  case AArch64::ST1Threev8b_POST:
  case AArch64::ST1Threev8h_POST:
  case AArch64::ST1Twov16b_POST:
  case AArch64::ST1Twov1d_POST:
  case AArch64::ST1Twov2d_POST:
  case AArch64::ST1Twov2s_POST:
  case AArch64::ST1Twov4h_POST:
  case AArch64::ST1Twov4s_POST:
  case AArch64::ST1Twov8b_POST:
  case AArch64::ST1Twov8h_POST:
  case AArch64::ST1i16_POST:
  case AArch64::ST1i32_POST:
  case AArch64::ST1i64_POST:
  case AArch64::ST1i8_POST:
  case AArch64::ST2GPostIndex:
  case AArch64::ST2Twov16b_POST:
  case AArch64::ST2Twov2d_POST:
  case AArch64::ST2Twov2s_POST:
  case AArch64::ST2Twov4h_POST:
  case AArch64::ST2Twov4s_POST:
  case AArch64::ST2Twov8b_POST:
  case AArch64::ST2Twov8h_POST:
  case AArch64::ST2i16_POST:
  case AArch64::ST2i32_POST:
  case AArch64::ST2i64_POST:
  case AArch64::ST2i8_POST:
  case AArch64::ST3Threev16b_POST:
  case AArch64::ST3Threev2d_POST:
  case AArch64::ST3Threev2s_POST:
  case AArch64::ST3Threev4h_POST:
  case AArch64::ST3Threev4s_POST:
  case AArch64::ST3Threev8b_POST:
  case AArch64::ST3Threev8h_POST:
  case AArch64::ST3i16_POST:
  case AArch64::ST3i32_POST:
  case AArch64::ST3i64_POST:
  case AArch64::ST3i8_POST:
  case AArch64::ST4Fourv16b_POST:
  case AArch64::ST4Fourv2d_POST:
  case AArch64::ST4Fourv2s_POST:
  case AArch64::ST4Fourv4h_POST:
  case AArch64::ST4Fourv4s_POST:
  case AArch64::ST4Fourv8b_POST:
  case AArch64::ST4Fourv8h_POST:
  case AArch64::ST4i16_POST:
  case AArch64::ST4i32_POST:
  case AArch64::ST4i64_POST:
  case AArch64::ST4i8_POST:
  case AArch64::STGPostIndex:
  case AArch64::STGPpost:
  case AArch64::STPDpost:
  case AArch64::STPQpost:
  case AArch64::STPSpost:
  case AArch64::STPWpost:
  case AArch64::STPXpost:
  case AArch64::STRBBpost:
  case AArch64::STRBpost:
  case AArch64::STRDpost:
  case AArch64::STRHHpost:
  case AArch64::STRHpost:
  case AArch64::STRQpost:
  case AArch64::STRSpost:
  case AArch64::STRWpost:
  case AArch64::STRXpost:
  case AArch64::STZ2GPostIndex:
  case AArch64::STZGPostIndex:
    return true;
  }
}
 
bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
    const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
    bool &OffsetIsScalable, TypeSize &Width,
    const TargetRegisterInfo *TRI) const {
  assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
  // Handle only loads/stores with base register followed by immediate offset.
  if (LdSt.getNumExplicitOperands() == 3) {
    // Non-paired instruction (e.g., ldr x1, [x0, #8]).
    if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
        !LdSt.getOperand(2).isImm())
      return false;
  } else if (LdSt.getNumExplicitOperands() == 4) {
    // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
    if (!LdSt.getOperand(1).isReg() ||
        (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()) ||
        !LdSt.getOperand(3).isImm())
      return false;
  } else
    return false;
 
  // Get the scaling factor for the instruction and set the width for the
  // instruction.
  TypeSize Scale(0U, false);
  int64_t Dummy1, Dummy2;
 
  // If this returns false, then it's an instruction we don't want to handle.
  if (!getMemOpInfo(LdSt.getOpcode(), Scale, Width, Dummy1, Dummy2))
    return false;
 
  // Compute the offset. Offset is calculated as the immediate operand
  // multiplied by the scaling factor. Unscaled instructions have scaling factor
  // set to 1. Postindex are a special case which have an offset of 0.
  if (isPostIndexLdStOpcode(LdSt.getOpcode())) {
    BaseOp = &LdSt.getOperand(2);
    Offset = 0;
  } else if (LdSt.getNumExplicitOperands() == 3) {
    BaseOp = &LdSt.getOperand(1);
    Offset = LdSt.getOperand(2).getImm() * Scale.getKnownMinValue();
  } else {
    assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
    BaseOp = &LdSt.getOperand(2);
    Offset = LdSt.getOperand(3).getImm() * Scale.getKnownMinValue();
  }
  OffsetIsScalable = Scale.isScalable();
 
  return BaseOp->isReg() || BaseOp->isFI();
}
 
MachineOperand &
AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
  assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
  MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1);
  assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
  return OfsOp;
}
 
bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
                                    TypeSize &Width, int64_t &MinOffset,
                                    int64_t &MaxOffset) {
  switch (Opcode) {
  // Not a memory operation or something we want to handle.
  default:
    Scale = Width = TypeSize::getFixed(0);
    MinOffset = MaxOffset = 0;
    return false;
  // LDR / STR
  case AArch64::LDRQui:
  case AArch64::STRQui:
    Scale = Width = TypeSize::getFixed(16);
    MinOffset = 0;
    MaxOffset = 4095;
    break;
  case AArch64::LDRXui:
  case AArch64::LDRDui:
  case AArch64::STRXui:
  case AArch64::STRDui:
  case AArch64::PRFMui:
    Scale = Width = TypeSize::getFixed(8);
    MinOffset = 0;
    MaxOffset = 4095;
    break;
  case AArch64::LDRWui:
  case AArch64::LDRSui:
  case AArch64::LDRSWui:
  case AArch64::STRWui:
  case AArch64::STRSui:
    Scale = Width = TypeSize::getFixed(4);
    MinOffset = 0;
    MaxOffset = 4095;
    break;
  case AArch64::LDRHui:
  case AArch64::LDRHHui:
  case AArch64::LDRSHWui:
  case AArch64::LDRSHXui:
  case AArch64::STRHui:
  case AArch64::STRHHui:
    Scale = Width = TypeSize::getFixed(2);
    MinOffset = 0;
    MaxOffset = 4095;
    break;
  case AArch64::LDRBui:
  case AArch64::LDRBBui:
  case AArch64::LDRSBWui:
  case AArch64::LDRSBXui:
  case AArch64::STRBui:
  case AArch64::STRBBui:
    Scale = Width = TypeSize::getFixed(1);
    MinOffset = 0;
    MaxOffset = 4095;
    break;
  // post/pre inc
  case AArch64::STRQpre:
  case AArch64::LDRQpost:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(16);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDRDpost:
  case AArch64::LDRDpre:
  case AArch64::LDRXpost:
  case AArch64::LDRXpre:
  case AArch64::STRDpost:
  case AArch64::STRDpre:
  case AArch64::STRXpost:
  case AArch64::STRXpre:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(8);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::STRWpost:
  case AArch64::STRWpre:
  case AArch64::LDRWpost:
  case AArch64::LDRWpre:
  case AArch64::STRSpost:
  case AArch64::STRSpre:
  case AArch64::LDRSpost:
  case AArch64::LDRSpre:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(4);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDRHpost:
  case AArch64::LDRHpre:
  case AArch64::STRHpost:
  case AArch64::STRHpre:
  case AArch64::LDRHHpost:
  case AArch64::LDRHHpre:
  case AArch64::STRHHpost:
  case AArch64::STRHHpre:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(2);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDRBpost:
  case AArch64::LDRBpre:
  case AArch64::STRBpost:
  case AArch64::STRBpre:
  case AArch64::LDRBBpost:
  case AArch64::LDRBBpre:
  case AArch64::STRBBpost:
  case AArch64::STRBBpre:
    Scale = Width = TypeSize::getFixed(1);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  // Unscaled
  case AArch64::LDURQi:
  case AArch64::STURQi:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(16);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDURXi:
  case AArch64::LDURDi:
  case AArch64::LDAPURXi:
  case AArch64::STURXi:
  case AArch64::STURDi:
  case AArch64::STLURXi:
  case AArch64::PRFUMi:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(8);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDURWi:
  case AArch64::LDURSi:
  case AArch64::LDURSWi:
  case AArch64::LDAPURi:
  case AArch64::LDAPURSWi:
  case AArch64::STURWi:
  case AArch64::STURSi:
  case AArch64::STLURWi:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(4);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDURHi:
  case AArch64::LDURHHi:
  case AArch64::LDURSHXi:
  case AArch64::LDURSHWi:
  case AArch64::LDAPURHi:
  case AArch64::LDAPURSHWi:
  case AArch64::LDAPURSHXi:
  case AArch64::STURHi:
  case AArch64::STURHHi:
  case AArch64::STLURHi:
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(2);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDURBi:
  case AArch64::LDURBBi:
  case AArch64::LDURSBXi:
  case AArch64::LDURSBWi:
  case AArch64::LDAPURBi:
  case AArch64::LDAPURSBWi:
  case AArch64::LDAPURSBXi:
  case AArch64::STURBi:
  case AArch64::STURBBi:
  case AArch64::STLURBi:
    Scale = Width = TypeSize::getFixed(1);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  // LDP / STP (including pre/post inc)
  case AArch64::LDPQi:
  case AArch64::LDNPQi:
  case AArch64::STPQi:
  case AArch64::STNPQi:
  case AArch64::LDPQpost:
  case AArch64::LDPQpre:
  case AArch64::STPQpost:
  case AArch64::STPQpre:
    Scale = TypeSize::getFixed(16);
    Width = TypeSize::getFixed(16 * 2);
    MinOffset = -64;
    MaxOffset = 63;
    break;
  case AArch64::LDPXi:
  case AArch64::LDPDi:
  case AArch64::LDNPXi:
  case AArch64::LDNPDi:
  case AArch64::STPXi:
  case AArch64::STPDi:
  case AArch64::STNPXi:
  case AArch64::STNPDi:
  case AArch64::LDPDpost:
  case AArch64::LDPDpre:
  case AArch64::LDPXpost:
  case AArch64::LDPXpre:
  case AArch64::STPDpost:
  case AArch64::STPDpre:
  case AArch64::STPXpost:
  case AArch64::STPXpre:
    Scale = TypeSize::getFixed(8);
    Width = TypeSize::getFixed(8 * 2);
    MinOffset = -64;
    MaxOffset = 63;
    break;
  case AArch64::LDPWi:
  case AArch64::LDPSi:
  case AArch64::LDNPWi:
  case AArch64::LDNPSi:
  case AArch64::STPWi:
  case AArch64::STPSi:
  case AArch64::STNPWi:
  case AArch64::STNPSi:
  case AArch64::LDPSpost:
  case AArch64::LDPSpre:
  case AArch64::LDPWpost:
  case AArch64::LDPWpre:
  case AArch64::STPSpost:
  case AArch64::STPSpre:
  case AArch64::STPWpost:
  case AArch64::STPWpre:
    Scale = TypeSize::getFixed(4);
    Width = TypeSize::getFixed(4 * 2);
    MinOffset = -64;
    MaxOffset = 63;
    break;
  case AArch64::StoreSwiftAsyncContext:
    // Store is an STRXui, but there might be an ADDXri in the expansion too.
    Scale = TypeSize::getFixed(1);
    Width = TypeSize::getFixed(8);
    MinOffset = 0;
    MaxOffset = 4095;
    break;
  case AArch64::ADDG:
    Scale = TypeSize::getFixed(16);
    Width = TypeSize::getFixed(0);
    MinOffset = 0;
    MaxOffset = 63;
    break;
  case AArch64::TAGPstack:
    Scale = TypeSize::getFixed(16);
    Width = TypeSize::getFixed(0);
    // TAGP with a negative offset turns into SUBP, which has a maximum offset
    // of 63 (not 64!).
    MinOffset = -63;
    MaxOffset = 63;
    break;
  case AArch64::LDG:
  case AArch64::STGi:
  case AArch64::STGPreIndex:
  case AArch64::STGPostIndex:
  case AArch64::STZGi:
  case AArch64::STZGPreIndex:
  case AArch64::STZGPostIndex:
    Scale = Width = TypeSize::getFixed(16);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  // SVE
  case AArch64::STR_ZZZZXI:
  case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
  case AArch64::LDR_ZZZZXI:
  case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
    Scale = TypeSize::getScalable(16);
    Width = TypeSize::getScalable(16 * 4);
    MinOffset = -256;
    MaxOffset = 252;
    break;
  case AArch64::STR_ZZZXI:
  case AArch64::LDR_ZZZXI:
    Scale = TypeSize::getScalable(16);
    Width = TypeSize::getScalable(16 * 3);
    MinOffset = -256;
    MaxOffset = 253;
    break;
  case AArch64::STR_ZZXI:
  case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
  case AArch64::LDR_ZZXI:
  case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
    Scale = TypeSize::getScalable(16);
    Width = TypeSize::getScalable(16 * 2);
    MinOffset = -256;
    MaxOffset = 254;
    break;
  case AArch64::LDR_PXI:
  case AArch64::STR_PXI:
    Scale = Width = TypeSize::getScalable(2);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LDR_PPXI:
  case AArch64::STR_PPXI:
    Scale = TypeSize::getScalable(2);
    Width = TypeSize::getScalable(2 * 2);
    MinOffset = -256;
    MaxOffset = 254;
    break;
  case AArch64::LDR_ZXI:
  case AArch64::STR_ZXI:
    Scale = Width = TypeSize::getScalable(16);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::LD1B_IMM:
  case AArch64::LD1H_IMM:
  case AArch64::LD1W_IMM:
  case AArch64::LD1D_IMM:
  case AArch64::LDNT1B_ZRI:
  case AArch64::LDNT1H_ZRI:
  case AArch64::LDNT1W_ZRI:
  case AArch64::LDNT1D_ZRI:
  case AArch64::ST1B_IMM:
  case AArch64::ST1H_IMM:
  case AArch64::ST1W_IMM:
  case AArch64::ST1D_IMM:
  case AArch64::STNT1B_ZRI:
  case AArch64::STNT1H_ZRI:
  case AArch64::STNT1W_ZRI:
  case AArch64::STNT1D_ZRI:
  case AArch64::LDNF1B_IMM:
  case AArch64::LDNF1H_IMM:
  case AArch64::LDNF1W_IMM:
  case AArch64::LDNF1D_IMM:
    // A full vectors worth of data
    // Width = mbytes * elements
    Scale = Width = TypeSize::getScalable(16);
    MinOffset = -8;
    MaxOffset = 7;
    break;
  case AArch64::LD2B_IMM:
  case AArch64::LD2H_IMM:
  case AArch64::LD2W_IMM:
  case AArch64::LD2D_IMM:
  case AArch64::ST2B_IMM:
  case AArch64::ST2H_IMM:
  case AArch64::ST2W_IMM:
  case AArch64::ST2D_IMM:
  case AArch64::LD1B_2Z_IMM:
  case AArch64::LD1B_2Z_STRIDED_IMM:
  case AArch64::LD1H_2Z_IMM:
  case AArch64::LD1H_2Z_STRIDED_IMM:
  case AArch64::LD1W_2Z_IMM:
  case AArch64::LD1W_2Z_STRIDED_IMM:
  case AArch64::LD1D_2Z_IMM:
  case AArch64::LD1D_2Z_STRIDED_IMM:
  case AArch64::LD1B_2Z_IMM_PSEUDO:
  case AArch64::LD1H_2Z_IMM_PSEUDO:
  case AArch64::LD1W_2Z_IMM_PSEUDO:
  case AArch64::LD1D_2Z_IMM_PSEUDO:
  case AArch64::ST1B_2Z_IMM:
  case AArch64::ST1B_2Z_STRIDED_IMM:
  case AArch64::ST1H_2Z_IMM:
  case AArch64::ST1H_2Z_STRIDED_IMM:
  case AArch64::ST1W_2Z_IMM:
  case AArch64::ST1W_2Z_STRIDED_IMM:
  case AArch64::ST1D_2Z_IMM:
  case AArch64::ST1D_2Z_STRIDED_IMM:
  case AArch64::LDNT1B_2Z_IMM_PSEUDO:
  case AArch64::LDNT1B_2Z_IMM:
  case AArch64::LDNT1B_2Z_STRIDED_IMM:
  case AArch64::LDNT1H_2Z_IMM_PSEUDO:
  case AArch64::LDNT1H_2Z_IMM:
  case AArch64::LDNT1H_2Z_STRIDED_IMM:
  case AArch64::LDNT1W_2Z_IMM_PSEUDO:
  case AArch64::LDNT1W_2Z_IMM:
  case AArch64::LDNT1W_2Z_STRIDED_IMM:
  case AArch64::LDNT1D_2Z_IMM_PSEUDO:
  case AArch64::LDNT1D_2Z_IMM:
  case AArch64::LDNT1D_2Z_STRIDED_IMM:
  case AArch64::STNT1B_2Z_IMM:
  case AArch64::STNT1B_2Z_STRIDED_IMM:
  case AArch64::STNT1H_2Z_IMM:
  case AArch64::STNT1H_2Z_STRIDED_IMM:
  case AArch64::STNT1W_2Z_IMM:
  case AArch64::STNT1W_2Z_STRIDED_IMM:
  case AArch64::STNT1D_2Z_IMM:
  case AArch64::STNT1D_2Z_STRIDED_IMM:
  case AArch64::ST1B_2Z_IMM_PSEUDO:
  case AArch64::ST1H_2Z_IMM_PSEUDO:
  case AArch64::ST1W_2Z_IMM_PSEUDO:
  case AArch64::ST1D_2Z_IMM_PSEUDO:
  case AArch64::STNT1B_2Z_IMM_PSEUDO:
  case AArch64::STNT1H_2Z_IMM_PSEUDO:
  case AArch64::STNT1W_2Z_IMM_PSEUDO:
  case AArch64::STNT1D_2Z_IMM_PSEUDO:
    Scale = Width = TypeSize::getScalable(16 * 2);
    MinOffset = -8;
    MaxOffset = 7;
    break;
  case AArch64::LD3B_IMM:
  case AArch64::LD3H_IMM:
  case AArch64::LD3W_IMM:
  case AArch64::LD3D_IMM:
  case AArch64::ST3B_IMM:
  case AArch64::ST3H_IMM:
  case AArch64::ST3W_IMM:
  case AArch64::ST3D_IMM:
    Scale = Width = TypeSize::getScalable(16 * 3);
    MinOffset = -8;
    MaxOffset = 7;
    break;
  case AArch64::LD4B_IMM:
  case AArch64::LD4H_IMM:
  case AArch64::LD4W_IMM:
  case AArch64::LD4D_IMM:
  case AArch64::ST4B_IMM:
  case AArch64::ST4H_IMM:
  case AArch64::ST4W_IMM:
  case AArch64::ST4D_IMM:
  case AArch64::LD1B_4Z_IMM:
  case AArch64::LD1B_4Z_STRIDED_IMM:
  case AArch64::LD1H_4Z_IMM:
  case AArch64::LD1H_4Z_STRIDED_IMM:
  case AArch64::LD1W_4Z_IMM:
  case AArch64::LD1W_4Z_STRIDED_IMM:
  case AArch64::LD1D_4Z_IMM:
  case AArch64::LD1D_4Z_STRIDED_IMM:
  case AArch64::LD1B_4Z_IMM_PSEUDO:
  case AArch64::LD1H_4Z_IMM_PSEUDO:
  case AArch64::LD1W_4Z_IMM_PSEUDO:
  case AArch64::LD1D_4Z_IMM_PSEUDO:
  case AArch64::ST1B_4Z_IMM:
  case AArch64::ST1B_4Z_STRIDED_IMM:
  case AArch64::ST1H_4Z_IMM:
  case AArch64::ST1H_4Z_STRIDED_IMM:
  case AArch64::ST1W_4Z_IMM:
  case AArch64::ST1W_4Z_STRIDED_IMM:
  case AArch64::ST1D_4Z_IMM:
  case AArch64::ST1D_4Z_STRIDED_IMM:
  case AArch64::LDNT1B_4Z_IMM_PSEUDO:
  case AArch64::LDNT1B_4Z_IMM:
  case AArch64::LDNT1B_4Z_STRIDED_IMM:
  case AArch64::LDNT1H_4Z_IMM_PSEUDO:
  case AArch64::LDNT1H_4Z_IMM:
  case AArch64::LDNT1H_4Z_STRIDED_IMM:
  case AArch64::LDNT1W_4Z_IMM_PSEUDO:
  case AArch64::LDNT1W_4Z_IMM:
  case AArch64::LDNT1W_4Z_STRIDED_IMM:
  case AArch64::LDNT1D_4Z_IMM_PSEUDO:
  case AArch64::LDNT1D_4Z_IMM:
  case AArch64::LDNT1D_4Z_STRIDED_IMM:
  case AArch64::STNT1B_4Z_IMM:
  case AArch64::STNT1B_4Z_STRIDED_IMM:
  case AArch64::STNT1H_4Z_IMM:
  case AArch64::STNT1H_4Z_STRIDED_IMM:
  case AArch64::STNT1W_4Z_IMM:
  case AArch64::STNT1W_4Z_STRIDED_IMM:
  case AArch64::STNT1D_4Z_IMM:
  case AArch64::STNT1D_4Z_STRIDED_IMM:
  case AArch64::ST1B_4Z_IMM_PSEUDO:
  case AArch64::ST1H_4Z_IMM_PSEUDO:
  case AArch64::ST1W_4Z_IMM_PSEUDO:
  case AArch64::ST1D_4Z_IMM_PSEUDO:
  case AArch64::STNT1B_4Z_IMM_PSEUDO:
  case AArch64::STNT1H_4Z_IMM_PSEUDO:
  case AArch64::STNT1W_4Z_IMM_PSEUDO:
  case AArch64::STNT1D_4Z_IMM_PSEUDO:
    Scale = Width = TypeSize::getScalable(16 * 4);
    MinOffset = -8;
    MaxOffset = 7;
    break;
  case AArch64::LD1B_H_IMM:
  case AArch64::LD1SB_H_IMM:
  case AArch64::LD1H_S_IMM:
  case AArch64::LD1SH_S_IMM:
  case AArch64::LD1W_D_IMM:
  case AArch64::LD1SW_D_IMM:
  case AArch64::ST1B_H_IMM:
  case AArch64::ST1H_S_IMM:
  case AArch64::ST1W_D_IMM:
  case AArch64::LDNF1B_H_IMM:
  case AArch64::LDNF1SB_H_IMM:
  case AArch64::LDNF1H_S_IMM:
  case AArch64::LDNF1SH_S_IMM:
  case AArch64::LDNF1W_D_IMM:
  case AArch64::LDNF1SW_D_IMM:
    // A half vector worth of data
    // Width = mbytes * elements
    Scale = Width = TypeSize::getScalable(8);
    MinOffset = -8;
    MaxOffset = 7;
    break;
  case AArch64::LD1B_S_IMM:
  case AArch64::LD1SB_S_IMM:
  case AArch64::LD1H_D_IMM:
  case AArch64::LD1SH_D_IMM:
  case AArch64::ST1B_S_IMM:
  case AArch64::ST1H_D_IMM:
  case AArch64::LDNF1B_S_IMM:
  case AArch64::LDNF1SB_S_IMM:
  case AArch64::LDNF1H_D_IMM:
  case AArch64::LDNF1SH_D_IMM:
    // A quarter vector worth of data
    // Width = mbytes * elements
    Scale = Width = TypeSize::getScalable(4);
    MinOffset = -8;
    MaxOffset = 7;
    break;
  case AArch64::LD1B_D_IMM:
  case AArch64::LD1SB_D_IMM:
  case AArch64::ST1B_D_IMM:
  case AArch64::LDNF1B_D_IMM:
  case AArch64::LDNF1SB_D_IMM:
    // A eighth vector worth of data
    // Width = mbytes * elements
    Scale = Width = TypeSize::getScalable(2);
    MinOffset = -8;
    MaxOffset = 7;
    break;
  case AArch64::ST2Gi:
  case AArch64::ST2GPreIndex:
  case AArch64::ST2GPostIndex:
  case AArch64::STZ2Gi:
  case AArch64::STZ2GPreIndex:
  case AArch64::STZ2GPostIndex:
    Scale = TypeSize::getFixed(16);
    Width = TypeSize::getFixed(32);
    MinOffset = -256;
    MaxOffset = 255;
    break;
  case AArch64::STGPi:
  case AArch64::STGPpost:
  case AArch64::STGPpre:
    Scale = Width = TypeSize::getFixed(16);
    MinOffset = -64;
    MaxOffset = 63;
    break;
  case AArch64::LD1RB_IMM:
  case AArch64::LD1RB_H_IMM:
  case AArch64::LD1RB_S_IMM:
  case AArch64::LD1RB_D_IMM:
  case AArch64::LD1RSB_H_IMM:
  case AArch64::LD1RSB_S_IMM:
  case AArch64::LD1RSB_D_IMM:
    Scale = Width = TypeSize::getFixed(1);
    MinOffset = 0;
    MaxOffset = 63;
    break;
  case AArch64::LD1RH_IMM:
  case AArch64::LD1RH_S_IMM:
  case AArch64::LD1RH_D_IMM:
  case AArch64::LD1RSH_S_IMM:
  case AArch64::LD1RSH_D_IMM:
    Scale = Width = TypeSize::getFixed(2);
    MinOffset = 0;
    MaxOffset = 63;
    break;
  case AArch64::LD1RW_IMM:
  case AArch64::LD1RW_D_IMM:
  case AArch64::LD1RSW_IMM:
    Scale = Width = TypeSize::getFixed(4);
    MinOffset = 0;
    MaxOffset = 63;
    break;
  case AArch64::LD1RD_IMM:
    Scale = Width = TypeSize::getFixed(8);
    MinOffset = 0;
    MaxOffset = 63;
    break;
  }
 
  return true;
}
 
// Scaling factor for unscaled load or store.
int AArch64InstrInfo::getMemScale(unsigned Opc) {
  switch (Opc) {
  default:
    llvm_unreachable("Opcode has unknown scale!");
  case AArch64::LDRBui:
  case AArch64::LDRBBui:
  case AArch64::LDURBBi:
  case AArch64::LDRSBWui:
  case AArch64::LDURSBWi:
  case AArch64::STRBui:
  case AArch64::STRBBui:
  case AArch64::STURBBi:
    return 1;
  case AArch64::LDRHui:
  case AArch64::LDRHHui:
  case AArch64::LDURHHi:
  case AArch64::LDRSHWui:
  case AArch64::LDURSHWi:
  case AArch64::STRHui:
  case AArch64::STRHHui:
  case AArch64::STURHHi:
    return 2;
  case AArch64::LDRSui:
  case AArch64::LDURSi:
  case AArch64::LDRSpre:
  case AArch64::LDRSWui:
  case AArch64::LDURSWi:
  case AArch64::LDRSWpre:
  case AArch64::LDRWpre:
  case AArch64::LDRWui:
  case AArch64::LDURWi:
  case AArch64::STRSui:
  case AArch64::STURSi:
  case AArch64::STRSpre:
  case AArch64::STRWui:
  case AArch64::STURWi:
  case AArch64::STRWpre:
  case AArch64::LDPSi:
  case AArch64::LDPSWi:
  case AArch64::LDPWi:
  case AArch64::STPSi:
  case AArch64::STPWi:
    return 4;
  case AArch64::LDRDui:
  case AArch64::LDURDi:
  case AArch64::LDRDpre:
  case AArch64::LDRXui:
  case AArch64::LDURXi:
  case AArch64::LDRXpre:
  case AArch64::STRDui:
  case AArch64::STURDi:
  case AArch64::STRDpre:
  case AArch64::STRXui:
  case AArch64::STURXi:
  case AArch64::STRXpre:
  case AArch64::LDPDi:
  case AArch64::LDPXi:
  case AArch64::STPDi:
  case AArch64::STPXi:
    return 8;
  case AArch64::LDRQui:
  case AArch64::LDURQi:
  case AArch64::STRQui:
  case AArch64::STURQi:
  case AArch64::STRQpre:
  case AArch64::LDPQi:
  case AArch64::LDRQpre:
  case AArch64::STPQi:
  case AArch64::STGi:
  case AArch64::STZGi:
  case AArch64::ST2Gi:
  case AArch64::STZ2Gi:
  case AArch64::STGPi:
    return 16;
  }
}
 
bool AArch64InstrInfo::isPreLd(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    return false;
  case AArch64::LDRWpre:
  case AArch64::LDRXpre:
  case AArch64::LDRSWpre:
  case AArch64::LDRSpre:
  case AArch64::LDRDpre:
  case AArch64::LDRQpre:
    return true;
  }
}
 
bool AArch64InstrInfo::isPreSt(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    return false;
  case AArch64::STRWpre:
  case AArch64::STRXpre:
  case AArch64::STRSpre:
  case AArch64::STRDpre:
  case AArch64::STRQpre:
    return true;
  }
}
 
bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
  return isPreLd(MI) || isPreSt(MI);
}
 
bool AArch64InstrInfo::isZExtLoad(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    return false;
  case AArch64::LDURBBi:
  case AArch64::LDURHHi:
  case AArch64::LDURWi:
  case AArch64::LDRBBui:
  case AArch64::LDRHHui:
  case AArch64::LDRWui:
  case AArch64::LDRBBroX:
  case AArch64::LDRHHroX:
  case AArch64::LDRWroX:
  case AArch64::LDRBBroW:
  case AArch64::LDRHHroW:
  case AArch64::LDRWroW:
    return true;
  }
}
 
bool AArch64InstrInfo::isSExtLoad(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    return false;
  case AArch64::LDURSBWi:
  case AArch64::LDURSHWi:
  case AArch64::LDURSBXi:
  case AArch64::LDURSHXi:
  case AArch64::LDURSWi:
  case AArch64::LDRSBWui:
  case AArch64::LDRSHWui:
  case AArch64::LDRSBXui:
  case AArch64::LDRSHXui:
  case AArch64::LDRSWui:
  case AArch64::LDRSBWroX:
  case AArch64::LDRSHWroX:
  case AArch64::LDRSBXroX:
  case AArch64::LDRSHXroX:
  case AArch64::LDRSWroX:
  case AArch64::LDRSBWroW:
  case AArch64::LDRSHWroW:
  case AArch64::LDRSBXroW:
  case AArch64::LDRSHXroW:
  case AArch64::LDRSWroW:
    return true;
  }
}
 
bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    return false;
  case AArch64::LDPSi:
  case AArch64::LDPSWi:
  case AArch64::LDPDi:
  case AArch64::LDPQi:
  case AArch64::LDPWi:
  case AArch64::LDPXi:
  case AArch64::STPSi:
  case AArch64::STPDi:
  case AArch64::STPQi:
  case AArch64::STPWi:
  case AArch64::STPXi:
  case AArch64::STGPi:
    return true;
  }
}
 
const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
  assert(MI.mayLoadOrStore() && "Load or store instruction expected");
  unsigned Idx =
      AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
                                                                            : 1;
  return MI.getOperand(Idx);
}
 
const MachineOperand &
AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
  assert(MI.mayLoadOrStore() && "Load or store instruction expected");
  unsigned Idx =
      AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
                                                                            : 2;
  return MI.getOperand(Idx);
}
 
const MachineOperand &
AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode");
  case AArch64::LDRBroX:
  case AArch64::LDRBBroX:
  case AArch64::LDRSBXroX:
  case AArch64::LDRSBWroX:
  case AArch64::LDRHroX:
  case AArch64::LDRHHroX:
  case AArch64::LDRSHXroX:
  case AArch64::LDRSHWroX:
  case AArch64::LDRWroX:
  case AArch64::LDRSroX:
  case AArch64::LDRSWroX:
  case AArch64::LDRDroX:
  case AArch64::LDRXroX:
  case AArch64::LDRQroX:
    return MI.getOperand(4);
  }
}
 
static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
                                              Register Reg) {
  if (MI.getParent() == nullptr)
    return nullptr;
  const MachineFunction *MF = MI.getParent()->getParent();
  return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
}
 
bool AArch64InstrInfo::isHForm(const MachineInstr &MI) {
  auto IsHFPR = [&](const MachineOperand &Op) {
    if (!Op.isReg())
      return false;
    auto Reg = Op.getReg();
    if (Reg.isPhysical())
      return AArch64::FPR16RegClass.contains(Reg);
    const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
    return TRC == &AArch64::FPR16RegClass ||
           TRC == &AArch64::FPR16_loRegClass;
  };
  return llvm::any_of(MI.operands(), IsHFPR);
}
 
bool AArch64InstrInfo::isQForm(const MachineInstr &MI) {
  auto IsQFPR = [&](const MachineOperand &Op) {
    if (!Op.isReg())
      return false;
    auto Reg = Op.getReg();
    if (Reg.isPhysical())
      return AArch64::FPR128RegClass.contains(Reg);
    const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
    return TRC == &AArch64::FPR128RegClass ||
           TRC == &AArch64::FPR128_loRegClass;
  };
  return llvm::any_of(MI.operands(), IsQFPR);
}
 
bool AArch64InstrInfo::hasBTISemantics(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  case AArch64::BRK:
  case AArch64::HLT:
  case AArch64::PACIASP:
  case AArch64::PACIBSP:
    // Implicit BTI behavior.
    return true;
  case AArch64::PAUTH_PROLOGUE:
    // PAUTH_PROLOGUE expands to PACI(A|B)SP.
    return true;
  case AArch64::HINT: {
    unsigned Imm = MI.getOperand(0).getImm();
    // Explicit BTI instruction.
    if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
      return true;
    // PACI(A|B)SP instructions.
    if (Imm == 25 || Imm == 27)
      return true;
    return false;
  }
  default:
    return false;
  }
}
 
bool AArch64InstrInfo::isFpOrNEON(Register Reg) {
  if (Reg == 0)
    return false;
  assert(Reg.isPhysical() && "Expected physical register in isFpOrNEON");
  return AArch64::FPR128RegClass.contains(Reg) ||
         AArch64::FPR64RegClass.contains(Reg) ||
         AArch64::FPR32RegClass.contains(Reg) ||
         AArch64::FPR16RegClass.contains(Reg) ||
         AArch64::FPR8RegClass.contains(Reg);
}
 
bool AArch64InstrInfo::isFpOrNEON(const MachineInstr &MI) {
  auto IsFPR = [&](const MachineOperand &Op) {
    if (!Op.isReg())
      return false;
    auto Reg = Op.getReg();
    if (Reg.isPhysical())
      return isFpOrNEON(Reg);
 
    const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
    return TRC == &AArch64::FPR128RegClass ||
           TRC == &AArch64::FPR128_loRegClass ||
           TRC == &AArch64::FPR64RegClass ||
           TRC == &AArch64::FPR64_loRegClass ||
           TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
           TRC == &AArch64::FPR8RegClass;
  };
  return llvm::any_of(MI.operands(), IsFPR);
}
 
// Scale the unscaled offsets.  Returns false if the unscaled offset can't be
// scaled.
static bool scaleOffset(unsigned Opc, int64_t &Offset) {
  int Scale = AArch64InstrInfo::getMemScale(Opc);
 
  // If the byte-offset isn't a multiple of the stride, we can't scale this
  // offset.
  if (Offset % Scale != 0)
    return false;
 
  // Convert the byte-offset used by unscaled into an "element" offset used
  // by the scaled pair load/store instructions.
  Offset /= Scale;
  return true;
}
 
static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
  if (FirstOpc == SecondOpc)
    return true;
  // We can also pair sign-ext and zero-ext instructions.
  switch (FirstOpc) {
  default:
    return false;
  case AArch64::STRSui:
  case AArch64::STURSi:
    return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
  case AArch64::STRDui:
  case AArch64::STURDi:
    return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
  case AArch64::STRQui:
  case AArch64::STURQi:
    return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
  case AArch64::STRWui:
  case AArch64::STURWi:
    return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
  case AArch64::STRXui:
  case AArch64::STURXi:
    return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
  case AArch64::LDRSui:
  case AArch64::LDURSi:
    return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
  case AArch64::LDRDui:
  case AArch64::LDURDi:
    return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
  case AArch64::LDRQui:
  case AArch64::LDURQi:
    return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
  case AArch64::LDRWui:
  case AArch64::LDURWi:
    return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
  case AArch64::LDRSWui:
  case AArch64::LDURSWi:
    return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
  case AArch64::LDRXui:
  case AArch64::LDURXi:
    return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
  }
  // These instructions can't be paired based on their opcodes.
  return false;
}
 
static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
                            int64_t Offset1, unsigned Opcode1, int FI2,
                            int64_t Offset2, unsigned Opcode2) {
  // Accesses through fixed stack object frame indices may access a different
  // fixed stack slot. Check that the object offsets + offsets match.
  if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) {
    int64_t ObjectOffset1 = MFI.getObjectOffset(FI1);
    int64_t ObjectOffset2 = MFI.getObjectOffset(FI2);
    assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
    // Convert to scaled object offsets.
    int Scale1 = AArch64InstrInfo::getMemScale(Opcode1);
    if (ObjectOffset1 % Scale1 != 0)
      return false;
    ObjectOffset1 /= Scale1;
    int Scale2 = AArch64InstrInfo::getMemScale(Opcode2);
    if (ObjectOffset2 % Scale2 != 0)
      return false;
    ObjectOffset2 /= Scale2;
    ObjectOffset1 += Offset1;
    ObjectOffset2 += Offset2;
    return ObjectOffset1 + 1 == ObjectOffset2;
  }
 
  return FI1 == FI2;
}
 
/// Detect opportunities for ldp/stp formation.
///
/// Only called for LdSt for which getMemOperandWithOffset returns true.
bool AArch64InstrInfo::shouldClusterMemOps(
    ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
    bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
    int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
    unsigned NumBytes) const {
  assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
  const MachineOperand &BaseOp1 = *BaseOps1.front();
  const MachineOperand &BaseOp2 = *BaseOps2.front();
  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
  if (BaseOp1.getType() != BaseOp2.getType())
    return false;
 
  assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
         "Only base registers and frame indices are supported.");
 
  // Check for both base regs and base FI.
  if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
    return false;
 
  // Only cluster up to a single pair.
  if (ClusterSize > 2)
    return false;
 
  if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt))
    return false;
 
  // Can we pair these instructions based on their opcodes?
  unsigned FirstOpc = FirstLdSt.getOpcode();
  unsigned SecondOpc = SecondLdSt.getOpcode();
  if (!canPairLdStOpc(FirstOpc, SecondOpc))
    return false;
 
  // Can't merge volatiles or load/stores that have a hint to avoid pair
  // formation, for example.
  if (!isCandidateToMergeOrPair(FirstLdSt) ||
      !isCandidateToMergeOrPair(SecondLdSt))
    return false;
 
  // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
  int64_t Offset1 = FirstLdSt.getOperand(2).getImm();
  if (hasUnscaledLdStOffset(FirstOpc) && !scaleOffset(FirstOpc, Offset1))
    return false;
 
  int64_t Offset2 = SecondLdSt.getOperand(2).getImm();
  if (hasUnscaledLdStOffset(SecondOpc) && !scaleOffset(SecondOpc, Offset2))
    return false;
 
  // Pairwise instructions have a 7-bit signed offset field.
  if (Offset1 > 63 || Offset1 < -64)
    return false;
 
  // The caller should already have ordered First/SecondLdSt by offset.
  // Note: except for non-equal frame index bases
  if (BaseOp1.isFI()) {
    assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
           "Caller should have ordered offsets.");
 
    const MachineFrameInfo &MFI =
        FirstLdSt.getParent()->getParent()->getFrameInfo();
    return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc,
                           BaseOp2.getIndex(), Offset2, SecondOpc);
  }
 
  assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
 
  return Offset1 + 1 == Offset2;
}
 
static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
                                            MCRegister Reg, unsigned SubIdx,
                                            RegState State,
                                            const TargetRegisterInfo *TRI) {
  if (!SubIdx)
    return MIB.addReg(Reg, State);
 
  if (Reg.isPhysical())
    return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
  return MIB.addReg(Reg, State, SubIdx);
}
 
void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator I,
                                        const DebugLoc &DL, MCRegister DestReg,
                                        MCRegister SrcReg, bool KillSrc,
                                        ArrayRef<unsigned> Indices) const {
  assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
  uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
  unsigned NumRegs = Indices.size();
  MCRegister DestSubReg = TRI->getSubReg(DestReg, Indices[0]);
  assert(!AArch64::PNRRegClass.contains(DestSubReg) &&
         "Unexpected predicate tuple copy");
  unsigned MaxRegs = AArch64::PPRRegClass.contains(DestSubReg) ? 15 : 31;
 
  int SubReg = 0, End = NumRegs, Incr = 1;
  // Copy in reverse if a forward copy will clobber the tuple
  if (((DestEncoding - SrcEncoding) & MaxRegs) < NumRegs) {
    SubReg = NumRegs - 1;
    End = -1;
    Incr = -1;
  }
 
  for (; SubReg != End; SubReg += Incr) {
    DestSubReg = TRI->getSubReg(DestReg, Indices[SubReg]);
    MCRegister SrcSubReg = TRI->getSubReg(SrcReg, Indices[SubReg]);
    copyPhysRegImpl(MBB, I, DL, DestSubReg, SrcSubReg, KillSrc);
  }
}
 
void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
                                       MachineBasicBlock::iterator I,
                                       const DebugLoc &DL, MCRegister DestReg,
                                       MCRegister SrcReg, bool KillSrc,
                                       unsigned Opcode, unsigned ZeroReg,
                                       llvm::ArrayRef<unsigned> Indices) const {
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  unsigned NumRegs = Indices.size();
 
#ifndef NDEBUG
  uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
  uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
  assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
         "GPR reg sequences should not be able to overlap");
#endif
 
  for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
    const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
    AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
    MIB.addReg(ZeroReg);
    AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
    MIB.addImm(0);
  }
}
 
/// Returns true if the instruction at I is in a streaming call site region,
/// within a single basic block.
/// A "call site streaming region" starts after smstart and ends at smstop
/// around a call to a streaming function. This walks backward from I.
static bool isInStreamingCallSiteRegion(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator I) {
  MachineFunction &MF = *MBB.getParent();
  AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
  if (!AFI->hasStreamingModeChanges())
    return false;
  // Walk backwards to find smstart/smstop
  for (MachineInstr &MI : reverse(make_range(MBB.begin(), I))) {
    unsigned Opc = MI.getOpcode();
    if (Opc == AArch64::MSRpstatesvcrImm1 || Opc == AArch64::MSRpstatePseudo) {
      // Check if this is SM change (not ZA)
      int64_t PState = MI.getOperand(0).getImm();
      if (PState == AArch64SVCR::SVCRSM || PState == AArch64SVCR::SVCRSMZA) {
        // Operand 1 is 1 for start, 0 for stop
        return MI.getOperand(1).getImm() == 1;
      }
    }
  }
  return false;
}
 
/// Returns true if in a streaming call site region without SME-FA64.
static bool mustAvoidNeonAtMBBI(const AArch64Subtarget &Subtarget,
                                MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator I) {
  return !Subtarget.hasSMEFA64() && isInStreamingCallSiteRegion(MBB, I);
}
 
void AArch64InstrInfo::copyPhysRegImpl(MachineBasicBlock &MBB,
                                       MachineBasicBlock::iterator I,
                                       const DebugLoc &DL, Register DestReg,
                                       Register SrcReg, bool KillSrc,
                                       bool RenamableDest,
                                       bool RenamableSrc) const {
  if (AArch64::GPR32spRegClass.contains(DestReg) &&
      AArch64::GPR32spRegClass.contains(SrcReg)) {
    if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
      // If either operand is WSP, expand to ADD #0.
      if (Subtarget.hasZeroCycleRegMoveGPR64() &&
          !Subtarget.hasZeroCycleRegMoveGPR32()) {
        // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
        MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
                                                     &AArch64::GPR64spRegClass);
        MCRegister SrcRegX = RI.getMatchingSuperReg(SrcReg, AArch64::sub_32,
                                                    &AArch64::GPR64spRegClass);
        // This instruction is reading and writing X registers.  This may upset
        // the register scavenger and machine verifier, so we need to indicate
        // that we are reading an undefined value from SrcRegX, but a proper
        // value from SrcReg.
        BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
            .addReg(SrcRegX, RegState::Undef)
            .addImm(0)
            .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
            .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
        ++NumZCRegMoveInstrsGPR;
      } else {
        BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
            .addReg(SrcReg, getKillRegState(KillSrc))
            .addImm(0)
            .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
        if (Subtarget.hasZeroCycleRegMoveGPR32())
          ++NumZCRegMoveInstrsGPR;
      }
    } else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
               !Subtarget.hasZeroCycleRegMoveGPR32()) {
      // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
      MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
                                                   &AArch64::GPR64spRegClass);
      assert(DestRegX.isValid() && "Destination super-reg not valid");
      MCRegister SrcRegX = RI.getMatchingSuperReg(SrcReg, AArch64::sub_32,
                                                  &AArch64::GPR64spRegClass);
      assert(SrcRegX.isValid() && "Source super-reg not valid");
      // This instruction is reading and writing X registers.  This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegX, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
          .addReg(AArch64::XZR)
          .addReg(SrcRegX, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
      ++NumZCRegMoveInstrsGPR;
    } else {
      // Otherwise, expand to ORR WZR.
      BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
          .addReg(AArch64::WZR)
          .addReg(SrcReg, getKillRegState(KillSrc));
      if (Subtarget.hasZeroCycleRegMoveGPR32())
        ++NumZCRegMoveInstrsGPR;
    }
    return;
  }
 
  // GPR32 zeroing
  if (AArch64::GPR32spRegClass.contains(DestReg) && SrcReg == AArch64::WZR) {
    if (Subtarget.hasZeroCycleZeroingGPR64() &&
        !Subtarget.hasZeroCycleZeroingGPR32()) {
      MCRegister DestRegX = RI.getMatchingSuperReg(DestReg, AArch64::sub_32,
                                                   &AArch64::GPR64spRegClass);
      assert(DestRegX.isValid() && "Destination super-reg not valid");
      BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestRegX)
          .addImm(0)
          .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
      ++NumZCZeroingInstrsGPR;
    } else if (Subtarget.hasZeroCycleZeroingGPR32()) {
      BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg)
          .addImm(0)
          .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
      ++NumZCZeroingInstrsGPR;
    } else {
      BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
          .addReg(AArch64::WZR)
          .addReg(AArch64::WZR);
    }
    return;
  }
 
  if (AArch64::GPR64spRegClass.contains(DestReg) &&
      AArch64::GPR64spRegClass.contains(SrcReg)) {
    if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
      // If either operand is SP, expand to ADD #0.
      BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
          .addReg(SrcReg, getKillRegState(KillSrc))
          .addImm(0)
          .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
      if (Subtarget.hasZeroCycleRegMoveGPR64())
        ++NumZCRegMoveInstrsGPR;
    } else {
      // Otherwise, expand to ORR XZR.
      BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
          .addReg(AArch64::XZR)
          .addReg(SrcReg, getKillRegState(KillSrc));
      if (Subtarget.hasZeroCycleRegMoveGPR64())
        ++NumZCRegMoveInstrsGPR;
    }
    return;
  }
 
  // GPR64 zeroing
  if (AArch64::GPR64spRegClass.contains(DestReg) && SrcReg == AArch64::XZR) {
    if (Subtarget.hasZeroCycleZeroingGPR64()) {
      BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg)
          .addImm(0)
          .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
      ++NumZCZeroingInstrsGPR;
    } else {
      BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
          .addReg(AArch64::XZR)
          .addReg(AArch64::XZR);
    }
    return;
  }
 
  // Copy a Predicate register by ORRing with itself.
  if (AArch64::PPRRegClass.contains(DestReg) &&
      AArch64::PPRRegClass.contains(SrcReg)) {
    assert(Subtarget.isSVEorStreamingSVEAvailable() &&
           "Unexpected SVE register.");
    BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
      .addReg(SrcReg) // Pg
      .addReg(SrcReg)
      .addReg(SrcReg, getKillRegState(KillSrc));
    return;
  }
 
  // Copy a predicate-as-counter register by ORRing with itself as if it
  // were a regular predicate (mask) register.
  bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
  bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
  if (DestIsPNR || SrcIsPNR) {
    auto ToPPR = [](MCRegister R) -> MCRegister {
      return (R - AArch64::PN0) + AArch64::P0;
    };
    MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg.asMCReg();
    MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg.asMCReg();
 
    if (PPRSrcReg != PPRDestReg) {
      auto NewMI = BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), PPRDestReg)
                       .addReg(PPRSrcReg) // Pg
                       .addReg(PPRSrcReg)
                       .addReg(PPRSrcReg, getKillRegState(KillSrc));
      if (DestIsPNR)
        NewMI.addDef(DestReg, RegState::Implicit);
    }
    return;
  }
 
  // Copy a predicate register pair by copying the individual sub-registers.
  if (AArch64::PPR2RegClass.contains(DestReg) &&
      AArch64::PPR2RegClass.contains(SrcReg)) {
    assert(Subtarget.isSVEorStreamingSVEAvailable() &&
           "Unexpected SVE predicate register.");
    static const unsigned Indices[] = {AArch64::psub0, AArch64::psub1};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a Z register by ORRing with itself.
  if (AArch64::ZPRRegClass.contains(DestReg) &&
      AArch64::ZPRRegClass.contains(SrcReg)) {
    assert(Subtarget.isSVEorStreamingSVEAvailable() &&
           "Unexpected SVE register.");
    BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg)
      .addReg(SrcReg)
      .addReg(SrcReg, getKillRegState(KillSrc));
    return;
  }
 
  // Copy a Z register pair by copying the individual sub-registers.
  if ((AArch64::ZPR2RegClass.contains(DestReg) ||
       AArch64::ZPR2StridedOrContiguousRegClass.contains(DestReg)) &&
      (AArch64::ZPR2RegClass.contains(SrcReg) ||
       AArch64::ZPR2StridedOrContiguousRegClass.contains(SrcReg))) {
    assert(Subtarget.isSVEorStreamingSVEAvailable() &&
           "Unexpected SVE register.");
    static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a Z register triple by copying the individual sub-registers.
  if (AArch64::ZPR3RegClass.contains(DestReg) &&
      AArch64::ZPR3RegClass.contains(SrcReg)) {
    assert(Subtarget.isSVEorStreamingSVEAvailable() &&
           "Unexpected SVE register.");
    static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
                                       AArch64::zsub2};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a Z register quad by copying the individual sub-registers.
  if ((AArch64::ZPR4RegClass.contains(DestReg) ||
       AArch64::ZPR4StridedOrContiguousRegClass.contains(DestReg)) &&
      (AArch64::ZPR4RegClass.contains(SrcReg) ||
       AArch64::ZPR4StridedOrContiguousRegClass.contains(SrcReg))) {
    assert(Subtarget.isSVEorStreamingSVEAvailable() &&
           "Unexpected SVE register.");
    static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
                                       AArch64::zsub2, AArch64::zsub3};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a DDDD register quad by copying the individual sub-registers.
  if (AArch64::DDDDRegClass.contains(DestReg) &&
      AArch64::DDDDRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
                                       AArch64::dsub2, AArch64::dsub3};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a DDD register triple by copying the individual sub-registers.
  if (AArch64::DDDRegClass.contains(DestReg) &&
      AArch64::DDDRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
                                       AArch64::dsub2};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a DD register pair by copying the individual sub-registers.
  if (AArch64::DDRegClass.contains(DestReg) &&
      AArch64::DDRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a QQQQ register quad by copying the individual sub-registers.
  if (AArch64::QQQQRegClass.contains(DestReg) &&
      AArch64::QQQQRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
                                       AArch64::qsub2, AArch64::qsub3};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a QQQ register triple by copying the individual sub-registers.
  if (AArch64::QQQRegClass.contains(DestReg) &&
      AArch64::QQQRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
                                       AArch64::qsub2};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  // Copy a QQ register pair by copying the individual sub-registers.
  if (AArch64::QQRegClass.contains(DestReg) &&
      AArch64::QQRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
    copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
    return;
  }
 
  if (AArch64::XSeqPairsClassRegClass.contains(DestReg) &&
      AArch64::XSeqPairsClassRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
    copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs,
                    AArch64::XZR, Indices);
    return;
  }
 
  if (AArch64::WSeqPairsClassRegClass.contains(DestReg) &&
      AArch64::WSeqPairsClassRegClass.contains(SrcReg)) {
    static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
    copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs,
                    AArch64::WZR, Indices);
    return;
  }
 
  if (AArch64::FPR128RegClass.contains(DestReg) &&
      AArch64::FPR128RegClass.contains(SrcReg)) {
    // In streaming regions, NEON is illegal but streaming-SVE is available.
    // Use SVE for copies if we're in a streaming region and SME is available.
    // With +sme-fa64, NEON is legal in streaming mode so we can use it.
    if ((Subtarget.isSVEorStreamingSVEAvailable() &&
         !Subtarget.isNeonAvailable()) ||
        mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
      BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ))
          .addReg(AArch64::Z0 + (DestReg - AArch64::Q0), RegState::Define)
          .addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
          .addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
    } else if (Subtarget.isNeonAvailable()) {
      BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
          .addReg(SrcReg)
          .addReg(SrcReg, getKillRegState(KillSrc));
      if (Subtarget.hasZeroCycleRegMoveFPR128())
        ++NumZCRegMoveInstrsFPR;
    } else {
      BuildMI(MBB, I, DL, get(AArch64::STRQpre))
          .addReg(AArch64::SP, RegState::Define)
          .addReg(SrcReg, getKillRegState(KillSrc))
          .addReg(AArch64::SP)
          .addImm(-16);
      BuildMI(MBB, I, DL, get(AArch64::LDRQpost))
          .addReg(AArch64::SP, RegState::Define)
          .addReg(DestReg, RegState::Define)
          .addReg(AArch64::SP)
          .addImm(16);
    }
    return;
  }
 
  if (AArch64::FPR64RegClass.contains(DestReg) &&
      AArch64::FPR64RegClass.contains(SrcReg)) {
    if (Subtarget.hasZeroCycleRegMoveFPR128() &&
        !Subtarget.hasZeroCycleRegMoveFPR64() &&
        !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
        !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
      MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::dsub,
                                                   &AArch64::FPR128RegClass);
      MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::dsub,
                                                  &AArch64::FPR128RegClass);
      // This instruction is reading and writing Q registers. This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegQ, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
      ++NumZCRegMoveInstrsFPR;
    } else {
      BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
          .addReg(SrcReg, getKillRegState(KillSrc));
      if (Subtarget.hasZeroCycleRegMoveFPR64())
        ++NumZCRegMoveInstrsFPR;
    }
    return;
  }
 
  if (AArch64::FPR32RegClass.contains(DestReg) &&
      AArch64::FPR32RegClass.contains(SrcReg)) {
    if (Subtarget.hasZeroCycleRegMoveFPR128() &&
        !Subtarget.hasZeroCycleRegMoveFPR64() &&
        !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
        !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
      MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
                                                   &AArch64::FPR128RegClass);
      MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
                                                  &AArch64::FPR128RegClass);
      // This instruction is reading and writing Q registers. This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegQ, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
      ++NumZCRegMoveInstrsFPR;
    } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
               !Subtarget.hasZeroCycleRegMoveFPR32()) {
      MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::ssub,
                                                   &AArch64::FPR64RegClass);
      MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::ssub,
                                                  &AArch64::FPR64RegClass);
      // This instruction is reading and writing D registers. This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegD, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestRegD)
          .addReg(SrcRegD, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
      ++NumZCRegMoveInstrsFPR;
    } else {
      BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
          .addReg(SrcReg, getKillRegState(KillSrc));
      if (Subtarget.hasZeroCycleRegMoveFPR32())
        ++NumZCRegMoveInstrsFPR;
    }
    return;
  }
 
  if (AArch64::FPR16RegClass.contains(DestReg) &&
      AArch64::FPR16RegClass.contains(SrcReg)) {
    if (Subtarget.hasZeroCycleRegMoveFPR128() &&
        !Subtarget.hasZeroCycleRegMoveFPR64() &&
        !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
        !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
      MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
                                                   &AArch64::FPR128RegClass);
      MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
                                                  &AArch64::FPR128RegClass);
      // This instruction is reading and writing Q registers. This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegQ, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
    } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
               !Subtarget.hasZeroCycleRegMoveFPR32()) {
      MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
                                                   &AArch64::FPR64RegClass);
      MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
                                                  &AArch64::FPR64RegClass);
      // This instruction is reading and writing D registers. This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegD, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestRegD)
          .addReg(SrcRegD, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
    } else {
      DestReg = RI.getMatchingSuperReg(DestReg, AArch64::hsub,
                                       &AArch64::FPR32RegClass);
      SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::hsub,
                                      &AArch64::FPR32RegClass);
      BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
          .addReg(SrcReg, getKillRegState(KillSrc));
    }
    return;
  }
 
  if (AArch64::FPR8RegClass.contains(DestReg) &&
      AArch64::FPR8RegClass.contains(SrcReg)) {
    if (Subtarget.hasZeroCycleRegMoveFPR128() &&
        !Subtarget.hasZeroCycleRegMoveFPR64() &&
        !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
        !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
      MCRegister DestRegQ = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
                                                   &AArch64::FPR128RegClass);
      MCRegister SrcRegQ = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
                                                  &AArch64::FPR128RegClass);
      // This instruction is reading and writing Q registers. This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegQ, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestRegQ)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcRegQ, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
    } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
               !Subtarget.hasZeroCycleRegMoveFPR32()) {
      MCRegister DestRegD = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
                                                   &AArch64::FPR64RegClass);
      MCRegister SrcRegD = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
                                                  &AArch64::FPR64RegClass);
      // This instruction is reading and writing D registers. This may upset
      // the register scavenger and machine verifier, so we need to indicate
      // that we are reading an undefined value from SrcRegD, but a proper
      // value from SrcReg.
      BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestRegD)
          .addReg(SrcRegD, RegState::Undef)
          .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
    } else {
      DestReg = RI.getMatchingSuperReg(DestReg, AArch64::bsub,
                                       &AArch64::FPR32RegClass);
      SrcReg = RI.getMatchingSuperReg(SrcReg, AArch64::bsub,
                                      &AArch64::FPR32RegClass);
      BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
          .addReg(SrcReg, getKillRegState(KillSrc));
    }
    return;
  }
 
  // Copies between GPR64 and FPR64.
  if (AArch64::FPR64RegClass.contains(DestReg) &&
      AArch64::GPR64RegClass.contains(SrcReg)) {
    if (AArch64::XZR == SrcReg) {
      BuildMI(MBB, I, DL, get(AArch64::FMOVD0), DestReg);
    } else {
      BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
          .addReg(SrcReg, getKillRegState(KillSrc));
    }
    return;
  }
  if (AArch64::GPR64RegClass.contains(DestReg) &&
      AArch64::FPR64RegClass.contains(SrcReg)) {
    BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
        .addReg(SrcReg, getKillRegState(KillSrc));
    return;
  }
  // Copies between GPR32 and FPR32.
  if (AArch64::FPR32RegClass.contains(DestReg) &&
      AArch64::GPR32RegClass.contains(SrcReg)) {
    if (AArch64::WZR == SrcReg) {
      BuildMI(MBB, I, DL, get(AArch64::FMOVS0), DestReg);
    } else {
      BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
          .addReg(SrcReg, getKillRegState(KillSrc));
    }
    return;
  }
  if (AArch64::GPR32RegClass.contains(DestReg) &&
      AArch64::FPR32RegClass.contains(SrcReg)) {
    BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
        .addReg(SrcReg, getKillRegState(KillSrc));
    return;
  }
 
  if (DestReg == AArch64::NZCV) {
    assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
    BuildMI(MBB, I, DL, get(AArch64::MSR))
        .addImm(AArch64SysReg::NZCV)
        .addReg(SrcReg, getKillRegState(KillSrc))
        .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
    return;
  }
 
  if (SrcReg == AArch64::NZCV) {
    assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
    BuildMI(MBB, I, DL, get(AArch64::MRS), DestReg)
        .addImm(AArch64SysReg::NZCV)
        .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
    return;
  }
 
#ifndef NDEBUG
  errs() << RI.getRegAsmName(DestReg) << " = COPY " << RI.getRegAsmName(SrcReg)
         << "\n";
#endif
  llvm_unreachable("unimplemented reg-to-reg copy");
}
 
void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator I,
                                   const DebugLoc &DL, Register DestReg,
                                   Register SrcReg, bool KillSrc,
                                   bool RenamableDest,
                                   bool RenamableSrc) const {
  ++NumCopyInstrs;
  copyPhysRegImpl(MBB, I, DL, DestReg, SrcReg, KillSrc, RenamableDest,
                  RenamableSrc);
  return;
}
 
static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
                                    MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator InsertBefore,
                                    const MCInstrDesc &MCID,
                                    Register SrcReg, bool IsKill,
                                    unsigned SubIdx0, unsigned SubIdx1, int FI,
                                    MachineMemOperand *MMO) {
  Register SrcReg0 = SrcReg;
  Register SrcReg1 = SrcReg;
  if (SrcReg.isPhysical()) {
    SrcReg0 = TRI.getSubReg(SrcReg, SubIdx0);
    SubIdx0 = 0;
    SrcReg1 = TRI.getSubReg(SrcReg, SubIdx1);
    SubIdx1 = 0;
  }
  BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
      .addReg(SrcReg0, getKillRegState(IsKill), SubIdx0)
      .addReg(SrcReg1, getKillRegState(IsKill), SubIdx1)
      .addFrameIndex(FI)
      .addImm(0)
      .addMemOperand(MMO);
}
 
void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator MBBI,
                                           Register SrcReg, bool isKill, int FI,
                                           const TargetRegisterClass *RC,
                                           Register VReg,
                                           MachineInstr::MIFlag Flags) const {
  MachineFunction &MF = *MBB.getParent();
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
  MachineMemOperand *MMO =
      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
                              MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
  unsigned Opc = 0;
  bool Offset = true;
  MCRegister PNRReg = MCRegister::NoRegister;
  unsigned StackID = TargetStackID::Default;
  switch (RI.getSpillSize(*RC)) {
  case 1:
    if (AArch64::FPR8RegClass.hasSubClassEq(RC))
      Opc = AArch64::STRBui;
    break;
  case 2: {
    if (AArch64::FPR16RegClass.hasSubClassEq(RC))
      Opc = AArch64::STRHui;
    else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
             AArch64::PPRRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register store without SVE store instructions");
      Opc = AArch64::STR_PXI;
      StackID = TargetStackID::ScalablePredicateVector;
    }
    break;
  }
  case 4:
    if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
      Opc = AArch64::STRWui;
      if (SrcReg.isVirtual())
        MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
      else
        assert(SrcReg != AArch64::WSP);
    } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
      Opc = AArch64::STRSui;
    else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
      Opc = AArch64::STR_PPXI;
      StackID = TargetStackID::ScalablePredicateVector;
    }
    break;
  case 8:
    if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
      Opc = AArch64::STRXui;
      if (SrcReg.isVirtual())
        MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
      else
        assert(SrcReg != AArch64::SP);
    } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
      Opc = AArch64::STRDui;
    } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
      storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
                              get(AArch64::STPWi), SrcReg, isKill,
                              AArch64::sube32, AArch64::subo32, FI, MMO);
      return;
    }
    break;
  case 16:
    if (AArch64::FPR128RegClass.hasSubClassEq(RC))
      Opc = AArch64::STRQui;
    else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
      Opc = AArch64::ST1Twov1d;
      Offset = false;
    } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
      storeRegPairToStackSlot(getRegisterInfo(), MBB, MBBI,
                              get(AArch64::STPXi), SrcReg, isKill,
                              AArch64::sube64, AArch64::subo64, FI, MMO);
      return;
    } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register store without SVE store instructions");
      Opc = AArch64::STR_ZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  case 24:
    if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
      Opc = AArch64::ST1Threev1d;
      Offset = false;
    }
    break;
  case 32:
    if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
      Opc = AArch64::ST1Fourv1d;
      Offset = false;
    } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
      Opc = AArch64::ST1Twov2d;
      Offset = false;
    } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register store without SVE store instructions");
      Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
      StackID = TargetStackID::ScalableVector;
    } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register store without SVE store instructions");
      Opc = AArch64::STR_ZZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  case 48:
    if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
      Opc = AArch64::ST1Threev2d;
      Offset = false;
    } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register store without SVE store instructions");
      Opc = AArch64::STR_ZZZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  case 64:
    if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
      Opc = AArch64::ST1Fourv2d;
      Offset = false;
    } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register store without SVE store instructions");
      Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
      StackID = TargetStackID::ScalableVector;
    } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register store without SVE store instructions");
      Opc = AArch64::STR_ZZZZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  }
  assert(Opc && "Unknown register class");
  MFI.setStackID(FI, StackID);
 
  const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
                                     .addReg(SrcReg, getKillRegState(isKill))
                                     .addFrameIndex(FI);
 
  if (Offset)
    MI.addImm(0);
  if (PNRReg.isValid())
    MI.addDef(PNRReg, RegState::Implicit);
  MI.addMemOperand(MMO);
}
 
static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
                                     MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator InsertBefore,
                                     const MCInstrDesc &MCID,
                                     Register DestReg, unsigned SubIdx0,
                                     unsigned SubIdx1, int FI,
                                     MachineMemOperand *MMO) {
  Register DestReg0 = DestReg;
  Register DestReg1 = DestReg;
  bool IsUndef = true;
  if (DestReg.isPhysical()) {
    DestReg0 = TRI.getSubReg(DestReg, SubIdx0);
    SubIdx0 = 0;
    DestReg1 = TRI.getSubReg(DestReg, SubIdx1);
    SubIdx1 = 0;
    IsUndef = false;
  }
  BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
      .addReg(DestReg0, RegState::Define | getUndefRegState(IsUndef), SubIdx0)
      .addReg(DestReg1, RegState::Define | getUndefRegState(IsUndef), SubIdx1)
      .addFrameIndex(FI)
      .addImm(0)
      .addMemOperand(MMO);
}
 
void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                            MachineBasicBlock::iterator MBBI,
                                            Register DestReg, int FI,
                                            const TargetRegisterClass *RC,
                                            Register VReg, unsigned SubReg,
                                            MachineInstr::MIFlag Flags) const {
  MachineFunction &MF = *MBB.getParent();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
  MachineMemOperand *MMO =
      MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
                              MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
 
  unsigned Opc = 0;
  bool Offset = true;
  unsigned StackID = TargetStackID::Default;
  Register PNRReg = MCRegister::NoRegister;
  switch (TRI.getSpillSize(*RC)) {
  case 1:
    if (AArch64::FPR8RegClass.hasSubClassEq(RC))
      Opc = AArch64::LDRBui;
    break;
  case 2: {
    bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
    if (AArch64::FPR16RegClass.hasSubClassEq(RC))
      Opc = AArch64::LDRHui;
    else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register load without SVE load instructions");
      if (IsPNR)
        PNRReg = DestReg;
      Opc = AArch64::LDR_PXI;
      StackID = TargetStackID::ScalablePredicateVector;
    }
    break;
  }
  case 4:
    if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
      Opc = AArch64::LDRWui;
      if (DestReg.isVirtual())
        MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
      else
        assert(DestReg != AArch64::WSP);
    } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
      Opc = AArch64::LDRSui;
    else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
      Opc = AArch64::LDR_PPXI;
      StackID = TargetStackID::ScalablePredicateVector;
    }
    break;
  case 8:
    if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
      Opc = AArch64::LDRXui;
      if (DestReg.isVirtual())
        MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
      else
        assert(DestReg != AArch64::SP);
    } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
      Opc = AArch64::LDRDui;
    } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
      loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
                               get(AArch64::LDPWi), DestReg, AArch64::sube32,
                               AArch64::subo32, FI, MMO);
      return;
    }
    break;
  case 16:
    if (AArch64::FPR128RegClass.hasSubClassEq(RC))
      Opc = AArch64::LDRQui;
    else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
      Opc = AArch64::LD1Twov1d;
      Offset = false;
    } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
      loadRegPairFromStackSlot(getRegisterInfo(), MBB, MBBI,
                               get(AArch64::LDPXi), DestReg, AArch64::sube64,
                               AArch64::subo64, FI, MMO);
      return;
    } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register load without SVE load instructions");
      Opc = AArch64::LDR_ZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  case 24:
    if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
      Opc = AArch64::LD1Threev1d;
      Offset = false;
    }
    break;
  case 32:
    if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
      Opc = AArch64::LD1Fourv1d;
      Offset = false;
    } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
      Opc = AArch64::LD1Twov2d;
      Offset = false;
    } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register load without SVE load instructions");
      Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
      StackID = TargetStackID::ScalableVector;
    } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register load without SVE load instructions");
      Opc = AArch64::LDR_ZZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  case 48:
    if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
      Opc = AArch64::LD1Threev2d;
      Offset = false;
    } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register load without SVE load instructions");
      Opc = AArch64::LDR_ZZZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  case 64:
    if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
      Opc = AArch64::LD1Fourv2d;
      Offset = false;
    } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register load without SVE load instructions");
      Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
      StackID = TargetStackID::ScalableVector;
    } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
      assert(Subtarget.isSVEorStreamingSVEAvailable() &&
             "Unexpected register load without SVE load instructions");
      Opc = AArch64::LDR_ZZZZXI;
      StackID = TargetStackID::ScalableVector;
    }
    break;
  }
 
  assert(Opc && "Unknown register class");
  MFI.setStackID(FI, StackID);
 
  const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
                                     .addReg(DestReg, getDefRegState(true))
                                     .addFrameIndex(FI);
  if (Offset)
    MI.addImm(0);
  if (PNRReg.isValid() && !PNRReg.isVirtual())
    MI.addDef(PNRReg, RegState::Implicit);
  MI.addMemOperand(MMO);
}
 
bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
                                           const MachineInstr &UseMI,
                                           const TargetRegisterInfo *TRI) {
  return any_of(instructionsWithoutDebug(std::next(DefMI.getIterator()),
                                         UseMI.getIterator()),
                [TRI](const MachineInstr &I) {
                  return I.modifiesRegister(AArch64::NZCV, TRI) ||
                         I.readsRegister(AArch64::NZCV, TRI);
                });
}
 
void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
    const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
  // The smallest scalable element supported by scaled SVE addressing
  // modes are predicates, which are 2 scalable bytes in size. So the scalable
  // byte offset must always be a multiple of 2.
  assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
 
  // VGSized offsets are divided by '2', because the VG register is the
  // the number of 64bit granules as opposed to 128bit vector chunks,
  // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
  // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
  // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
  ByteSized = Offset.getFixed();
  VGSized = Offset.getScalable() / 2;
}
 
/// Returns the offset in parts to which this frame offset can be
/// decomposed for the purpose of describing a frame offset.
/// For non-scalable offsets this is simply its byte size.
void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
    const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
    int64_t &NumDataVectors) {
  // The smallest scalable element supported by scaled SVE addressing
  // modes are predicates, which are 2 scalable bytes in size. So the scalable
  // byte offset must always be a multiple of 2.
  assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
 
  NumBytes = Offset.getFixed();
  NumDataVectors = 0;
  NumPredicateVectors = Offset.getScalable() / 2;
  // This method is used to get the offsets to adjust the frame offset.
  // If the function requires ADDPL to be used and needs more than two ADDPL
  // instructions, part of the offset is folded into NumDataVectors so that it
  // uses ADDVL for part of it, reducing the number of ADDPL instructions.
  if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
      NumPredicateVectors > 62) {
    NumDataVectors = NumPredicateVectors / 8;
    NumPredicateVectors -= NumDataVectors * 8;
  }
}
 
// Convenience function to create a DWARF expression for: Constant `Operation`.
// This helper emits compact sequences for common cases. For example, for`-15
// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
                               dwarf::LocationAtom Operation) {
  if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
    // -Constant (1 to 31)
    Expr.push_back(dwarf::DW_OP_lit0 - Constant);
    Operation = dwarf::DW_OP_minus;
  } else if (Constant >= 0 && Constant <= 31) {
    // Literal value 0 to 31
    Expr.push_back(dwarf::DW_OP_lit0 + Constant);
  } else {
    // Signed constant
    Expr.push_back(dwarf::DW_OP_consts);
    appendLEB128<LEB128Sign::Signed>(Expr, Constant);
  }
  return Expr.push_back(Operation);
}
 
// Convenience function to create a DWARF expression for a register.
static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
  Expr.push_back((char)dwarf::DW_OP_bregx);
  appendLEB128<LEB128Sign::Unsigned>(Expr, RegNum);
  Expr.push_back(0);
}
 
// Convenience function to create a DWARF expression for loading a register from
// a CFA offset.
static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
                              int64_t OffsetFromDefCFA) {
  // This assumes the top of the DWARF stack contains the CFA.
  Expr.push_back(dwarf::DW_OP_dup);
  // Add the offset to the register.
  appendConstantExpr(Expr, OffsetFromDefCFA, dwarf::DW_OP_plus);
  // Dereference the address (loads a 64 bit value)..
  Expr.push_back(dwarf::DW_OP_deref);
}
 
// Convenience function to create a comment for
//  (+/-) NumBytes (* RegScale)?
static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
                                StringRef RegScale = {}) {
  if (NumBytes) {
    Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes);
    if (!RegScale.empty())
      Comment << ' ' << RegScale;
  }
}
 
// Creates an MCCFIInstruction:
//    { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
                                               unsigned Reg,
                                               const StackOffset &Offset) {
  int64_t NumBytes, NumVGScaledBytes;
  AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(Offset, NumBytes,
                                                        NumVGScaledBytes);
  std::string CommentBuffer;
  llvm::raw_string_ostream Comment(CommentBuffer);
 
  if (Reg == AArch64::SP)
    Comment << "sp";
  else if (Reg == AArch64::FP)
    Comment << "fp";
  else
    Comment << printReg(Reg, &TRI);
 
  // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
  SmallString<64> Expr;
  unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
  assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
  // Reg + NumBytes
  Expr.push_back(dwarf::DW_OP_breg0 + DwarfReg);
  appendLEB128<LEB128Sign::Signed>(Expr, NumBytes);
  appendOffsetComment(NumBytes, Comment);
  if (NumVGScaledBytes) {
    // + VG * NumVGScaledBytes
    appendOffsetComment(NumVGScaledBytes, Comment, "* VG");
    appendReadRegExpr(Expr, TRI.getDwarfRegNum(AArch64::VG, true));
    appendConstantExpr(Expr, NumVGScaledBytes, dwarf::DW_OP_mul);
    Expr.push_back(dwarf::DW_OP_plus);
  }
 
  // Wrap this into DW_CFA_def_cfa.
  SmallString<64> DefCfaExpr;
  DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
  appendLEB128<LEB128Sign::Unsigned>(DefCfaExpr, Expr.size());
  DefCfaExpr.append(Expr.str());
  return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(), SMLoc(),
                                        Comment.str());
}
 
MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
                                    unsigned FrameReg, unsigned Reg,
                                    const StackOffset &Offset,
                                    bool LastAdjustmentWasScalable) {
  if (Offset.getScalable())
    return createDefCFAExpression(TRI, Reg, Offset);
 
  if (FrameReg == Reg && !LastAdjustmentWasScalable)
    return MCCFIInstruction::cfiDefCfaOffset(nullptr, int(Offset.getFixed()));
 
  unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
  return MCCFIInstruction::cfiDefCfa(nullptr, DwarfReg, (int)Offset.getFixed());
}
 
MCCFIInstruction
llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
                      const StackOffset &OffsetFromDefCFA,
                      std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
  int64_t NumBytes, NumVGScaledBytes;
  AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
      OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
 
  unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
 
  // Non-scalable offsets can use DW_CFA_offset directly.
  if (!NumVGScaledBytes)
    return MCCFIInstruction::createOffset(nullptr, DwarfReg, NumBytes);
 
  std::string CommentBuffer;
  llvm::raw_string_ostream Comment(CommentBuffer);
  Comment << printReg(Reg, &TRI) << "  @ cfa";
 
  // Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
  assert(NumVGScaledBytes && "Expected scalable offset");
  SmallString<64> OffsetExpr;
  // + VG * NumVGScaledBytes
  StringRef VGRegScale;
  if (IncomingVGOffsetFromDefCFA) {
    appendLoadRegExpr(OffsetExpr, *IncomingVGOffsetFromDefCFA);
    VGRegScale = "* IncomingVG";
  } else {
    appendReadRegExpr(OffsetExpr, TRI.getDwarfRegNum(AArch64::VG, true));
    VGRegScale = "* VG";
  }
  appendConstantExpr(OffsetExpr, NumVGScaledBytes, dwarf::DW_OP_mul);
  appendOffsetComment(NumVGScaledBytes, Comment, VGRegScale);
  OffsetExpr.push_back(dwarf::DW_OP_plus);
  if (NumBytes) {
    // + NumBytes
    appendOffsetComment(NumBytes, Comment);
    appendConstantExpr(OffsetExpr, NumBytes, dwarf::DW_OP_plus);
  }
 
  // Wrap this into DW_CFA_expression
  SmallString<64> CfaExpr;
  CfaExpr.push_back(dwarf::DW_CFA_expression);
  appendLEB128<LEB128Sign::Unsigned>(CfaExpr, DwarfReg);
  appendLEB128<LEB128Sign::Unsigned>(CfaExpr, OffsetExpr.size());
  CfaExpr.append(OffsetExpr.str());
 
  return MCCFIInstruction::createEscape(nullptr, CfaExpr.str(), SMLoc(),
                                        Comment.str());
}
 
// Helper function to emit a frame offset adjustment from a given
// pointer (SrcReg), stored into DestReg. This function is explicit
// in that it requires the opcode.
static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator MBBI,
                               const DebugLoc &DL, unsigned DestReg,
                               unsigned SrcReg, int64_t Offset, unsigned Opc,
                               const TargetInstrInfo *TII,
                               MachineInstr::MIFlag Flag, bool NeedsWinCFI,
                               bool *HasWinCFI, bool EmitCFAOffset,
                               StackOffset CFAOffset, unsigned FrameReg) {
  int Sign = 1;
  unsigned MaxEncoding, ShiftSize;
  switch (Opc) {
  case AArch64::ADDXri:
  case AArch64::ADDSXri:
  case AArch64::SUBXri:
  case AArch64::SUBSXri:
    MaxEncoding = 0xfff;
    ShiftSize = 12;
    break;
  case AArch64::ADDVL_XXI:
  case AArch64::ADDPL_XXI:
  case AArch64::ADDSVL_XXI:
  case AArch64::ADDSPL_XXI:
    MaxEncoding = 31;
    ShiftSize = 0;
    if (Offset < 0) {
      MaxEncoding = 32;
      Sign = -1;
      Offset = -Offset;
    }
    break;
  default:
    llvm_unreachable("Unsupported opcode");
  }
 
  // `Offset` can be in bytes or in "scalable bytes".
  int VScale = 1;
  if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
    VScale = 16;
  else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
    VScale = 2;
 
  // FIXME: If the offset won't fit in 24-bits, compute the offset into a
  // scratch register.  If DestReg is a virtual register, use it as the
  // scratch register; otherwise, create a new virtual register (to be
  // replaced by the scavenger at the end of PEI).  That case can be optimized
  // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
  // register can be loaded with offset%8 and the add/sub can use an extending
  // instruction with LSL#3.
  // Currently the function handles any offsets but generates a poor sequence
  // of code.
  //  assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
 
  const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
  Register TmpReg = DestReg;
  if (TmpReg == AArch64::XZR)
    TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
        &AArch64::GPR64RegClass);
  do {
    uint64_t ThisVal = std::min<uint64_t>(Offset, MaxEncodableValue);
    unsigned LocalShiftSize = 0;
    if (ThisVal > MaxEncoding) {
      ThisVal = ThisVal >> ShiftSize;
      LocalShiftSize = ShiftSize;
    }
    assert((ThisVal >> ShiftSize) <= MaxEncoding &&
           "Encoding cannot handle value that big");
 
    Offset -= ThisVal << LocalShiftSize;
    if (Offset == 0)
      TmpReg = DestReg;
    auto MBI = BuildMI(MBB, MBBI, DL, TII->get(Opc), TmpReg)
                   .addReg(SrcReg)
                   .addImm(Sign * (int)ThisVal);
    if (ShiftSize)
      MBI = MBI.addImm(
          AArch64_AM::getShifterImm(AArch64_AM::LSL, LocalShiftSize));
    MBI = MBI.setMIFlag(Flag);
 
    auto Change =
        VScale == 1
            ? StackOffset::getFixed(ThisVal << LocalShiftSize)
            : StackOffset::getScalable(VScale * (ThisVal << LocalShiftSize));
    if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
      CFAOffset += Change;
    else
      CFAOffset -= Change;
    if (EmitCFAOffset && DestReg == TmpReg) {
      MachineFunction &MF = *MBB.getParent();
      const TargetSubtargetInfo &STI = MF.getSubtarget();
      const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
 
      unsigned CFIIndex = MF.addFrameInst(
          createDefCFA(TRI, FrameReg, DestReg, CFAOffset, VScale != 1));
      BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
          .addCFIIndex(CFIIndex)
          .setMIFlags(Flag);
    }
 
    if (NeedsWinCFI) {
      int Imm = (int)(ThisVal << LocalShiftSize);
      if (VScale != 1 && DestReg == AArch64::SP) {
        if (HasWinCFI)
          *HasWinCFI = true;
        BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AllocZ))
            .addImm(ThisVal)
            .setMIFlag(Flag);
      } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
                 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
        assert(VScale == 1 && "Expected non-scalable operation");
        if (HasWinCFI)
          *HasWinCFI = true;
        if (Imm == 0)
          BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)).setMIFlag(Flag);
        else
          BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP))
              .addImm(Imm)
              .setMIFlag(Flag);
        assert(Offset == 0 && "Expected remaining offset to be zero to "
                              "emit a single SEH directive");
      } else if (DestReg == AArch64::SP) {
        assert(VScale == 1 && "Expected non-scalable operation");
        if (HasWinCFI)
          *HasWinCFI = true;
        assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
        BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
            .addImm(Imm)
            .setMIFlag(Flag);
      }
    }
 
    SrcReg = TmpReg;
  } while (Offset);
}
 
void llvm::emitFrameOffset(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
                           unsigned DestReg, unsigned SrcReg,
                           StackOffset Offset, const TargetInstrInfo *TII,
                           MachineInstr::MIFlag Flag, bool SetNZCV,
                           bool NeedsWinCFI, bool *HasWinCFI,
                           bool EmitCFAOffset, StackOffset CFAOffset,
                           unsigned FrameReg) {
  // If a function is marked as arm_locally_streaming, then the runtime value of
  // vscale in the prologue/epilogue is different the runtime value of vscale
  // in the function's body. To avoid having to consider multiple vscales,
  // we can use `addsvl` to allocate any scalable stack-slots, which under
  // most circumstances will be only locals, not callee-save slots.
  const Function &F = MBB.getParent()->getFunction();
  bool UseSVL = F.hasFnAttribute("aarch64_pstate_sm_body");
 
  int64_t Bytes, NumPredicateVectors, NumDataVectors;
  AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
      Offset, Bytes, NumPredicateVectors, NumDataVectors);
 
  // Insert ADDSXri for scalable offset at the end.
  bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
  if (NeedsFinalDefNZCV)
    SetNZCV = false;
 
  // First emit non-scalable frame offsets, or a simple 'mov'.
  if (Bytes || (!Offset && SrcReg != DestReg)) {
    assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
           "SP increment/decrement not 8-byte aligned");
    unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
    if (Bytes < 0) {
      Bytes = -Bytes;
      Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
    }
    emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Bytes, Opc, TII, Flag,
                       NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
                       FrameReg);
    CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
                     ? StackOffset::getFixed(-Bytes)
                     : StackOffset::getFixed(Bytes);
    SrcReg = DestReg;
    FrameReg = DestReg;
  }
 
  assert(!(NeedsWinCFI && NumPredicateVectors) &&
         "WinCFI can't allocate fractions of an SVE data vector");
 
  if (NumDataVectors) {
    emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumDataVectors,
                       UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
                       Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
                       FrameReg);
    CFAOffset += StackOffset::getScalable(-NumDataVectors * 16);
    SrcReg = DestReg;
  }
 
  if (NumPredicateVectors) {
    assert(DestReg != AArch64::SP && "Unaligned access to SP");
    emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumPredicateVectors,
                       UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
                       Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
                       FrameReg);
  }
 
  if (NeedsFinalDefNZCV)
    BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDSXri), DestReg)
        .addReg(DestReg)
        .addImm(0)
        .addImm(0);
}
 
MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
    MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
    int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS,
    VirtRegMap *VRM) const {
  MachineBasicBlock::iterator InsertPt = MI;
  // This is a bit of a hack. Consider this instruction:
  //
  //   %0 = COPY %sp; GPR64all:%0
  //
  // We explicitly chose GPR64all for the virtual register so such a copy might
  // be eliminated by RegisterCoalescer. However, that may not be possible, and
  // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
  // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
  //
  // To prevent that, we are going to constrain the %0 register class here.
  if (MI.isFullCopy()) {
    Register DstReg = MI.getOperand(0).getReg();
    Register SrcReg = MI.getOperand(1).getReg();
    if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
      MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
      return nullptr;
    }
    if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
      MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
      return nullptr;
    }
    // Nothing can folded with copy from/to NZCV.
    if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
      return nullptr;
  }
 
  // Handle the case where a copy is being spilled or filled but the source
  // and destination register class don't match.  For example:
  //
  //   %0 = COPY %xzr; GPR64common:%0
  //
  // In this case we can still safely fold away the COPY and generate the
  // following spill code:
  //
  //   STRXui %xzr, %stack.0
  //
  // This also eliminates spilled cross register class COPYs (e.g. between x and
  // d regs) of the same size.  For example:
  //
  //   %0 = COPY %1; GPR64:%0, FPR64:%1
  //
  // will be filled as
  //
  //   LDRDui %0, fi<#0>
  //
  // instead of
  //
  //   LDRXui %Temp, fi<#0>
  //   %0 = FMOV %Temp
  //
  if (MI.isCopy() && Ops.size() == 1 &&
      // Make sure we're only folding the explicit COPY defs/uses.
      (Ops[0] == 0 || Ops[0] == 1)) {
    bool IsSpill = Ops[0] == 0;
    bool IsFill = !IsSpill;
    const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
    const MachineRegisterInfo &MRI = MF.getRegInfo();
    MachineBasicBlock &MBB = *MI.getParent();
    const MachineOperand &DstMO = MI.getOperand(0);
    const MachineOperand &SrcMO = MI.getOperand(1);
    Register DstReg = DstMO.getReg();
    Register SrcReg = SrcMO.getReg();
    // This is slightly expensive to compute for physical regs since
    // getMinimalPhysRegClass is slow.
    auto getRegClass = [&](unsigned Reg) {
      return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
                                              : TRI.getMinimalPhysRegClass(Reg);
    };
 
    if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
      assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
                 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
             "Mismatched register size in non subreg COPY");
      if (IsSpill)
        storeRegToStackSlot(MBB, InsertPt, SrcReg, SrcMO.isKill(), FrameIndex,
                            getRegClass(SrcReg), Register());
      else
        loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex,
                             getRegClass(DstReg), Register());
      return &*--InsertPt;
    }
 
    // Handle cases like spilling def of:
    //
    //   %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
    //
    // where the physical register source can be widened and stored to the full
    // virtual reg destination stack slot, in this case producing:
    //
    //   STRXui %xzr, %stack.0
    //
    if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
        TRI.getRegSizeInBits(*getRegClass(DstReg)) == 64) {
      assert(SrcMO.getSubReg() == 0 &&
             "Unexpected subreg on physical register");
      storeRegToStackSlot(MBB, InsertPt, AArch64::XZR, SrcMO.isKill(),
                          FrameIndex, &AArch64::GPR64RegClass, Register());
      return &*--InsertPt;
    }
 
    // Handle cases like filling use of:
    //
    //   %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
    //
    // where we can load the full virtual reg source stack slot, into the subreg
    // destination, in this case producing:
    //
    //   LDRWui %0:sub_32<def,read-undef>, %stack.0
    //
    if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
      const TargetRegisterClass *FillRC = nullptr;
      switch (DstMO.getSubReg()) {
      default:
        break;
      case AArch64::sub_32:
        if (AArch64::GPR64RegClass.hasSubClassEq(getRegClass(DstReg)))
          FillRC = &AArch64::GPR32RegClass;
        break;
      case AArch64::ssub:
        FillRC = &AArch64::FPR32RegClass;
        break;
      case AArch64::dsub:
        FillRC = &AArch64::FPR64RegClass;
        break;
      }
 
      if (FillRC) {
        assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
                   TRI.getRegSizeInBits(*FillRC) &&
               "Mismatched regclass size on folded subreg COPY");
        loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex, FillRC,
                             Register());
        MachineInstr &LoadMI = *--InsertPt;
        MachineOperand &LoadDst = LoadMI.getOperand(0);
        assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
        LoadDst.setSubReg(DstMO.getSubReg());
        LoadDst.setIsUndef();
        return &LoadMI;
      }
    }
  }
 
  // Cannot fold.
  return nullptr;
}
 
int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
                                    StackOffset &SOffset,
                                    bool *OutUseUnscaledOp,
                                    unsigned *OutUnscaledOp,
                                    int64_t *EmittableOffset) {
  // Set output values in case of early exit.
  if (EmittableOffset)
    *EmittableOffset = 0;
  if (OutUseUnscaledOp)
    *OutUseUnscaledOp = false;
  if (OutUnscaledOp)
    *OutUnscaledOp = 0;
 
  // Exit early for structured vector spills/fills as they can't take an
  // immediate offset.
  switch (MI.getOpcode()) {
  default:
    break;
  case AArch64::LD1Rv1d:
  case AArch64::LD1Rv2s:
  case AArch64::LD1Rv2d:
  case AArch64::LD1Rv4h:
  case AArch64::LD1Rv4s:
  case AArch64::LD1Rv8b:
  case AArch64::LD1Rv8h:
  case AArch64::LD1Rv16b:
  case AArch64::LD1Twov2d:
  case AArch64::LD1Threev2d:
  case AArch64::LD1Fourv2d:
  case AArch64::LD1Twov1d:
  case AArch64::LD1Threev1d:
  case AArch64::LD1Fourv1d:
  case AArch64::ST1Twov2d:
  case AArch64::ST1Threev2d:
  case AArch64::ST1Fourv2d:
  case AArch64::ST1Twov1d:
  case AArch64::ST1Threev1d:
  case AArch64::ST1Fourv1d:
  case AArch64::ST1i8:
  case AArch64::ST1i16:
  case AArch64::ST1i32:
  case AArch64::ST1i64:
  case AArch64::IRG:
  case AArch64::IRGstack:
  case AArch64::STGloop:
  case AArch64::STZGloop:
    return AArch64FrameOffsetCannotUpdate;
  }
 
  // Get the min/max offset and the scale.
  TypeSize ScaleValue(0U, false), Width(0U, false);
  int64_t MinOff, MaxOff;
  if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), ScaleValue, Width, MinOff,
                                      MaxOff))
    llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
 
  // Construct the complete offset.
  bool IsMulVL = ScaleValue.isScalable();
  unsigned Scale = ScaleValue.getKnownMinValue();
  int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
 
  const MachineOperand &ImmOpnd =
      MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode()));
  Offset += ImmOpnd.getImm() * Scale;
 
  // If the offset doesn't match the scale, we rewrite the instruction to
  // use the unscaled instruction instead. Likewise, if we have a negative
  // offset and there is an unscaled op to use.
  std::optional<unsigned> UnscaledOp =
      AArch64InstrInfo::getUnscaledLdSt(MI.getOpcode());
  bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
  if (useUnscaledOp &&
      !AArch64InstrInfo::getMemOpInfo(*UnscaledOp, ScaleValue, Width, MinOff,
                                      MaxOff))
    llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
 
  Scale = ScaleValue.getKnownMinValue();
  assert(IsMulVL == ScaleValue.isScalable() &&
         "Unscaled opcode has different value for scalable");
 
  int64_t Remainder = Offset % Scale;
  assert(!(Remainder && useUnscaledOp) &&
         "Cannot have remainder when using unscaled op");
 
  assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
  int64_t NewOffset = Offset / Scale;
  if (MinOff <= NewOffset && NewOffset <= MaxOff)
    Offset = Remainder;
  else {
    // Try to minimise the number of instructions required to materialise the
    // offset calculation. Specifically, for fixed offsets, if masking out the
    // low 12 bits leaves a legal add immediate, we can realise the offset
    // calculation with a single add instruction. Whenever this is possible,
    // prefer this split.
    int64_t HighPart = Offset & ~0xFFF;
    int64_t LowPart = Offset & 0xFFF;
    int64_t LowScaled = LowPart / Scale;
    if (!IsMulVL && NewOffset >= 0 && LowPart % Scale == 0 &&
        MinOff <= LowScaled && LowScaled <= MaxOff &&
        AArch64_AM::isLegalArithImmed(HighPart)) {
      NewOffset = LowScaled;
      Offset = HighPart;
    } else {
      // Default to a greedy split: take the memop immediate to be maximum /
      // minimum expressible offset and materialise the remainder.
      NewOffset = NewOffset < 0 ? MinOff : MaxOff;
      Offset = Offset - (NewOffset * Scale);
    }
  }
 
  if (EmittableOffset)
    *EmittableOffset = NewOffset;
  if (OutUseUnscaledOp)
    *OutUseUnscaledOp = useUnscaledOp;
  if (OutUnscaledOp && UnscaledOp)
    *OutUnscaledOp = *UnscaledOp;
 
  if (IsMulVL)
    SOffset = StackOffset::get(SOffset.getFixed(), Offset);
  else
    SOffset = StackOffset::get(Offset, SOffset.getScalable());
  return AArch64FrameOffsetCanUpdate |
         (SOffset ? 0 : AArch64FrameOffsetIsLegal);
}
 
bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
                                    unsigned FrameReg, StackOffset &Offset,
                                    const AArch64InstrInfo *TII) {
  unsigned Opcode = MI.getOpcode();
  unsigned ImmIdx = FrameRegIdx + 1;
 
  if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
    Offset += StackOffset::getFixed(MI.getOperand(ImmIdx).getImm());
    emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
                    MI.getOperand(0).getReg(), FrameReg, Offset, TII,
                    MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
    MI.eraseFromParent();
    Offset = StackOffset();
    return true;
  }
 
  int64_t NewOffset;
  unsigned UnscaledOp;
  bool UseUnscaledOp;
  int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
                                         &UnscaledOp, &NewOffset);
  if (Status & AArch64FrameOffsetCanUpdate) {
    if (Status & AArch64FrameOffsetIsLegal)
      // Replace the FrameIndex with FrameReg.
      MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
    if (UseUnscaledOp)
      MI.setDesc(TII->get(UnscaledOp));
 
    MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
    return !Offset;
  }
 
  return false;
}
 
void AArch64InstrInfo::insertNoop(MachineBasicBlock &MBB,
                                  MachineBasicBlock::iterator MI) const {
  DebugLoc DL;
  BuildMI(MBB, MI, DL, get(AArch64::NOP));
}
 
MCInst AArch64InstrInfo::getNop() const { return MCInstBuilder(AArch64::NOP); }
 
// AArch64 supports MachineCombiner.
bool AArch64InstrInfo::useMachineCombiner() const { return true; }
 
// True when Opc sets flag
static bool isCombineInstrSettingFlag(unsigned Opc) {
  switch (Opc) {
  case AArch64::ADDSWrr:
  case AArch64::ADDSWri:
  case AArch64::ADDSXrr:
  case AArch64::ADDSXri:
  case AArch64::SUBSWrr:
  case AArch64::SUBSXrr:
  // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
  case AArch64::SUBSWri:
  case AArch64::SUBSXri:
    return true;
  default:
    break;
  }
  return false;
}
 
// 32b Opcodes that can be combined with a MUL
static bool isCombineInstrCandidate32(unsigned Opc) {
  switch (Opc) {
  case AArch64::ADDWrr:
  case AArch64::ADDWri:
  case AArch64::SUBWrr:
  case AArch64::ADDSWrr:
  case AArch64::ADDSWri:
  case AArch64::SUBSWrr:
  // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
  case AArch64::SUBWri:
  case AArch64::SUBSWri:
    return true;
  default:
    break;
  }
  return false;
}
 
// 64b Opcodes that can be combined with a MUL
static bool isCombineInstrCandidate64(unsigned Opc) {
  switch (Opc) {
  case AArch64::ADDXrr:
  case AArch64::ADDXri:
  case AArch64::SUBXrr:
  case AArch64::ADDSXrr:
  case AArch64::ADDSXri:
  case AArch64::SUBSXrr:
  // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
  case AArch64::SUBXri:
  case AArch64::SUBSXri:
  case AArch64::ADDv8i8:
  case AArch64::ADDv16i8:
  case AArch64::ADDv4i16:
  case AArch64::ADDv8i16:
  case AArch64::ADDv2i32:
  case AArch64::ADDv4i32:
  case AArch64::SUBv8i8:
  case AArch64::SUBv16i8:
  case AArch64::SUBv4i16:
  case AArch64::SUBv8i16:
  case AArch64::SUBv2i32:
  case AArch64::SUBv4i32:
    return true;
  default:
    break;
  }
  return false;
}
 
// FP Opcodes that can be combined with a FMUL.
static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
  switch (Inst.getOpcode()) {
  default:
    break;
  case AArch64::FADDHrr:
  case AArch64::FADDSrr:
  case AArch64::FADDDrr:
  case AArch64::FADDv4f16:
  case AArch64::FADDv8f16:
  case AArch64::FADDv2f32:
  case AArch64::FADDv2f64:
  case AArch64::FADDv4f32:
  case AArch64::FSUBHrr:
  case AArch64::FSUBSrr:
  case AArch64::FSUBDrr:
  case AArch64::FSUBv4f16:
  case AArch64::FSUBv8f16:
  case AArch64::FSUBv2f32:
  case AArch64::FSUBv2f64:
  case AArch64::FSUBv4f32:
    TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
    // We can fuse FADD/FSUB with FMUL, if fusion is either allowed globally by
    // the target options or if FADD/FSUB has the contract fast-math flag.
    return Options.AllowFPOpFusion == FPOpFusion::Fast ||
           Inst.getFlag(MachineInstr::FmContract);
  }
  return false;
}
 
// Opcodes that can be combined with a MUL
static bool isCombineInstrCandidate(unsigned Opc) {
  return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
}
 
//
// Utility routine that checks if \param MO is defined by an
// \param CombineOpc instruction in the basic block \param MBB
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
                       unsigned CombineOpc, unsigned ZeroReg = 0,
                       bool CheckZeroReg = false) {
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
  MachineInstr *MI = nullptr;
 
  if (MO.isReg() && MO.getReg().isVirtual())
    MI = MRI.getUniqueVRegDef(MO.getReg());
  // And it needs to be in the trace (otherwise, it won't have a depth).
  if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
    return false;
  // Must only used by the user we combine with.
  if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
    return false;
 
  if (CheckZeroReg) {
    assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
           MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
           MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
    // The third input reg must be zero.
    if (MI->getOperand(3).getReg() != ZeroReg)
      return false;
  }
 
  if (isCombineInstrSettingFlag(CombineOpc) &&
      MI->findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, true) == -1)
    return false;
 
  return true;
}
 
//
// Is \param MO defined by an integer multiply and can be combined?
static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
                              unsigned MulOpc, unsigned ZeroReg) {
  return canCombine(MBB, MO, MulOpc, ZeroReg, true);
}
 
//
// Is \param MO defined by a floating-point multiply and can be combined?
static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
                               unsigned MulOpc) {
  return canCombine(MBB, MO, MulOpc);
}
 
// TODO: There are many more machine instruction opcodes to match:
//       1. Other data types (integer, vectors)
//       2. Other math / logic operations (xor, or)
//       3. Other forms of the same operation (intrinsics and other variants)
bool AArch64InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
                                                   bool Invert) const {
  if (Invert)
    return false;
  switch (Inst.getOpcode()) {
  // == Floating-point types ==
  // -- Floating-point instructions --
  case AArch64::FADDHrr:
  case AArch64::FADDSrr:
  case AArch64::FADDDrr:
  case AArch64::FMULHrr:
  case AArch64::FMULSrr:
  case AArch64::FMULDrr:
  case AArch64::FMULX16:
  case AArch64::FMULX32:
  case AArch64::FMULX64:
  // -- Advanced SIMD instructions --
  case AArch64::FADDv4f16:
  case AArch64::FADDv8f16:
  case AArch64::FADDv2f32:
  case AArch64::FADDv4f32:
  case AArch64::FADDv2f64:
  case AArch64::FMULv4f16:
  case AArch64::FMULv8f16:
  case AArch64::FMULv2f32:
  case AArch64::FMULv4f32:
  case AArch64::FMULv2f64:
  case AArch64::FMULXv4f16:
  case AArch64::FMULXv8f16:
  case AArch64::FMULXv2f32:
  case AArch64::FMULXv4f32:
  case AArch64::FMULXv2f64:
  // -- SVE instructions --
  // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
  // in the SVE instruction set (though there are predicated ones).
  case AArch64::FADD_ZZZ_H:
  case AArch64::FADD_ZZZ_S:
  case AArch64::FADD_ZZZ_D:
  case AArch64::FMUL_ZZZ_H:
  case AArch64::FMUL_ZZZ_S:
  case AArch64::FMUL_ZZZ_D:
    return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
           Inst.getFlag(MachineInstr::MIFlag::FmNsz);
 
  // == Integer types ==
  // -- Base instructions --
  // Opcodes MULWrr and MULXrr don't exist because
  // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
  // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
  // The machine-combiner does not support three-source-operands machine
  // instruction. So we cannot reassociate MULs.
  case AArch64::ADDWrr:
  case AArch64::ADDXrr:
  case AArch64::ANDWrr:
  case AArch64::ANDXrr:
  case AArch64::ORRWrr:
  case AArch64::ORRXrr:
  case AArch64::EORWrr:
  case AArch64::EORXrr:
  case AArch64::EONWrr:
  case AArch64::EONXrr:
  // -- Advanced SIMD instructions --
  // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
  // in the Advanced SIMD instruction set.
  case AArch64::ADDv8i8:
  case AArch64::ADDv16i8:
  case AArch64::ADDv4i16:
  case AArch64::ADDv8i16:
  case AArch64::ADDv2i32:
  case AArch64::ADDv4i32:
  case AArch64::ADDv1i64:
  case AArch64::ADDv2i64:
  case AArch64::MULv8i8:
  case AArch64::MULv16i8:
  case AArch64::MULv4i16:
  case AArch64::MULv8i16:
  case AArch64::MULv2i32:
  case AArch64::MULv4i32:
  case AArch64::ANDv8i8:
  case AArch64::ANDv16i8:
  case AArch64::ORRv8i8:
  case AArch64::ORRv16i8:
  case AArch64::EORv8i8:
  case AArch64::EORv16i8:
  // -- SVE instructions --
  case AArch64::ADD_ZZZ_B:
  case AArch64::ADD_ZZZ_H:
  case AArch64::ADD_ZZZ_S:
  case AArch64::ADD_ZZZ_D:
  case AArch64::MUL_ZZZ_B:
  case AArch64::MUL_ZZZ_H:
  case AArch64::MUL_ZZZ_S:
  case AArch64::MUL_ZZZ_D:
  case AArch64::AND_ZZZ:
  case AArch64::ORR_ZZZ:
  case AArch64::EOR_ZZZ:
    return true;
 
  default:
    return false;
  }
}
 
/// Find instructions that can be turned into madd.
static bool getMaddPatterns(MachineInstr &Root,
                            SmallVectorImpl<unsigned> &Patterns) {
  unsigned Opc = Root.getOpcode();
  MachineBasicBlock &MBB = *Root.getParent();
  bool Found = false;
 
  if (!isCombineInstrCandidate(Opc))
    return false;
  if (isCombineInstrSettingFlag(Opc)) {
    int Cmp_NZCV =
        Root.findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, true);
    // When NZCV is live bail out.
    if (Cmp_NZCV == -1)
      return false;
    unsigned NewOpc = convertToNonFlagSettingOpc(Root);
    // When opcode can't change bail out.
    // CHECKME: do we miss any cases for opcode conversion?
    if (NewOpc == Opc)
      return false;
    Opc = NewOpc;
  }
 
  auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
                      unsigned Pattern) {
    if (canCombineWithMUL(MBB, Root.getOperand(Operand), Opcode, ZeroReg)) {
      Patterns.push_back(Pattern);
      Found = true;
    }
  };
 
  auto setVFound = [&](int Opcode, int Operand, unsigned Pattern) {
    if (canCombine(MBB, Root.getOperand(Operand), Opcode)) {
      Patterns.push_back(Pattern);
      Found = true;
    }
  };
 
  typedef AArch64MachineCombinerPattern MCP;
 
  switch (Opc) {
  default:
    break;
  case AArch64::ADDWrr:
    assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
           "ADDWrr does not have register operands");
    setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
    setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
    break;
  case AArch64::ADDXrr:
    setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
    setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
    break;
  case AArch64::SUBWrr:
    setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
    setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
    break;
  case AArch64::SUBXrr:
    setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
    setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
    break;
  case AArch64::ADDWri:
    setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
    break;
  case AArch64::ADDXri:
    setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
    break;
  case AArch64::SUBWri:
    setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
    break;
  case AArch64::SUBXri:
    setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
    break;
  case AArch64::ADDv8i8:
    setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
    setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
    break;
  case AArch64::ADDv16i8:
    setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
    setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
    break;
  case AArch64::ADDv4i16:
    setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
    setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
    setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
    setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
    break;
  case AArch64::ADDv8i16:
    setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
    setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
    setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
    setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
    break;
  case AArch64::ADDv2i32:
    setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
    setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
    setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
    setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
    break;
  case AArch64::ADDv4i32:
    setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
    setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
    setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
    setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
    break;
  case AArch64::SUBv8i8:
    setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
    setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
    break;
  case AArch64::SUBv16i8:
    setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
    setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
    break;
  case AArch64::SUBv4i16:
    setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
    setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
    setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
    setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
    break;
  case AArch64::SUBv8i16:
    setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
    setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
    setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
    setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
    break;
  case AArch64::SUBv2i32:
    setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
    setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
    setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
    setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
    break;
  case AArch64::SUBv4i32:
    setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
    setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
    setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
    setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
    break;
  }
  return Found;
}
 
bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
  switch (Opcode) {
  default:
    break;
  case AArch64::UABALB_ZZZ_D:
  case AArch64::UABALB_ZZZ_H:
  case AArch64::UABALB_ZZZ_S:
  case AArch64::UABALT_ZZZ_D:
  case AArch64::UABALT_ZZZ_H:
  case AArch64::UABALT_ZZZ_S:
  case AArch64::SABALB_ZZZ_D:
  case AArch64::SABALB_ZZZ_S:
  case AArch64::SABALB_ZZZ_H:
  case AArch64::SABALT_ZZZ_D:
  case AArch64::SABALT_ZZZ_S:
  case AArch64::SABALT_ZZZ_H:
  case AArch64::UABALv16i8_v8i16:
  case AArch64::UABALv2i32_v2i64:
  case AArch64::UABALv4i16_v4i32:
  case AArch64::UABALv4i32_v2i64:
  case AArch64::UABALv8i16_v4i32:
  case AArch64::UABALv8i8_v8i16:
  case AArch64::UABAv16i8:
  case AArch64::UABAv2i32:
  case AArch64::UABAv4i16:
  case AArch64::UABAv4i32:
  case AArch64::UABAv8i16:
  case AArch64::UABAv8i8:
  case AArch64::SABALv16i8_v8i16:
  case AArch64::SABALv2i32_v2i64:
  case AArch64::SABALv4i16_v4i32:
  case AArch64::SABALv4i32_v2i64:
  case AArch64::SABALv8i16_v4i32:
  case AArch64::SABALv8i8_v8i16:
  case AArch64::SABAv16i8:
  case AArch64::SABAv2i32:
  case AArch64::SABAv4i16:
  case AArch64::SABAv4i32:
  case AArch64::SABAv8i16:
  case AArch64::SABAv8i8:
    return true;
  }
 
  return false;
}
 
unsigned AArch64InstrInfo::getAccumulationStartOpcode(
    unsigned AccumulationOpcode) const {
  switch (AccumulationOpcode) {
  default:
    llvm_unreachable("Unsupported accumulation Opcode!");
  case AArch64::UABALB_ZZZ_D:
    return AArch64::UABDLB_ZZZ_D;
  case AArch64::UABALB_ZZZ_H:
    return AArch64::UABDLB_ZZZ_H;
  case AArch64::UABALB_ZZZ_S:
    return AArch64::UABDLB_ZZZ_S;
  case AArch64::UABALT_ZZZ_D:
    return AArch64::UABDLT_ZZZ_D;
  case AArch64::UABALT_ZZZ_H:
    return AArch64::UABDLT_ZZZ_H;
  case AArch64::UABALT_ZZZ_S:
    return AArch64::UABDLT_ZZZ_S;
  case AArch64::UABALv16i8_v8i16:
    return AArch64::UABDLv16i8_v8i16;
  case AArch64::UABALv2i32_v2i64:
    return AArch64::UABDLv2i32_v2i64;
  case AArch64::UABALv4i16_v4i32:
    return AArch64::UABDLv4i16_v4i32;
  case AArch64::UABALv4i32_v2i64:
    return AArch64::UABDLv4i32_v2i64;
  case AArch64::UABALv8i16_v4i32:
    return AArch64::UABDLv8i16_v4i32;
  case AArch64::UABALv8i8_v8i16:
    return AArch64::UABDLv8i8_v8i16;
  case AArch64::UABAv16i8:
    return AArch64::UABDv16i8;
  case AArch64::UABAv2i32:
    return AArch64::UABDv2i32;
  case AArch64::UABAv4i16:
    return AArch64::UABDv4i16;
  case AArch64::UABAv4i32:
    return AArch64::UABDv4i32;
  case AArch64::UABAv8i16:
    return AArch64::UABDv8i16;
  case AArch64::UABAv8i8:
    return AArch64::UABDv8i8;
  case AArch64::SABALB_ZZZ_D:
    return AArch64::SABDLB_ZZZ_D;
  case AArch64::SABALB_ZZZ_S:
    return AArch64::SABDLB_ZZZ_S;
  case AArch64::SABALB_ZZZ_H:
    return AArch64::SABDLB_ZZZ_H;
  case AArch64::SABALT_ZZZ_D:
    return AArch64::SABDLT_ZZZ_D;
  case AArch64::SABALT_ZZZ_S:
    return AArch64::SABDLT_ZZZ_S;
  case AArch64::SABALT_ZZZ_H:
    return AArch64::SABDLT_ZZZ_H;
  case AArch64::SABALv16i8_v8i16:
    return AArch64::SABDLv16i8_v8i16;
  case AArch64::SABALv2i32_v2i64:
    return AArch64::SABDLv2i32_v2i64;
  case AArch64::SABALv4i16_v4i32:
    return AArch64::SABDLv4i16_v4i32;
  case AArch64::SABALv4i32_v2i64:
    return AArch64::SABDLv4i32_v2i64;
  case AArch64::SABALv8i16_v4i32:
    return AArch64::SABDLv8i16_v4i32;
  case AArch64::SABALv8i8_v8i16:
    return AArch64::SABDLv8i8_v8i16;
  case AArch64::SABAv16i8:
    return AArch64::SABDv16i8;
  case AArch64::SABAv2i32:
    return AArch64::SABAv2i32;
  case AArch64::SABAv4i16:
    return AArch64::SABDv4i16;
  case AArch64::SABAv4i32:
    return AArch64::SABDv4i32;
  case AArch64::SABAv8i16:
    return AArch64::SABDv8i16;
  case AArch64::SABAv8i8:
    return AArch64::SABDv8i8;
  }
}
 
/// Floating-Point Support
 
/// Find instructions that can be turned into madd.
static bool getFMAPatterns(MachineInstr &Root,
                           SmallVectorImpl<unsigned> &Patterns) {
 
  if (!isCombineInstrCandidateFP(Root))
    return false;
 
  MachineBasicBlock &MBB = *Root.getParent();
  bool Found = false;
 
  auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool {
    if (canCombineWithFMUL(MBB, Root.getOperand(Operand), Opcode)) {
      Patterns.push_back(Pattern);
      return true;
    }
    return false;
  };
 
  typedef AArch64MachineCombinerPattern MCP;
 
  switch (Root.getOpcode()) {
  default:
    assert(false && "Unsupported FP instruction in combiner\n");
    break;
  case AArch64::FADDHrr:
    assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
           "FADDHrr does not have register operands");
 
    Found  = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
    Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
    break;
  case AArch64::FADDSrr:
    assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
           "FADDSrr does not have register operands");
 
    Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
             Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
 
    Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
             Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
    break;
  case AArch64::FADDDrr:
    Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
             Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
 
    Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
             Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
    break;
  case AArch64::FADDv4f16:
    Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
             Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
 
    Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
             Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
    break;
  case AArch64::FADDv8f16:
    Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
             Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
 
    Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
             Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
    break;
  case AArch64::FADDv2f32:
    Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
             Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
 
    Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
             Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
    break;
  case AArch64::FADDv2f64:
    Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
             Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
 
    Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
             Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
    break;
  case AArch64::FADDv4f32:
    Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
             Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
 
    Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
             Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
    break;
  case AArch64::FSUBHrr:
    Found  = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
    Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
    Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
    break;
  case AArch64::FSUBSrr:
    Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
 
    Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
             Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
 
    Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
    break;
  case AArch64::FSUBDrr:
    Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
 
    Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
             Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
 
    Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
    break;
  case AArch64::FSUBv4f16:
    Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
             Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
 
    Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
             Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
    break;
  case AArch64::FSUBv8f16:
    Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
             Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
 
    Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
             Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
    break;
  case AArch64::FSUBv2f32:
    Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
             Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
 
    Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
             Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
    break;
  case AArch64::FSUBv2f64:
    Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
             Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
 
    Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
             Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
    break;
  case AArch64::FSUBv4f32:
    Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
             Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
 
    Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
             Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
    break;
  }
  return Found;
}
 
static bool getFMULPatterns(MachineInstr &Root,
                            SmallVectorImpl<unsigned> &Patterns) {
  MachineBasicBlock &MBB = *Root.getParent();
  bool Found = false;
 
  auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool {
    MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
    MachineOperand &MO = Root.getOperand(Operand);
    MachineInstr *MI = nullptr;
    if (MO.isReg() && MO.getReg().isVirtual())
      MI = MRI.getUniqueVRegDef(MO.getReg());
    // Ignore No-op COPYs in FMUL(COPY(DUP(..)))
    if (MI && MI->getOpcode() == TargetOpcode::COPY &&
        MI->getOperand(1).getReg().isVirtual())
      MI = MRI.getUniqueVRegDef(MI->getOperand(1).getReg());
    if (MI && MI->getOpcode() == Opcode) {
      Patterns.push_back(Pattern);
      return true;
    }
    return false;
  };
 
  typedef AArch64MachineCombinerPattern MCP;
 
  switch (Root.getOpcode()) {
  default:
    return false;
  case AArch64::FMULv2f32:
    Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
    Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
    break;
  case AArch64::FMULv2f64:
    Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
    Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
    break;
  case AArch64::FMULv4f16:
    Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
    Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
    break;
  case AArch64::FMULv4f32:
    Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
    Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
    break;
  case AArch64::FMULv8f16:
    Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
    Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
    break;
  }
 
  return Found;
}
 
static bool getFNEGPatterns(MachineInstr &Root,
                            SmallVectorImpl<unsigned> &Patterns) {
  unsigned Opc = Root.getOpcode();
  MachineBasicBlock &MBB = *Root.getParent();
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
 
  auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool {
    MachineOperand &MO = Root.getOperand(1);
    MachineInstr *MI = MRI.getUniqueVRegDef(MO.getReg());
    if (MI != nullptr && (MI->getOpcode() == Opcode) &&
        MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()) &&
        Root.getFlag(MachineInstr::MIFlag::FmContract) &&
        Root.getFlag(MachineInstr::MIFlag::FmNsz) &&
        MI->getFlag(MachineInstr::MIFlag::FmContract) &&
        MI->getFlag(MachineInstr::MIFlag::FmNsz)) {
      Patterns.push_back(Pattern);
      return true;
    }
    return false;
  };
 
  switch (Opc) {
  default:
    break;
  case AArch64::FNEGDr:
    return Match(AArch64::FMADDDrrr, AArch64MachineCombinerPattern::FNMADD);
  case AArch64::FNEGSr:
    return Match(AArch64::FMADDSrrr, AArch64MachineCombinerPattern::FNMADD);
  }
 
  return false;
}
 
/// Return true when a code sequence can improve throughput. It
/// should be called only for instructions in loops.
/// \param Pattern - combiner pattern
bool AArch64InstrInfo::isThroughputPattern(unsigned Pattern) const {
  switch (Pattern) {
  default:
    break;
  case AArch64MachineCombinerPattern::FMULADDH_OP1:
  case AArch64MachineCombinerPattern::FMULADDH_OP2:
  case AArch64MachineCombinerPattern::FMULSUBH_OP1:
  case AArch64MachineCombinerPattern::FMULSUBH_OP2:
  case AArch64MachineCombinerPattern::FMULADDS_OP1:
  case AArch64MachineCombinerPattern::FMULADDS_OP2:
  case AArch64MachineCombinerPattern::FMULSUBS_OP1:
  case AArch64MachineCombinerPattern::FMULSUBS_OP2:
  case AArch64MachineCombinerPattern::FMULADDD_OP1:
  case AArch64MachineCombinerPattern::FMULADDD_OP2:
  case AArch64MachineCombinerPattern::FMULSUBD_OP1:
  case AArch64MachineCombinerPattern::FMULSUBD_OP2:
  case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
  case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
  case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
  case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
  case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
  case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
  case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
  case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
  case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
  case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
  case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
  case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
  case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
  case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
  case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
  case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
  case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
  case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
  case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
  case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
  case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2:
  case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2:
  case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2:
  case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
  case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
  case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
  case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
  case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
  case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
  case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
  case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
  case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
  case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
  case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
  case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
  case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
  case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
  case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
  case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
  case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
  case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
  case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
  case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
  case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
  case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
  case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
  case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
  case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
  case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
  case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
  case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
  case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
  case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
  case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
  case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
  case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
  case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
  case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
  case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
  case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
  case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
  case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
  case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
    return true;
  } // end switch (Pattern)
  return false;
}
 
/// Find other MI combine patterns.
static bool getMiscPatterns(MachineInstr &Root,
                            SmallVectorImpl<unsigned> &Patterns) {
  // A - (B + C)  ==>   (A - B) - C  or  (A - C) - B
  unsigned Opc = Root.getOpcode();
  MachineBasicBlock &MBB = *Root.getParent();
 
  switch (Opc) {
  case AArch64::SUBWrr:
  case AArch64::SUBSWrr:
  case AArch64::SUBXrr:
  case AArch64::SUBSXrr:
    // Found candidate root.
    break;
  default:
    return false;
  }
 
  if (isCombineInstrSettingFlag(Opc) &&
      Root.findRegisterDefOperandIdx(AArch64::NZCV, /*TRI=*/nullptr, true) ==
          -1)
    return false;
 
  if (canCombine(MBB, Root.getOperand(2), AArch64::ADDWrr) ||
      canCombine(MBB, Root.getOperand(2), AArch64::ADDSWrr) ||
      canCombine(MBB, Root.getOperand(2), AArch64::ADDXrr) ||
      canCombine(MBB, Root.getOperand(2), AArch64::ADDSXrr)) {
    Patterns.push_back(AArch64MachineCombinerPattern::SUBADD_OP1);
    Patterns.push_back(AArch64MachineCombinerPattern::SUBADD_OP2);
    return true;
  }
 
  return false;
}
 
/// Check if the given instruction forms a gather load pattern that can be
/// optimized for better Memory-Level Parallelism (MLP). This function
/// identifies chains of NEON lane load instructions that load data from
/// different memory addresses into individual lanes of a 128-bit vector
/// register, then attempts to split the pattern into parallel loads to break
/// the serial dependency between instructions.
///
/// Pattern Matched:
///   Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
///   LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
///
/// Transformed Into:
///   Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
///   to combine the results, enabling better memory-level parallelism.
///
/// Supported Element Types:
///   - 32-bit elements (LD1i32, 4 lanes total)
///   - 16-bit elements (LD1i16, 8 lanes total)
///   - 8-bit elements (LD1i8, 16 lanes total)
static bool getGatherLanePattern(MachineInstr &Root,
                                 SmallVectorImpl<unsigned> &Patterns,
                                 unsigned LoadLaneOpCode, unsigned NumLanes) {
  const MachineFunction *MF = Root.getMF();
 
  // Early exit if optimizing for size.
  if (MF->getFunction().hasMinSize())
    return false;
 
  const MachineRegisterInfo &MRI = MF->getRegInfo();
  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
 
  // The root of the pattern must load into the last lane of the vector.
  if (Root.getOperand(2).getImm() != NumLanes - 1)
    return false;
 
  // Check that we have load into all lanes except lane 0.
  // For each load we also want to check that:
  // 1. It has a single non-debug use (since we will be replacing the virtual
  // register)
  // 2. That the addressing mode only uses a single pointer operand
  auto *CurrInstr = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
  auto Range = llvm::seq<unsigned>(1, NumLanes - 1);
  SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
  SmallVector<const MachineInstr *, 16> LoadInstrs;
  while (!RemainingLanes.empty() && CurrInstr &&
         CurrInstr->getOpcode() == LoadLaneOpCode &&
         MRI.hasOneNonDBGUse(CurrInstr->getOperand(0).getReg()) &&
         CurrInstr->getNumOperands() == 4) {
    RemainingLanes.erase(CurrInstr->getOperand(2).getImm());
    LoadInstrs.push_back(CurrInstr);
    CurrInstr = MRI.getUniqueVRegDef(CurrInstr->getOperand(1).getReg());
  }
 
  // Check that we have found a match for lanes N-1.. 1.
  if (!RemainingLanes.empty())
    return false;
 
  // Match the SUBREG_TO_REG sequence.
  if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
    return false;
 
  // Verify that the subreg to reg loads an integer into the first lane.
  auto Lane0LoadReg = CurrInstr->getOperand(1).getReg();
  unsigned SingleLaneSizeInBits = 128 / NumLanes;
  if (TRI->getRegSizeInBits(Lane0LoadReg, MRI) != SingleLaneSizeInBits)
    return false;
 
  // Verify that it also has a single non debug use.
  if (!MRI.hasOneNonDBGUse(Lane0LoadReg))
    return false;
 
  LoadInstrs.push_back(MRI.getUniqueVRegDef(Lane0LoadReg));
 
  // If there is any chance of aliasing, do not apply the pattern.
  // Walk backward through the MBB starting from Root.
  // Exit early if we've encountered all load instructions or hit the search
  // limit.
  auto MBBItr = Root.getIterator();
  unsigned RemainingSteps = GatherOptSearchLimit;
  SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
  RemainingLoadInstrs.insert(LoadInstrs.begin(), LoadInstrs.end());
  const MachineBasicBlock *MBB = Root.getParent();
 
  for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
         !RemainingLoadInstrs.empty();
       --MBBItr, --RemainingSteps) {
    const MachineInstr &CurrInstr = *MBBItr;
 
    // Remove this instruction from remaining loads if it's one we're tracking.
    RemainingLoadInstrs.erase(&CurrInstr);
 
    // Check for potential aliasing with any of the load instructions to
    // optimize.
    if (CurrInstr.isLoadFoldBarrier())
      return false;
  }
 
  // If we hit the search limit without finding all load instructions,
  // don't match the pattern.
  if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
    return false;
 
  switch (NumLanes) {
  case 4:
    Patterns.push_back(AArch64MachineCombinerPattern::GATHER_LANE_i32);
    break;
  case 8:
    Patterns.push_back(AArch64MachineCombinerPattern::GATHER_LANE_i16);
    break;
  case 16:
    Patterns.push_back(AArch64MachineCombinerPattern::GATHER_LANE_i8);
    break;
  default:
    llvm_unreachable("Got bad number of lanes for gather pattern.");
  }
 
  return true;
}
 
/// Search for patterns of LD instructions we can optimize.
static bool getLoadPatterns(MachineInstr &Root,
                            SmallVectorImpl<unsigned> &Patterns) {
 
  // The pattern searches for loads into single lanes.
  switch (Root.getOpcode()) {
  case AArch64::LD1i32:
    return getGatherLanePattern(Root, Patterns, Root.getOpcode(), 4);
  case AArch64::LD1i16:
    return getGatherLanePattern(Root, Patterns, Root.getOpcode(), 8);
  case AArch64::LD1i8:
    return getGatherLanePattern(Root, Patterns, Root.getOpcode(), 16);
  default:
    return false;
  }
}
 
/// Generate optimized instruction sequence for gather load patterns to improve
/// Memory-Level Parallelism (MLP). This function transforms a chain of
/// sequential NEON lane loads into parallel vector loads that can execute
/// concurrently.
static void
generateGatherLanePattern(MachineInstr &Root,
                          SmallVectorImpl<MachineInstr *> &InsInstrs,
                          SmallVectorImpl<MachineInstr *> &DelInstrs,
                          DenseMap<Register, unsigned> &InstrIdxForVirtReg,
                          unsigned Pattern, unsigned NumLanes) {
  MachineFunction &MF = *Root.getParent()->getParent();
  MachineRegisterInfo &MRI = MF.getRegInfo();
  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
 
  // Gather the initial load instructions to build the pattern.
  SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
  MachineInstr *CurrInstr = &Root;
  for (unsigned i = 0; i < NumLanes - 1; ++i) {
    LoadToLaneInstrs.push_back(CurrInstr);
    CurrInstr = MRI.getUniqueVRegDef(CurrInstr->getOperand(1).getReg());
  }
 
  // Sort the load instructions according to the lane.
  llvm::sort(LoadToLaneInstrs,
             [](const MachineInstr *A, const MachineInstr *B) {
               return A->getOperand(2).getImm() > B->getOperand(2).getImm();
             });
 
  MachineInstr *SubregToReg = CurrInstr;
  LoadToLaneInstrs.push_back(
      MRI.getUniqueVRegDef(SubregToReg->getOperand(1).getReg()));
  auto LoadToLaneInstrsAscending = llvm::reverse(LoadToLaneInstrs);
 
  const TargetRegisterClass *FPR128RegClass =
      MRI.getRegClass(Root.getOperand(0).getReg());
 
  // Helper lambda to create a LD1 instruction.
  auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
                                  Register SrcRegister, unsigned Lane,
                                  Register OffsetRegister,
                                  bool OffsetRegisterKillState) {
    auto NewRegister = MRI.createVirtualRegister(FPR128RegClass);
    MachineInstrBuilder LoadIndexIntoRegister =
        BuildMI(MF, MIMetadata(*OriginalInstr), TII->get(Root.getOpcode()),
                NewRegister)
            .addReg(SrcRegister)
            .addImm(Lane)
            .addReg(OffsetRegister, getKillRegState(OffsetRegisterKillState))
            .setMemRefs(OriginalInstr->memoperands());
    InstrIdxForVirtReg.insert(std::make_pair(NewRegister, InsInstrs.size()));
    InsInstrs.push_back(LoadIndexIntoRegister);
    return NewRegister;
  };
 
  // Helper to create load instruction based on the NumLanes in the NEON
  // register we are rewriting.
  auto CreateLDRInstruction =
      [&](unsigned NumLanes, Register DestReg, Register OffsetReg,
          ArrayRef<MachineMemOperand *> MMOs) -> MachineInstrBuilder {
    unsigned Opcode;
    switch (NumLanes) {
    case 4:
      Opcode = AArch64::LDRSui;
      break;
    case 8:
      Opcode = AArch64::LDRHui;
      break;
    case 16:
      Opcode = AArch64::LDRBui;
      break;
    default:
      llvm_unreachable(
          "Got unsupported number of lanes in machine-combiner gather pattern");
    }
    // Immediate offset load
    return BuildMI(MF, MIMetadata(Root), TII->get(Opcode), DestReg)
        .addReg(OffsetReg)
        .addImm(0)
        .setMemRefs(MMOs);
  };
 
  // Load the remaining lanes into register 0.
  auto LanesToLoadToReg0 =
      llvm::make_range(LoadToLaneInstrsAscending.begin() + 1,
                       LoadToLaneInstrsAscending.begin() + NumLanes / 2);
  Register PrevReg = SubregToReg->getOperand(0).getReg();
  for (auto [Index, LoadInstr] : llvm::enumerate(LanesToLoadToReg0)) {
    const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
    PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
                                   OffsetRegOperand.getReg(),
                                   OffsetRegOperand.isKill());
    DelInstrs.push_back(LoadInstr);
  }
  Register LastLoadReg0 = PrevReg;
 
  // First load into register 1. Perform an integer load to zero out the upper
  // lanes in a single instruction.
  MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
  MachineInstr *OriginalSplitLoad =
      *std::next(LoadToLaneInstrsAscending.begin(), NumLanes / 2);
  Register DestRegForMiddleIndex = MRI.createVirtualRegister(
      MRI.getRegClass(Lane0Load->getOperand(0).getReg()));
 
  const MachineOperand &OriginalSplitToLoadOffsetOperand =
      OriginalSplitLoad->getOperand(3);
  MachineInstrBuilder MiddleIndexLoadInstr =
      CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
                           OriginalSplitToLoadOffsetOperand.getReg(),
                           OriginalSplitLoad->memoperands());
 
  InstrIdxForVirtReg.insert(
      std::make_pair(DestRegForMiddleIndex, InsInstrs.size()));
  InsInstrs.push_back(MiddleIndexLoadInstr);
  DelInstrs.push_back(OriginalSplitLoad);
 
  // Subreg To Reg instruction for register 1.
  Register DestRegForSubregToReg = MRI.createVirtualRegister(FPR128RegClass);
  unsigned SubregType;
  switch (NumLanes) {
  case 4:
    SubregType = AArch64::ssub;
    break;
  case 8:
    SubregType = AArch64::hsub;
    break;
  case 16:
    SubregType = AArch64::bsub;
    break;
  default:
    llvm_unreachable(
        "Got invalid NumLanes for machine-combiner gather pattern");
  }
 
  auto SubRegToRegInstr =
      BuildMI(MF, MIMetadata(Root), TII->get(SubregToReg->getOpcode()),
              DestRegForSubregToReg)
          .addReg(DestRegForMiddleIndex, getKillRegState(true))
          .addImm(SubregType);
  InstrIdxForVirtReg.insert(
      std::make_pair(DestRegForSubregToReg, InsInstrs.size()));
  InsInstrs.push_back(SubRegToRegInstr);
 
  // Load remaining lanes into register 1.
  auto LanesToLoadToReg1 =
      llvm::make_range(LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
                       LoadToLaneInstrsAscending.end());
  PrevReg = SubRegToRegInstr->getOperand(0).getReg();
  for (auto [Index, LoadInstr] : llvm::enumerate(LanesToLoadToReg1)) {
    const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
    PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
                                   OffsetRegOperand.getReg(),
                                   OffsetRegOperand.isKill());
 
    // Do not add the last reg to DelInstrs - it will be removed later.
    if (Index == NumLanes / 2 - 2) {
      break;
    }
    DelInstrs.push_back(LoadInstr);
  }
  Register LastLoadReg1 = PrevReg;
 
  // Create the final zip instruction to combine the results.
  MachineInstrBuilder ZipInstr =
      BuildMI(MF, MIMetadata(Root), TII->get(AArch64::ZIP1v2i64),
              Root.getOperand(0).getReg())
          .addReg(LastLoadReg0)
          .addReg(LastLoadReg1);
  InsInstrs.push_back(ZipInstr);
}
 
CombinerObjective
AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
  switch (Pattern) {
  case AArch64MachineCombinerPattern::SUBADD_OP1:
  case AArch64MachineCombinerPattern::SUBADD_OP2:
  case AArch64MachineCombinerPattern::GATHER_LANE_i32:
  case AArch64MachineCombinerPattern::GATHER_LANE_i16:
  case AArch64MachineCombinerPattern::GATHER_LANE_i8:
    return CombinerObjective::MustReduceDepth;
  default:
    return TargetInstrInfo::getCombinerObjective(Pattern);
  }
}
 
/// Return true when there is potentially a faster code sequence for an
/// instruction chain ending in \p Root. All potential patterns are listed in
/// the \p Pattern vector. Pattern should be sorted in priority order since the
/// pattern evaluator stops checking as soon as it finds a faster sequence.
 
bool AArch64InstrInfo::getMachineCombinerPatterns(
    MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
    bool DoRegPressureReduce) const {
  // Integer patterns
  if (getMaddPatterns(Root, Patterns))
    return true;
  // Floating point patterns
  if (getFMULPatterns(Root, Patterns))
    return true;
  if (getFMAPatterns(Root, Patterns))
    return true;
  if (getFNEGPatterns(Root, Patterns))
    return true;
 
  // Other patterns
  if (getMiscPatterns(Root, Patterns))
    return true;
 
  // Load patterns
  if (getLoadPatterns(Root, Patterns))
    return true;
 
  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
                                                     DoRegPressureReduce);
}
 
enum class FMAInstKind { Default, Indexed, Accumulator };
/// genFusedMultiply - Generate fused multiply instructions.
/// This function supports both integer and floating point instructions.
/// A typical example:
///  F|MUL I=A,B,0
///  F|ADD R,I,C
///  ==> F|MADD R,A,B,C
/// \param MF Containing MachineFunction
/// \param MRI Register information
/// \param TII Target information
/// \param Root is the F|ADD instruction
/// \param [out] InsInstrs is a vector of machine instructions and will
/// contain the generated madd instruction
/// \param IdxMulOpd is index of operand in Root that is the result of
/// the F|MUL. In the example above IdxMulOpd is 1.
/// \param MaddOpc the opcode fo the f|madd instruction
/// \param RC Register class of operands
/// \param kind of fma instruction (addressing mode) to be generated
/// \param ReplacedAddend is the result register from the instruction
/// replacing the non-combined operand, if any.
static MachineInstr *
genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
                 const TargetInstrInfo *TII, MachineInstr &Root,
                 SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
                 unsigned MaddOpc, const TargetRegisterClass *RC,
                 FMAInstKind kind = FMAInstKind::Default,
                 const Register *ReplacedAddend = nullptr) {
  assert(IdxMulOpd == 1 || IdxMulOpd == 2);
 
  unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
  MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
  Register ResultReg = Root.getOperand(0).getReg();
  Register SrcReg0 = MUL->getOperand(1).getReg();
  bool Src0IsKill = MUL->getOperand(1).isKill();
  Register SrcReg1 = MUL->getOperand(2).getReg();
  bool Src1IsKill = MUL->getOperand(2).isKill();
 
  Register SrcReg2;
  bool Src2IsKill;
  if (ReplacedAddend) {
    // If we just generated a new addend, we must be it's only use.
    SrcReg2 = *ReplacedAddend;
    Src2IsKill = true;
  } else {
    SrcReg2 = Root.getOperand(IdxOtherOpd).getReg();
    Src2IsKill = Root.getOperand(IdxOtherOpd).isKill();
  }
 
  if (ResultReg.isVirtual())
    MRI.constrainRegClass(ResultReg, RC);
  if (SrcReg0.isVirtual())
    MRI.constrainRegClass(SrcReg0, RC);
  if (SrcReg1.isVirtual())
    MRI.constrainRegClass(SrcReg1, RC);
  if (SrcReg2.isVirtual())
    MRI.constrainRegClass(SrcReg2, RC);
 
  MachineInstrBuilder MIB;
  if (kind == FMAInstKind::Default)
    MIB = BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg)
              .addReg(SrcReg0, getKillRegState(Src0IsKill))
              .addReg(SrcReg1, getKillRegState(Src1IsKill))
              .addReg(SrcReg2, getKillRegState(Src2IsKill));
  else if (kind == FMAInstKind::Indexed)
    MIB = BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg)
              .addReg(SrcReg2, getKillRegState(Src2IsKill))
              .addReg(SrcReg0, getKillRegState(Src0IsKill))
              .addReg(SrcReg1, getKillRegState(Src1IsKill))
              .addImm(MUL->getOperand(3).getImm());
  else if (kind == FMAInstKind::Accumulator)
    MIB = BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg)
              .addReg(SrcReg2, getKillRegState(Src2IsKill))
              .addReg(SrcReg0, getKillRegState(Src0IsKill))
              .addReg(SrcReg1, getKillRegState(Src1IsKill));
  else
    assert(false && "Invalid FMA instruction kind \n");
  // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
  InsInstrs.push_back(MIB);
  return MUL;
}
 
static MachineInstr *
genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
               const TargetInstrInfo *TII, MachineInstr &Root,
               SmallVectorImpl<MachineInstr *> &InsInstrs) {
  MachineInstr *MAD = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
 
  unsigned Opc = 0;
  const TargetRegisterClass *RC = MRI.getRegClass(MAD->getOperand(0).getReg());
  if (AArch64::FPR32RegClass.hasSubClassEq(RC))
    Opc = AArch64::FNMADDSrrr;
  else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
    Opc = AArch64::FNMADDDrrr;
  else
    return nullptr;
 
  Register ResultReg = Root.getOperand(0).getReg();
  Register SrcReg0 = MAD->getOperand(1).getReg();
  Register SrcReg1 = MAD->getOperand(2).getReg();
  Register SrcReg2 = MAD->getOperand(3).getReg();
  bool Src0IsKill = MAD->getOperand(1).isKill();
  bool Src1IsKill = MAD->getOperand(2).isKill();
  bool Src2IsKill = MAD->getOperand(3).isKill();
  if (ResultReg.isVirtual())
    MRI.constrainRegClass(ResultReg, RC);
  if (SrcReg0.isVirtual())
    MRI.constrainRegClass(SrcReg0, RC);
  if (SrcReg1.isVirtual())
    MRI.constrainRegClass(SrcReg1, RC);
  if (SrcReg2.isVirtual())
    MRI.constrainRegClass(SrcReg2, RC);
 
  MachineInstrBuilder MIB =
      BuildMI(MF, MIMetadata(Root), TII->get(Opc), ResultReg)
          .addReg(SrcReg0, getKillRegState(Src0IsKill))
          .addReg(SrcReg1, getKillRegState(Src1IsKill))
          .addReg(SrcReg2, getKillRegState(Src2IsKill));
  InsInstrs.push_back(MIB);
 
  return MAD;
}
 
/// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static MachineInstr *
genIndexedMultiply(MachineInstr &Root,
                   SmallVectorImpl<MachineInstr *> &InsInstrs,
                   unsigned IdxDupOp, unsigned MulOpc,
                   const TargetRegisterClass *RC, MachineRegisterInfo &MRI) {
  assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
         "Invalid index of FMUL operand");
 
  MachineFunction &MF = *Root.getMF();
  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
 
  MachineInstr *Dup =
      MF.getRegInfo().getUniqueVRegDef(Root.getOperand(IdxDupOp).getReg());
 
  if (Dup->getOpcode() == TargetOpcode::COPY)
    Dup = MRI.getUniqueVRegDef(Dup->getOperand(1).getReg());
 
  Register DupSrcReg = Dup->getOperand(1).getReg();
  MRI.clearKillFlags(DupSrcReg);
  MRI.constrainRegClass(DupSrcReg, RC);
 
  unsigned DupSrcLane = Dup->getOperand(2).getImm();
 
  unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
  MachineOperand &MulOp = Root.getOperand(IdxMulOp);
 
  Register ResultReg = Root.getOperand(0).getReg();
 
  MachineInstrBuilder MIB;
  MIB = BuildMI(MF, MIMetadata(Root), TII->get(MulOpc), ResultReg)
            .add(MulOp)
            .addReg(DupSrcReg)
            .addImm(DupSrcLane);
 
  InsInstrs.push_back(MIB);
  return &Root;
}
 
/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
/// instructions.
///
/// \see genFusedMultiply
static MachineInstr *genFusedMultiplyAcc(
    MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
    MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
    unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
  return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
                          FMAInstKind::Accumulator);
}
 
/// genNeg - Helper to generate an intermediate negation of the second operand
/// of Root
static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
                       const TargetInstrInfo *TII, MachineInstr &Root,
                       SmallVectorImpl<MachineInstr *> &InsInstrs,
                       DenseMap<Register, unsigned> &InstrIdxForVirtReg,
                       unsigned MnegOpc, const TargetRegisterClass *RC) {
  Register NewVR = MRI.createVirtualRegister(RC);
  MachineInstrBuilder MIB =
      BuildMI(MF, MIMetadata(Root), TII->get(MnegOpc), NewVR)
          .add(Root.getOperand(2));
  InsInstrs.push_back(MIB);
 
  assert(InstrIdxForVirtReg.empty());
  InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
 
  return NewVR;
}
 
/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
/// instructions with an additional negation of the accumulator
static MachineInstr *genFusedMultiplyAccNeg(
    MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
    MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
    DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
    unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
  assert(IdxMulOpd == 1);
 
  Register NewVR =
      genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
  return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
                          FMAInstKind::Accumulator, &NewVR);
}
 
/// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
/// instructions.
///
/// \see genFusedMultiply
static MachineInstr *genFusedMultiplyIdx(
    MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
    MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
    unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
  return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
                          FMAInstKind::Indexed);
}
 
/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
/// instructions with an additional negation of the accumulator
static MachineInstr *genFusedMultiplyIdxNeg(
    MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
    MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
    DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
    unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
  assert(IdxMulOpd == 1);
 
  Register NewVR =
      genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
 
  return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
                          FMAInstKind::Indexed, &NewVR);
}
 
/// genMaddR - Generate madd instruction and combine mul and add using
/// an extra virtual register
/// Example - an ADD intermediate needs to be stored in a register:
///   MUL I=A,B,0
///   ADD R,I,Imm
///   ==> ORR  V, ZR, Imm
///   ==> MADD R,A,B,V
/// \param MF Containing MachineFunction
/// \param MRI Register information
/// \param TII Target information
/// \param Root is the ADD instruction
/// \param [out] InsInstrs is a vector of machine instructions and will
/// contain the generated madd instruction
/// \param IdxMulOpd is index of operand in Root that is the result of
/// the MUL. In the example above IdxMulOpd is 1.
/// \param MaddOpc the opcode fo the madd instruction
/// \param VR is a virtual register that holds the value of an ADD operand
/// (V in the example above).
/// \param RC Register class of operands
static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
                              const TargetInstrInfo *TII, MachineInstr &Root,
                              SmallVectorImpl<MachineInstr *> &InsInstrs,
                              unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
                              const TargetRegisterClass *RC) {
  assert(IdxMulOpd == 1 || IdxMulOpd == 2);
 
  MachineInstr *MUL = MRI.getUniqueVRegDef(Root.getOperand(IdxMulOpd).getReg());
  Register ResultReg = Root.getOperand(0).getReg();
  Register SrcReg0 = MUL->getOperand(1).getReg();
  bool Src0IsKill = MUL->getOperand(1).isKill();
  Register SrcReg1 = MUL->getOperand(2).getReg();
  bool Src1IsKill = MUL->getOperand(2).isKill();
 
  if (ResultReg.isVirtual())
    MRI.constrainRegClass(ResultReg, RC);
  if (SrcReg0.isVirtual())
    MRI.constrainRegClass(SrcReg0, RC);
  if (SrcReg1.isVirtual())
    MRI.constrainRegClass(SrcReg1, RC);
  if (Register::isVirtualRegister(VR))
    MRI.constrainRegClass(VR, RC);
 
  MachineInstrBuilder MIB =
      BuildMI(MF, MIMetadata(Root), TII->get(MaddOpc), ResultReg)
          .addReg(SrcReg0, getKillRegState(Src0IsKill))
          .addReg(SrcReg1, getKillRegState(Src1IsKill))
          .addReg(VR);
  // Insert the MADD
  InsInstrs.push_back(MIB);
  return MUL;
}
 
/// Do the following transformation
/// A - (B + C)  ==>   (A - B) - C
/// A - (B + C)  ==>   (A - C) - B
static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI,
                             const TargetInstrInfo *TII, MachineInstr &Root,
                             SmallVectorImpl<MachineInstr *> &InsInstrs,
                             SmallVectorImpl<MachineInstr *> &DelInstrs,
                             unsigned IdxOpd1,
                             DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
  assert(IdxOpd1 == 1 || IdxOpd1 == 2);
  unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
  MachineInstr *AddMI = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
 
  Register ResultReg = Root.getOperand(0).getReg();
  Register RegA = Root.getOperand(1).getReg();
  bool RegAIsKill = Root.getOperand(1).isKill();
  Register RegB = AddMI->getOperand(IdxOpd1).getReg();
  bool RegBIsKill = AddMI->getOperand(IdxOpd1).isKill();
  Register RegC = AddMI->getOperand(IdxOtherOpd).getReg();
  bool RegCIsKill = AddMI->getOperand(IdxOtherOpd).isKill();
  Register NewVR =
      MRI.createVirtualRegister(MRI.getRegClass(Root.getOperand(2).getReg()));
 
  unsigned Opcode = Root.getOpcode();
  if (Opcode == AArch64::SUBSWrr)
    Opcode = AArch64::SUBWrr;
  else if (Opcode == AArch64::SUBSXrr)
    Opcode = AArch64::SUBXrr;
  else
    assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
           "Unexpected instruction opcode.");
 
  uint32_t Flags = Root.mergeFlagsWith(*AddMI);
  Flags &= ~MachineInstr::NoSWrap;
  Flags &= ~MachineInstr::NoUWrap;
 
  MachineInstrBuilder MIB1 =
      BuildMI(MF, MIMetadata(Root), TII->get(Opcode), NewVR)
          .addReg(RegA, getKillRegState(RegAIsKill))
          .addReg(RegB, getKillRegState(RegBIsKill))
          .setMIFlags(Flags);
  MachineInstrBuilder MIB2 =
      BuildMI(MF, MIMetadata(Root), TII->get(Opcode), ResultReg)
          .addReg(NewVR, getKillRegState(true))
          .addReg(RegC, getKillRegState(RegCIsKill))
          .setMIFlags(Flags);
 
  InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
  InsInstrs.push_back(MIB1);
  InsInstrs.push_back(MIB2);
  DelInstrs.push_back(AddMI);
  DelInstrs.push_back(&Root);
}
 
unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
    unsigned int AccumulatorOpCode) const {
  switch (AccumulatorOpCode) {
  case AArch64::UABALB_ZZZ_D:
  case AArch64::SABALB_ZZZ_D:
  case AArch64::UABALT_ZZZ_D:
  case AArch64::SABALT_ZZZ_D:
    return AArch64::ADD_ZZZ_D;
  case AArch64::UABALB_ZZZ_H:
  case AArch64::SABALB_ZZZ_H:
  case AArch64::UABALT_ZZZ_H:
  case AArch64::SABALT_ZZZ_H:
    return AArch64::ADD_ZZZ_H;
  case AArch64::UABALB_ZZZ_S:
  case AArch64::SABALB_ZZZ_S:
  case AArch64::UABALT_ZZZ_S:
  case AArch64::SABALT_ZZZ_S:
    return AArch64::ADD_ZZZ_S;
  case AArch64::UABALv16i8_v8i16:
  case AArch64::SABALv8i8_v8i16:
  case AArch64::SABAv8i16:
  case AArch64::UABAv8i16:
    return AArch64::ADDv8i16;
  case AArch64::SABALv2i32_v2i64:
  case AArch64::UABALv2i32_v2i64:
  case AArch64::SABALv4i32_v2i64:
    return AArch64::ADDv2i64;
  case AArch64::UABALv4i16_v4i32:
  case AArch64::SABALv4i16_v4i32:
  case AArch64::SABALv8i16_v4i32:
  case AArch64::SABAv4i32:
  case AArch64::UABAv4i32:
    return AArch64::ADDv4i32;
  case AArch64::UABALv4i32_v2i64:
    return AArch64::ADDv2i64;
  case AArch64::UABALv8i16_v4i32:
    return AArch64::ADDv4i32;
  case AArch64::UABALv8i8_v8i16:
  case AArch64::SABALv16i8_v8i16:
    return AArch64::ADDv8i16;
  case AArch64::UABAv16i8:
  case AArch64::SABAv16i8:
    return AArch64::ADDv16i8;
  case AArch64::UABAv4i16:
  case AArch64::SABAv4i16:
    return AArch64::ADDv4i16;
  case AArch64::UABAv2i32:
  case AArch64::SABAv2i32:
    return AArch64::ADDv2i32;
  case AArch64::UABAv8i8:
  case AArch64::SABAv8i8:
    return AArch64::ADDv8i8;
  default:
    llvm_unreachable("Unknown accumulator opcode");
  }
}
 
/// When getMachineCombinerPatterns() finds potential patterns,
/// this function generates the instructions that could replace the
/// original code sequence
void AArch64InstrInfo::genAlternativeCodeSequence(
    MachineInstr &Root, unsigned Pattern,
    SmallVectorImpl<MachineInstr *> &InsInstrs,
    SmallVectorImpl<MachineInstr *> &DelInstrs,
    DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
  MachineBasicBlock &MBB = *Root.getParent();
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
  MachineFunction &MF = *MBB.getParent();
  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
 
  MachineInstr *MUL = nullptr;
  const TargetRegisterClass *RC;
  unsigned Opc;
  switch (Pattern) {
  default:
    // Reassociate instructions.
    TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
                                                DelInstrs, InstrIdxForVirtReg);
    return;
  case AArch64MachineCombinerPattern::SUBADD_OP1:
    // A - (B + C)
    // ==> (A - B) - C
    genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, 1,
                     InstrIdxForVirtReg);
    return;
  case AArch64MachineCombinerPattern::SUBADD_OP2:
    // A - (B + C)
    // ==> (A - C) - B
    genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, 2,
                     InstrIdxForVirtReg);
    return;
  case AArch64MachineCombinerPattern::MULADDW_OP1:
  case AArch64MachineCombinerPattern::MULADDX_OP1:
    // MUL I=A,B,0
    // ADD R,I,C
    // ==> MADD R,A,B,C
    // --- Create(MADD);
    if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP1) {
      Opc = AArch64::MADDWrrr;
      RC = &AArch64::GPR32RegClass;
    } else {
      Opc = AArch64::MADDXrrr;
      RC = &AArch64::GPR64RegClass;
    }
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDW_OP2:
  case AArch64MachineCombinerPattern::MULADDX_OP2:
    // MUL I=A,B,0
    // ADD R,C,I
    // ==> MADD R,A,B,C
    // --- Create(MADD);
    if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP2) {
      Opc = AArch64::MADDWrrr;
      RC = &AArch64::GPR32RegClass;
    } else {
      Opc = AArch64::MADDXrrr;
      RC = &AArch64::GPR64RegClass;
    }
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDWI_OP1:
  case AArch64MachineCombinerPattern::MULADDXI_OP1:
  case AArch64MachineCombinerPattern::MULSUBWI_OP1:
  case AArch64MachineCombinerPattern::MULSUBXI_OP1: {
    // MUL I=A,B,0
    // ADD/SUB R,I,Imm
    // ==> MOV V, Imm/-Imm
    // ==> MADD R,A,B,V
    // --- Create(MADD);
    const TargetRegisterClass *RC;
    unsigned BitSize, MovImm;
    if (Pattern == AArch64MachineCombinerPattern::MULADDWI_OP1 ||
        Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1) {
      MovImm = AArch64::MOVi32imm;
      RC = &AArch64::GPR32spRegClass;
      BitSize = 32;
      Opc = AArch64::MADDWrrr;
      RC = &AArch64::GPR32RegClass;
    } else {
      MovImm = AArch64::MOVi64imm;
      RC = &AArch64::GPR64spRegClass;
      BitSize = 64;
      Opc = AArch64::MADDXrrr;
      RC = &AArch64::GPR64RegClass;
    }
    Register NewVR = MRI.createVirtualRegister(RC);
    uint64_t Imm = Root.getOperand(2).getImm();
 
    if (Root.getOperand(3).isImm()) {
      unsigned Val = Root.getOperand(3).getImm();
      Imm = Imm << Val;
    }
    bool IsSub = Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1 ||
                 Pattern == AArch64MachineCombinerPattern::MULSUBXI_OP1;
    uint64_t UImm = SignExtend64(IsSub ? -Imm : Imm, BitSize);
    // Check that the immediate can be composed via a single instruction.
    SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
    AArch64_IMM::expandMOVImm(UImm, BitSize, Insn);
    if (Insn.size() != 1)
      return;
    MachineInstrBuilder MIB1 =
        BuildMI(MF, MIMetadata(Root), TII->get(MovImm), NewVR)
            .addImm(IsSub ? -Imm : Imm);
    InsInstrs.push_back(MIB1);
    InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
    MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
    break;
  }
  case AArch64MachineCombinerPattern::MULSUBW_OP1:
  case AArch64MachineCombinerPattern::MULSUBX_OP1: {
    // MUL I=A,B,0
    // SUB R,I, C
    // ==> SUB  V, 0, C
    // ==> MADD R,A,B,V // = -C + A*B
    // --- Create(MADD);
    const TargetRegisterClass *SubRC;
    unsigned SubOpc, ZeroReg;
    if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP1) {
      SubOpc = AArch64::SUBWrr;
      SubRC = &AArch64::GPR32spRegClass;
      ZeroReg = AArch64::WZR;
      Opc = AArch64::MADDWrrr;
      RC = &AArch64::GPR32RegClass;
    } else {
      SubOpc = AArch64::SUBXrr;
      SubRC = &AArch64::GPR64spRegClass;
      ZeroReg = AArch64::XZR;
      Opc = AArch64::MADDXrrr;
      RC = &AArch64::GPR64RegClass;
    }
    Register NewVR = MRI.createVirtualRegister(SubRC);
    // SUB NewVR, 0, C
    MachineInstrBuilder MIB1 =
        BuildMI(MF, MIMetadata(Root), TII->get(SubOpc), NewVR)
            .addReg(ZeroReg)
            .add(Root.getOperand(2));
    InsInstrs.push_back(MIB1);
    InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
    MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, 1, Opc, NewVR, RC);
    break;
  }
  case AArch64MachineCombinerPattern::MULSUBW_OP2:
  case AArch64MachineCombinerPattern::MULSUBX_OP2:
    // MUL I=A,B,0
    // SUB R,C,I
    // ==> MSUB R,A,B,C (computes C - A*B)
    // --- Create(MSUB);
    if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP2) {
      Opc = AArch64::MSUBWrrr;
      RC = &AArch64::GPR32RegClass;
    } else {
      Opc = AArch64::MSUBXrrr;
      RC = &AArch64::GPR64RegClass;
    }
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
    Opc = AArch64::MLAv8i8;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
    Opc = AArch64::MLAv8i8;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
    Opc = AArch64::MLAv16i8;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
    Opc = AArch64::MLAv16i8;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
    Opc = AArch64::MLAv4i16;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
    Opc = AArch64::MLAv4i16;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
    Opc = AArch64::MLAv8i16;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
    Opc = AArch64::MLAv8i16;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
    Opc = AArch64::MLAv2i32;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
    Opc = AArch64::MLAv2i32;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
    Opc = AArch64::MLAv4i32;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
    Opc = AArch64::MLAv4i32;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
    Opc = AArch64::MLAv8i8;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i8,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
    Opc = AArch64::MLSv8i8;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
    Opc = AArch64::MLAv16i8;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv16i8,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
    Opc = AArch64::MLSv16i8;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
    Opc = AArch64::MLAv4i16;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
    Opc = AArch64::MLSv4i16;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
    Opc = AArch64::MLAv8i16;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
    Opc = AArch64::MLSv8i16;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
    Opc = AArch64::MLAv2i32;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
    Opc = AArch64::MLSv2i32;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
    Opc = AArch64::MLAv4i32;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
    Opc = AArch64::MLSv4i32;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
    Opc = AArch64::MLAv4i16_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
    Opc = AArch64::MLAv4i16_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
    Opc = AArch64::MLAv8i16_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
    Opc = AArch64::MLAv8i16_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
    Opc = AArch64::MLAv2i32_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
    Opc = AArch64::MLAv2i32_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
    Opc = AArch64::MLAv4i32_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
    Opc = AArch64::MLAv4i32_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
    Opc = AArch64::MLAv4i16_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i16,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
    Opc = AArch64::MLSv4i16_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
    Opc = AArch64::MLAv8i16_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv8i16,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
    Opc = AArch64::MLSv8i16_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
    Opc = AArch64::MLAv2i32_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv2i32,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
    Opc = AArch64::MLSv2i32_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
    Opc = AArch64::MLAv4i32_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
                                 InstrIdxForVirtReg, 1, Opc, AArch64::NEGv4i32,
                                 RC);
    break;
  case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
    Opc = AArch64::MLSv4i32_indexed;
    RC = &AArch64::FPR128RegClass;
    MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
 
  // Floating Point Support
  case AArch64MachineCombinerPattern::FMULADDH_OP1:
    Opc = AArch64::FMADDHrrr;
    RC = &AArch64::FPR16RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULADDS_OP1:
    Opc = AArch64::FMADDSrrr;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULADDD_OP1:
    Opc = AArch64::FMADDDrrr;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::FMULADDH_OP2:
    Opc = AArch64::FMADDHrrr;
    RC = &AArch64::FPR16RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULADDS_OP2:
    Opc = AArch64::FMADDSrrr;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULADDD_OP2:
    Opc = AArch64::FMADDDrrr;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
    Opc = AArch64::FMLAv1i32_indexed;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                           FMAInstKind::Indexed);
    break;
  case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
    Opc = AArch64::FMLAv1i32_indexed;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
 
  case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
    Opc = AArch64::FMLAv1i64_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                           FMAInstKind::Indexed);
    break;
  case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
    Opc = AArch64::FMLAv1i64_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
 
  case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
    RC = &AArch64::FPR64RegClass;
    Opc = AArch64::FMLAv4i16_indexed;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                           FMAInstKind::Indexed);
    break;
  case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
    RC = &AArch64::FPR64RegClass;
    Opc = AArch64::FMLAv4f16;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                           FMAInstKind::Accumulator);
    break;
  case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
    RC = &AArch64::FPR64RegClass;
    Opc = AArch64::FMLAv4i16_indexed;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
  case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
    RC = &AArch64::FPR64RegClass;
    Opc = AArch64::FMLAv4f16;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Accumulator);
    break;
 
  case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
    RC = &AArch64::FPR64RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
      Opc = AArch64::FMLAv2i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLAv2f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
  case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
    RC = &AArch64::FPR64RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
      Opc = AArch64::FMLAv2i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLAv2f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
 
  case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
    RC = &AArch64::FPR128RegClass;
    Opc = AArch64::FMLAv8i16_indexed;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                           FMAInstKind::Indexed);
    break;
  case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
    RC = &AArch64::FPR128RegClass;
    Opc = AArch64::FMLAv8f16;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                           FMAInstKind::Accumulator);
    break;
  case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
    RC = &AArch64::FPR128RegClass;
    Opc = AArch64::FMLAv8i16_indexed;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
  case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
    RC = &AArch64::FPR128RegClass;
    Opc = AArch64::FMLAv8f16;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Accumulator);
    break;
 
  case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
    RC = &AArch64::FPR128RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
      Opc = AArch64::FMLAv2i64_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLAv2f64;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
  case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
    RC = &AArch64::FPR128RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
      Opc = AArch64::FMLAv2i64_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLAv2f64;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
 
  case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
    RC = &AArch64::FPR128RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
      Opc = AArch64::FMLAv4i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLAv4f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
 
  case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
  case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
    RC = &AArch64::FPR128RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
      Opc = AArch64::FMLAv4i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLAv4f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
 
  case AArch64MachineCombinerPattern::FMULSUBH_OP1:
    Opc = AArch64::FNMSUBHrrr;
    RC = &AArch64::FPR16RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULSUBS_OP1:
    Opc = AArch64::FNMSUBSrrr;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULSUBD_OP1:
    Opc = AArch64::FNMSUBDrrr;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
    Opc = AArch64::FNMADDHrrr;
    RC = &AArch64::FPR16RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
    Opc = AArch64::FNMADDSrrr;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
    Opc = AArch64::FNMADDDrrr;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::FMULSUBH_OP2:
    Opc = AArch64::FMSUBHrrr;
    RC = &AArch64::FPR16RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULSUBS_OP2:
    Opc = AArch64::FMSUBSrrr;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
  case AArch64MachineCombinerPattern::FMULSUBD_OP2:
    Opc = AArch64::FMSUBDrrr;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC);
    break;
 
  case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
    Opc = AArch64::FMLSv1i32_indexed;
    RC = &AArch64::FPR32RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
 
  case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
    Opc = AArch64::FMLSv1i64_indexed;
    RC = &AArch64::FPR64RegClass;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
 
  case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
  case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
    RC = &AArch64::FPR64RegClass;
    Register NewVR = MRI.createVirtualRegister(RC);
    MachineInstrBuilder MIB1 =
        BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv4f16), NewVR)
            .add(Root.getOperand(2));
    InsInstrs.push_back(MIB1);
    InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
    if (Pattern == AArch64MachineCombinerPattern::FMLSv4f16_OP1) {
      Opc = AArch64::FMLAv4f16;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator, &NewVR);
    } else {
      Opc = AArch64::FMLAv4i16_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed, &NewVR);
    }
    break;
  }
  case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
    RC = &AArch64::FPR64RegClass;
    Opc = AArch64::FMLSv4f16;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Accumulator);
    break;
  case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
    RC = &AArch64::FPR64RegClass;
    Opc = AArch64::FMLSv4i16_indexed;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
 
  case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
  case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
    RC = &AArch64::FPR64RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
      Opc = AArch64::FMLSv2i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLSv2f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
 
  case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
  case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
    RC = &AArch64::FPR128RegClass;
    Register NewVR = MRI.createVirtualRegister(RC);
    MachineInstrBuilder MIB1 =
        BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv8f16), NewVR)
            .add(Root.getOperand(2));
    InsInstrs.push_back(MIB1);
    InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
    if (Pattern == AArch64MachineCombinerPattern::FMLSv8f16_OP1) {
      Opc = AArch64::FMLAv8f16;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator, &NewVR);
    } else {
      Opc = AArch64::FMLAv8i16_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed, &NewVR);
    }
    break;
  }
  case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
    RC = &AArch64::FPR128RegClass;
    Opc = AArch64::FMLSv8f16;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Accumulator);
    break;
  case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
    RC = &AArch64::FPR128RegClass;
    Opc = AArch64::FMLSv8i16_indexed;
    MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                           FMAInstKind::Indexed);
    break;
 
  case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
  case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
    RC = &AArch64::FPR128RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
      Opc = AArch64::FMLSv2i64_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLSv2f64;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
 
  case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
  case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
    RC = &AArch64::FPR128RegClass;
    if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
      Opc = AArch64::FMLSv4i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Indexed);
    } else {
      Opc = AArch64::FMLSv4f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 2, Opc, RC,
                             FMAInstKind::Accumulator);
    }
    break;
  case AArch64MachineCombinerPattern::FMLSv2f32_OP1:
  case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
    RC = &AArch64::FPR64RegClass;
    Register NewVR = MRI.createVirtualRegister(RC);
    MachineInstrBuilder MIB1 =
        BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv2f32), NewVR)
            .add(Root.getOperand(2));
    InsInstrs.push_back(MIB1);
    InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
    if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
      Opc = AArch64::FMLAv2i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed, &NewVR);
    } else {
      Opc = AArch64::FMLAv2f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator, &NewVR);
    }
    break;
  }
  case AArch64MachineCombinerPattern::FMLSv4f32_OP1:
  case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
    RC = &AArch64::FPR128RegClass;
    Register NewVR = MRI.createVirtualRegister(RC);
    MachineInstrBuilder MIB1 =
        BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv4f32), NewVR)
            .add(Root.getOperand(2));
    InsInstrs.push_back(MIB1);
    InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
    if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
      Opc = AArch64::FMLAv4i32_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed, &NewVR);
    } else {
      Opc = AArch64::FMLAv4f32;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator, &NewVR);
    }
    break;
  }
  case AArch64MachineCombinerPattern::FMLSv2f64_OP1:
  case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
    RC = &AArch64::FPR128RegClass;
    Register NewVR = MRI.createVirtualRegister(RC);
    MachineInstrBuilder MIB1 =
        BuildMI(MF, MIMetadata(Root), TII->get(AArch64::FNEGv2f64), NewVR)
            .add(Root.getOperand(2));
    InsInstrs.push_back(MIB1);
    InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
    if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
      Opc = AArch64::FMLAv2i64_indexed;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Indexed, &NewVR);
    } else {
      Opc = AArch64::FMLAv2f64;
      MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, 1, Opc, RC,
                             FMAInstKind::Accumulator, &NewVR);
    }
    break;
  }
  case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2: {
    unsigned IdxDupOp =
        (Pattern == AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1) ? 1
                                                                          : 2;
    genIndexedMultiply(Root, InsInstrs, IdxDupOp, AArch64::FMULv2i32_indexed,
                       &AArch64::FPR128RegClass, MRI);
    break;
  }
  case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2: {
    unsigned IdxDupOp =
        (Pattern == AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1) ? 1
                                                                          : 2;
    genIndexedMultiply(Root, InsInstrs, IdxDupOp, AArch64::FMULv2i64_indexed,
                       &AArch64::FPR128RegClass, MRI);
    break;
  }
  case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2: {
    unsigned IdxDupOp =
        (Pattern == AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1) ? 1
                                                                          : 2;
    genIndexedMultiply(Root, InsInstrs, IdxDupOp, AArch64::FMULv4i16_indexed,
                       &AArch64::FPR128_loRegClass, MRI);
    break;
  }
  case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2: {
    unsigned IdxDupOp =
        (Pattern == AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1) ? 1
                                                                          : 2;
    genIndexedMultiply(Root, InsInstrs, IdxDupOp, AArch64::FMULv4i32_indexed,
                       &AArch64::FPR128RegClass, MRI);
    break;
  }
  case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
  case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2: {
    unsigned IdxDupOp =
        (Pattern == AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1) ? 1
                                                                          : 2;
    genIndexedMultiply(Root, InsInstrs, IdxDupOp, AArch64::FMULv8i16_indexed,
                       &AArch64::FPR128_loRegClass, MRI);
    break;
  }
  case