SIISelLowering.cpp

Go to the documentation of this file.

//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file
/// Custom DAG lowering for SI
//
//===----------------------------------------------------------------------===//
 
#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUSelectionDAGInfo.h"
#include "AMDGPUTargetMachine.h"
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/PseudoSourceValueManager.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/ModRef.h"
#include "llvm/TargetParser/AtomicScope.h"
#include "llvm/Transforms/Utils/LowerAtomic.h"
#include <optional>
 
using namespace llvm;
using namespace llvm::SDPatternMatch;
 
#define DEBUG_TYPE "si-lower"
 
STATISTIC(NumTailCalls, "Number of tail calls");
 
static cl::opt<bool>
    DisableLoopAlignment("amdgpu-disable-loop-alignment",
                         cl::desc("Do not align and prefetch loops"),
                         cl::init(false));
 
static cl::opt<bool> UseDivergentRegisterIndexing(
    "amdgpu-use-divergent-register-indexing", cl::Hidden,
    cl::desc("Use indirect register addressing for divergent indexes"),
    cl::init(false));
 
static DenormalFPEnv getDenormalFPEnv(const MachineFunction &MF) {
  return MF.getInfo<SIMachineFunctionInfo>()->getMode().getDenormalFPEnv();
}
 
static bool denormalModeIsFlushAllF32(const MachineFunction &MF) {
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign();
}
 
static bool denormalModeIsFlushAllF64F16(const MachineFunction &MF) {
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  return Info->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign();
}
 
static unsigned findFirstFreeSGPR(CCState &CCInfo) {
  unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
  for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
    if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
      return AMDGPU::SGPR0 + Reg;
    }
  }
  llvm_unreachable("Cannot allocate sgpr");
}
 
SITargetLowering::SITargetLowering(const TargetMachine &TM,
                                   const GCNSubtarget &STI)
    : AMDGPUTargetLowering(TM, STI, STI), Subtarget(&STI) {
  addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
  addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
 
  addRegisterClass(MVT::i32, &AMDGPU::SReg_32RegClass);
 
  const SIRegisterInfo *TRI = STI.getRegisterInfo();
  const TargetRegisterClass *V32RegClass =
      TRI->getDefaultVectorSuperClassForBitWidth(32);
  addRegisterClass(MVT::f32, V32RegClass);
 
  addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
 
  const TargetRegisterClass *V64RegClass =
      TRI->getDefaultVectorSuperClassForBitWidth(64);
 
  addRegisterClass(MVT::f64, V64RegClass);
  addRegisterClass(MVT::v2f32, V64RegClass);
  addRegisterClass(MVT::Untyped, V64RegClass);
 
  addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass);
  addRegisterClass(MVT::v3f32, TRI->getDefaultVectorSuperClassForBitWidth(96));
 
  addRegisterClass(MVT::v2i64, &AMDGPU::SGPR_128RegClass);
  addRegisterClass(MVT::v2f64, &AMDGPU::SGPR_128RegClass);
 
  addRegisterClass(MVT::v4i32, &AMDGPU::SGPR_128RegClass);
  addRegisterClass(MVT::v4f32, TRI->getDefaultVectorSuperClassForBitWidth(128));
 
  addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass);
  addRegisterClass(MVT::v5f32, TRI->getDefaultVectorSuperClassForBitWidth(160));
 
  addRegisterClass(MVT::v6i32, &AMDGPU::SGPR_192RegClass);
  addRegisterClass(MVT::v6f32, TRI->getDefaultVectorSuperClassForBitWidth(192));
 
  addRegisterClass(MVT::v3i64, &AMDGPU::SGPR_192RegClass);
  addRegisterClass(MVT::v3f64, TRI->getDefaultVectorSuperClassForBitWidth(192));
 
  addRegisterClass(MVT::v7i32, &AMDGPU::SGPR_224RegClass);
  addRegisterClass(MVT::v7f32, TRI->getDefaultVectorSuperClassForBitWidth(224));
 
  addRegisterClass(MVT::v8i32, &AMDGPU::SGPR_256RegClass);
  addRegisterClass(MVT::v8f32, TRI->getDefaultVectorSuperClassForBitWidth(256));
 
  addRegisterClass(MVT::v4i64, &AMDGPU::SGPR_256RegClass);
  addRegisterClass(MVT::v4f64, TRI->getDefaultVectorSuperClassForBitWidth(256));
 
  addRegisterClass(MVT::v9i32, &AMDGPU::SGPR_288RegClass);
  addRegisterClass(MVT::v9f32, TRI->getDefaultVectorSuperClassForBitWidth(288));
 
  addRegisterClass(MVT::v10i32, &AMDGPU::SGPR_320RegClass);
  addRegisterClass(MVT::v10f32,
                   TRI->getDefaultVectorSuperClassForBitWidth(320));
 
  addRegisterClass(MVT::v11i32, &AMDGPU::SGPR_352RegClass);
  addRegisterClass(MVT::v11f32,
                   TRI->getDefaultVectorSuperClassForBitWidth(352));
 
  addRegisterClass(MVT::v12i32, &AMDGPU::SGPR_384RegClass);
  addRegisterClass(MVT::v12f32,
                   TRI->getDefaultVectorSuperClassForBitWidth(384));
 
  addRegisterClass(MVT::v16i32, &AMDGPU::SGPR_512RegClass);
  addRegisterClass(MVT::v16f32,
                   TRI->getDefaultVectorSuperClassForBitWidth(512));
 
  addRegisterClass(MVT::v8i64, &AMDGPU::SGPR_512RegClass);
  addRegisterClass(MVT::v8f64, TRI->getDefaultVectorSuperClassForBitWidth(512));
 
  addRegisterClass(MVT::v16i64, &AMDGPU::SGPR_1024RegClass);
  addRegisterClass(MVT::v16f64,
                   TRI->getDefaultVectorSuperClassForBitWidth(1024));
 
  if (Subtarget->has16BitInsts()) {
    if (Subtarget->useRealTrue16Insts()) {
      addRegisterClass(MVT::i16, &AMDGPU::VGPR_16RegClass);
      addRegisterClass(MVT::f16, &AMDGPU::VGPR_16RegClass);
      addRegisterClass(MVT::bf16, &AMDGPU::VGPR_16RegClass);
    } else {
      addRegisterClass(MVT::i16, &AMDGPU::SReg_32RegClass);
      addRegisterClass(MVT::f16, &AMDGPU::SReg_32RegClass);
      addRegisterClass(MVT::bf16, &AMDGPU::SReg_32RegClass);
    }
 
    // Unless there are also VOP3P operations, not operations are really legal.
    addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v2bf16, &AMDGPU::SReg_32RegClass);
    addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v4bf16, &AMDGPU::SReg_64RegClass);
    addRegisterClass(MVT::v8i16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v8f16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v8bf16, &AMDGPU::SGPR_128RegClass);
    addRegisterClass(MVT::v16i16, &AMDGPU::SGPR_256RegClass);
    addRegisterClass(MVT::v16f16, &AMDGPU::SGPR_256RegClass);
    addRegisterClass(MVT::v16bf16, &AMDGPU::SGPR_256RegClass);
    addRegisterClass(MVT::v32i16, &AMDGPU::SGPR_512RegClass);
    addRegisterClass(MVT::v32f16, &AMDGPU::SGPR_512RegClass);
    addRegisterClass(MVT::v32bf16, &AMDGPU::SGPR_512RegClass);
  }
 
  addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
  addRegisterClass(MVT::v32f32,
                   TRI->getDefaultVectorSuperClassForBitWidth(1024));
 
  computeRegisterProperties(Subtarget->getRegisterInfo());
 
  setMinFunctionAlignment(Align(4));
  setPrefFunctionAlignment(Align(STI.getInstCacheLineSize()));
 
  // The boolean content concept here is too inflexible. Compares only ever
  // really produce a 1-bit result. Any copy/extend from these will turn into a
  // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as
  // it's what most targets use.
  setBooleanContents(ZeroOrOneBooleanContent);
  setBooleanVectorContents(ZeroOrOneBooleanContent);
 
  // We need to custom lower vector stores from local memory
  setOperationAction(ISD::LOAD,
                     {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
                      MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
                      MVT::i1, MVT::v32i32},
                     Custom);
 
  setOperationAction(ISD::STORE,
                     {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
                      MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32,
                      MVT::i1, MVT::v32i32},
                     Custom);
 
  if (isTypeLegal(MVT::bf16)) {
    for (unsigned Opc :
         {ISD::FADD,     ISD::FSUB,       ISD::FMUL,    ISD::FDIV,
          ISD::FREM,     ISD::FMA,        ISD::FMINNUM, ISD::FMAXNUM,
          ISD::FMINIMUM, ISD::FMAXIMUM,   ISD::FSQRT,   ISD::FCBRT,
          ISD::FSIN,     ISD::FCOS,       ISD::FPOW,    ISD::FPOWI,
          ISD::FLDEXP,   ISD::FFREXP,     ISD::FLOG,    ISD::FLOG2,
          ISD::FLOG10,   ISD::FEXP,       ISD::FEXP2,   ISD::FEXP10,
          ISD::FCEIL,    ISD::FTRUNC,     ISD::FRINT,   ISD::FNEARBYINT,
          ISD::FROUND,   ISD::FROUNDEVEN, ISD::FFLOOR,  ISD::FCANONICALIZE,
          ISD::SETCC}) {
      setOperationAction(Opc, MVT::bf16, Promote);
    }
 
    // Only targets with packed bf16 instructions, e.g. gfx13.
    if (Subtarget->hasBF16PackedInsts()) {
      // Don't use Expand for fsub - the DAG combiner will undo fadd+fneg back
      // to fsub, causing a libcall (which doesn't exist for bf16). Instead,
      // directly expand to widened v2bf16 operations.
      setOperationAction(ISD::FSUB, MVT::bf16, Custom);
      // Promote scalar operations to a v2bf16 operation with an unused high
      // lane.
      for (unsigned Opc : {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FMAXNUM,
                           ISD::FMINNUM, ISD::FCANONICALIZE})
        AddPromotedToType(Opc, MVT::bf16, MVT::v2bf16);
    }
 
    setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
 
    setOperationAction(ISD::SELECT, MVT::bf16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::bf16, MVT::i16);
 
    setOperationAction(ISD::FABS, MVT::bf16, Legal);
    setOperationAction(ISD::FNEG, MVT::bf16, Legal);
    setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Legal);
 
    // We only need to custom lower because we can't specify an action for bf16
    // sources.
    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
  }
 
  setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
  setTruncStoreAction(MVT::v3i32, MVT::v3i16, Expand);
  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
  setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
  setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
  setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand);
  setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand);
  setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand);
  setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand);
  setTruncStoreAction(MVT::v2i16, MVT::v2i8, Expand);
  setTruncStoreAction(MVT::v4i16, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i16, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v16i16, MVT::v16i8, Expand);
  setTruncStoreAction(MVT::v32i16, MVT::v32i8, Expand);
 
  setTruncStoreAction(MVT::v3i64, MVT::v3i16, Expand);
  setTruncStoreAction(MVT::v3i64, MVT::v3i32, Expand);
  setTruncStoreAction(MVT::v4i64, MVT::v4i8, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i8, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i16, Expand);
  setTruncStoreAction(MVT::v8i64, MVT::v8i32, Expand);
  setTruncStoreAction(MVT::v16i64, MVT::v16i32, Expand);
 
  setOperationAction(ISD::GlobalAddress, {MVT::i32, MVT::i64}, Custom);
  setOperationAction(ISD::BlockAddress, {MVT::i32, MVT::i64}, Custom);
  setOperationAction(ISD::ExternalSymbol, {MVT::i32, MVT::i64}, Custom);
 
  setOperationAction(ISD::SELECT, MVT::i1, Promote);
  setOperationAction(ISD::SELECT, MVT::i64, Custom);
  setOperationAction(ISD::SELECT, MVT::f64, Promote);
  AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64);
 
  setOperationAction(ISD::FSQRT, {MVT::f32, MVT::f64}, Custom);
 
  setOperationAction(ISD::SELECT_CC,
                     {MVT::f32, MVT::i32, MVT::i64, MVT::f64, MVT::i1}, Expand);
 
  setOperationAction(ISD::SETCC, MVT::i1, Promote);
  setOperationAction(ISD::SETCC, {MVT::v2i1, MVT::v4i1}, Expand);
  AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32);
 
  setOperationAction(ISD::TRUNCATE,
                     {MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32,
                      MVT::v6i32, MVT::v7i32, MVT::v8i32, MVT::v9i32,
                      MVT::v10i32, MVT::v11i32, MVT::v12i32, MVT::v16i32},
                     Expand);
  setOperationAction(ISD::FP_ROUND,
                     {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
                      MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v9f32,
                      MVT::v10f32, MVT::v11f32, MVT::v12f32, MVT::v16f32},
                     Expand);
 
  setOperationAction(ISD::SIGN_EXTEND_INREG,
                     {MVT::v2i1, MVT::v4i1, MVT::v2i8, MVT::v4i8, MVT::v2i16,
                      MVT::v3i16, MVT::v4i16, MVT::Other},
                     Custom);
 
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
  setOperationAction(ISD::BR_CC,
                     {MVT::i1, MVT::i32, MVT::i64, MVT::f32, MVT::f64}, Expand);
 
  setOperationAction({ISD::ABS, ISD::UADDO, ISD::USUBO}, MVT::i32, Legal);
  setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i64, Legal);
 
  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i32, Legal);
  setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY}, MVT::i64, Legal);
 
  setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64,
                     Expand);
 
  setOperationAction(ISD::INLINEASM, MVT::Other, Custom);
 
  // We only support LOAD/STORE and vector manipulation ops for vectors
  // with > 4 elements.
  for (MVT VT :
       {MVT::v8i32,   MVT::v8f32,  MVT::v9i32,  MVT::v9f32,  MVT::v10i32,
        MVT::v10f32,  MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32,
        MVT::v16i32,  MVT::v16f32, MVT::v2i64,  MVT::v2f64,  MVT::v4i16,
        MVT::v4f16,   MVT::v4bf16, MVT::v3i64,  MVT::v3f64,  MVT::v6i32,
        MVT::v6f32,   MVT::v4i64,  MVT::v4f64,  MVT::v8i64,  MVT::v8f64,
        MVT::v8i16,   MVT::v8f16,  MVT::v8bf16, MVT::v16i16, MVT::v16f16,
        MVT::v16bf16, MVT::v16i64, MVT::v16f64, MVT::v32i32, MVT::v32f32,
        MVT::v32i16,  MVT::v32f16, MVT::v32bf16}) {
    for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
      switch (Op) {
      case ISD::LOAD:
      case ISD::STORE:
      case ISD::BUILD_VECTOR:
      case ISD::BITCAST:
      case ISD::UNDEF:
      case ISD::POISON:
      case ISD::EXTRACT_VECTOR_ELT:
      case ISD::INSERT_VECTOR_ELT:
      case ISD::SCALAR_TO_VECTOR:
      case ISD::IS_FPCLASS:
        break;
      case ISD::EXTRACT_SUBVECTOR:
      case ISD::INSERT_SUBVECTOR:
      case ISD::CONCAT_VECTORS:
        setOperationAction(Op, VT, Custom);
        break;
      default:
        setOperationAction(Op, VT, Expand);
        break;
      }
    }
  }
 
  setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand);
 
  // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
  // is expanded to avoid having two separate loops in case the index is a VGPR.
 
  // Most operations are naturally 32-bit vector operations. We only support
  // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
  for (MVT Vec64 : {MVT::v2i64, MVT::v2f64}) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32);
  }
 
  for (MVT Vec64 : {MVT::v3i64, MVT::v3f64}) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v6i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v6i32);
  }
 
  for (MVT Vec64 : {MVT::v4i64, MVT::v4f64}) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v8i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v8i32);
  }
 
  for (MVT Vec64 : {MVT::v8i64, MVT::v8f64}) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v16i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v16i32);
  }
 
  for (MVT Vec64 : {MVT::v16i64, MVT::v16f64}) {
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v32i32);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v32i32);
  }
 
  setOperationAction(ISD::VECTOR_SHUFFLE,
                     {MVT::v4i32, MVT::v4f32, MVT::v8i32, MVT::v8f32,
                      MVT::v16i32, MVT::v16f32, MVT::v32i32, MVT::v32f32},
                     Custom);
 
  if (Subtarget->hasPkMovB32()) {
    // TODO: 16-bit element vectors should be legal with even aligned elements.
    // TODO: Can be legal with wider source types than the result with
    // subregister extracts.
    setOperationAction(ISD::VECTOR_SHUFFLE, {MVT::v2i32, MVT::v2f32}, Legal);
  }
 
  setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, MVT::v2i32, Legal);
  // Prevent SELECT v2i32 from being implemented with the above bitwise ops and
  // instead lower to cndmask in SITargetLowering::LowerSELECT().
  setOperationAction(ISD::SELECT, MVT::v2i32, Custom);
  // Enable MatchRotate to produce ISD::ROTR, which is later transformed to
  // alignbit.
  setOperationAction(ISD::ROTR, MVT::v2i32, Custom);
 
  setOperationAction(ISD::BUILD_VECTOR, {MVT::v4f16, MVT::v4i16, MVT::v4bf16},
                     Custom);
 
  // Avoid stack access for these.
  // TODO: Generalize to more vector types.
  setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
                     {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v2i8, MVT::v4i8,
                      MVT::v8i8, MVT::v4i16, MVT::v4f16, MVT::v4bf16},
                     Custom);
 
  // Deal with vec3 vector operations when widened to vec4.
  setOperationAction(ISD::INSERT_SUBVECTOR,
                     {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32}, Custom);
 
  // Deal with vec5/6/7 vector operations when widened to vec8.
  setOperationAction(ISD::INSERT_SUBVECTOR,
                     {MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
                      MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
                      MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
                      MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
                     Custom);
 
  // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
  // and output demarshalling
  setOperationAction(ISD::ATOMIC_CMP_SWAP, {MVT::i32, MVT::i64}, Custom);
 
  // We can't return success/failure, only the old value,
  // let LLVM add the comparison
  setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, {MVT::i32, MVT::i64},
                     Expand);
 
  setOperationAction(ISD::ADDRSPACECAST, {MVT::i32, MVT::i64}, Custom);
 
  setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal);
 
  // FIXME: This should be narrowed to i32, but that only happens if i64 is
  // illegal.
  // FIXME: Should lower sub-i32 bswaps to bit-ops without v_perm_b32.
  setOperationAction(ISD::BSWAP, {MVT::i64, MVT::i32}, Legal);
 
  // On SI this is s_memtime and s_memrealtime on VI.
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
 
  if (Subtarget->hasSMemRealTime() ||
      Subtarget->getGeneration() >= AMDGPUSubtarget::GFX11)
    setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64, Legal);
  setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction({ISD::FPOW, ISD::FPOWI}, MVT::f16, Promote);
    setOperationAction({ISD::FLOG, ISD::FEXP, ISD::FLOG10}, MVT::f16, Custom);
    setOperationAction(ISD::IS_FPCLASS, {MVT::f16, MVT::f32, MVT::f64}, Legal);
    setOperationAction({ISD::FLOG2, ISD::FEXP2}, MVT::f16, Legal);
    setOperationAction(ISD::FCANONICALIZE, MVT::f16, Legal);
  } else {
    setOperationAction(ISD::FSQRT, MVT::f16, Custom);
  }
 
  if (Subtarget->hasMadMacF32Insts())
    setOperationAction(ISD::FMAD, MVT::f32, Legal);
 
  setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, MVT::i32, Custom);
  setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, MVT::i32, Custom);
  setOperationAction(ISD::CTLS, MVT::i32, Custom);
 
  // We only really have 32-bit BFE instructions (and 16-bit on VI).
  //
  // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
  // effort to match them now. We want this to be false for i64 cases when the
  // extraction isn't restricted to the upper or lower half. Ideally we would
  // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
  // span the midpoint are probably relatively rare, so don't worry about them
  // for now.
  setHasExtractBitsInsn(true);
 
  // Clamp modifier on add/sub
  if (Subtarget->hasIntClamp())
    setOperationAction({ISD::UADDSAT, ISD::USUBSAT}, MVT::i32, Legal);
 
  if (Subtarget->hasAddNoCarryInsts())
    setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32},
                       Legal);
 
  setOperationAction(
      {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
      {MVT::f32, MVT::f64}, Custom);
 
  // These are really only legal for ieee_mode functions. We should be avoiding
  // them for functions that don't have ieee_mode enabled, so just say they are
  // legal.
  setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE},
                     {MVT::f32, MVT::f64}, Legal);
 
  if (Subtarget->haveRoundOpsF64())
    setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FROUNDEVEN}, MVT::f64,
                       Legal);
  else
    setOperationAction({ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
                       MVT::f64, Custom);
 
  setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
  setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, {MVT::f32, MVT::f64},
                     Legal);
  setOperationAction(ISD::FFREXP, {MVT::f32, MVT::f64}, Custom);
 
  setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FDIV}, MVT::f32, Custom);
  setOperationAction(ISD::FDIV, MVT::f64, Custom);
 
  setOperationAction(ISD::BF16_TO_FP, {MVT::i16, MVT::f32, MVT::f64}, Expand);
  setOperationAction(ISD::FP_TO_BF16, {MVT::i16, MVT::f32, MVT::f64}, Expand);
 
  setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, MVT::i32,
                     Custom);
  setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, MVT::i16,
                     Custom);
  setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, MVT::i1,
                     Custom);
 
  // Custom lower these because we can't specify a rule based on an illegal
  // source bf16.
  setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f32, Custom);
  setOperationAction({ISD::FP_EXTEND, ISD::STRICT_FP_EXTEND}, MVT::f64, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction({ISD::Constant, ISD::SMIN, ISD::SMAX, ISD::UMIN,
                        ISD::UMAX, ISD::UADDSAT, ISD::USUBSAT},
                       MVT::i16, Legal);
 
    AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32);
 
    setOperationAction({ISD::ROTR, ISD::ROTL, ISD::SELECT_CC, ISD::BR_CC},
                       MVT::i16, Expand);
 
    setOperationAction({ISD::SIGN_EXTEND, ISD::SDIV, ISD::UDIV, ISD::SREM,
                        ISD::UREM, ISD::BITREVERSE, ISD::CTTZ,
                        ISD::CTTZ_ZERO_POISON, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
                        ISD::CTPOP},
                       MVT::i16, Promote);
 
    setOperationAction(ISD::LOAD, MVT::i16, Custom);
 
    setTruncStoreAction(MVT::i64, MVT::i16, Expand);
 
    setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote);
    AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32);
    setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote);
    AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32);
 
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom);
    setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i32, Custom);
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom);
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i1, Custom);
 
    setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i32, Custom);
 
    // F16 - Constant Actions.
    setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
    setOperationAction(ISD::ConstantFP, MVT::bf16, Legal);
 
    // F16 - Load/Store Actions.
    setOperationAction(ISD::LOAD, MVT::f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16);
    setOperationAction(ISD::STORE, MVT::f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16);
 
    // BF16 - Load/Store Actions.
    setOperationAction(ISD::LOAD, MVT::bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::bf16, MVT::i16);
    setOperationAction(ISD::STORE, MVT::bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::bf16, MVT::i16);
 
    // F16 - VOP1 Actions.
    setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND, ISD::FCOS,
                        ISD::FSIN, ISD::FROUND},
                       MVT::f16, Custom);
 
    // BF16 - VOP1 Actions.
    if (Subtarget->hasBF16TransInsts())
      setOperationAction({ISD::FCOS, ISD::FSIN, ISD::FDIV}, MVT::bf16, Custom);
 
    // F16 - VOP2 Actions.
    setOperationAction({ISD::BR_CC, ISD::SELECT_CC}, {MVT::f16, MVT::bf16},
                       Expand);
    setOperationAction({ISD::FLDEXP, ISD::STRICT_FLDEXP}, MVT::f16, Custom);
    setOperationAction(ISD::FFREXP, MVT::f16, Custom);
    setOperationAction(ISD::FDIV, MVT::f16, Custom);
 
    // F16 - VOP3 Actions.
    setOperationAction(ISD::FMA, MVT::f16, Legal);
    if (STI.hasMadF16())
      setOperationAction(ISD::FMAD, MVT::f16, Legal);
 
    for (MVT VT :
         {MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::v4i16, MVT::v4f16,
          MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16,
          MVT::v16f16, MVT::v16bf16, MVT::v32i16, MVT::v32f16}) {
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op) {
        switch (Op) {
        case ISD::LOAD:
        case ISD::STORE:
        case ISD::BUILD_VECTOR:
        case ISD::BITCAST:
        case ISD::UNDEF:
        case ISD::POISON:
        case ISD::EXTRACT_VECTOR_ELT:
        case ISD::INSERT_VECTOR_ELT:
        case ISD::INSERT_SUBVECTOR:
        case ISD::SCALAR_TO_VECTOR:
        case ISD::IS_FPCLASS:
          break;
        case ISD::EXTRACT_SUBVECTOR:
        case ISD::CONCAT_VECTORS:
        case ISD::FSIN:
        case ISD::FCOS:
          setOperationAction(Op, VT, Custom);
          break;
        default:
          setOperationAction(Op, VT, Expand);
          break;
        }
      }
    }
 
    // v_perm_b32 can handle either of these.
    setOperationAction(ISD::BSWAP, {MVT::i16, MVT::v2i16}, Legal);
    setOperationAction(ISD::BSWAP, MVT::v4i16, Custom);
 
    // Legalize vector types for sat conversions to select v_cvt_pk_[iu]16_f32.
    if (Subtarget->hasVCvtPkIU16F32())
      setOperationAction(
          {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT},
          {MVT::v2i16, MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16},
          Custom);
 
    // XXX - Do these do anything? Vector constants turn into build_vector.
    setOperationAction(ISD::Constant, {MVT::v2i16, MVT::v2f16}, Legal);
 
    setOperationAction({ISD::UNDEF, ISD::POISON},
                       {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Legal);
 
    setOperationAction(ISD::STORE, MVT::v2i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::STORE, MVT::v2f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::LOAD, MVT::v2i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::ATOMIC_LOAD, MVT::v2i16, Promote);
    AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::ATOMIC_LOAD, MVT::v2f16, Promote);
    AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::ATOMIC_STORE, MVT::v2i16, Promote);
    AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::ATOMIC_STORE, MVT::v2f16, Promote);
    AddPromotedToType(ISD::ATOMIC_STORE, MVT::v2f16, MVT::i32);
 
    setOperationAction(ISD::AND, MVT::v2i16, Promote);
    AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::OR, MVT::v2i16, Promote);
    AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::XOR, MVT::v2i16, Promote);
    AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32);
 
    setOperationAction(ISD::LOAD, MVT::v4i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::LOAD, MVT::v4f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32);
    setOperationAction(ISD::LOAD, MVT::v4bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v4bf16, MVT::v2i32);
 
    setOperationAction(ISD::ATOMIC_LOAD, MVT::v4i16, Promote);
    AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4i16, MVT::i64);
    setOperationAction(ISD::ATOMIC_LOAD, MVT::v4f16, Promote);
    AddPromotedToType(ISD::ATOMIC_LOAD, MVT::v4f16, MVT::i64);
 
    setOperationAction(ISD::ATOMIC_STORE, MVT::v4i16, Promote);
    AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4i16, MVT::i64);
    setOperationAction(ISD::ATOMIC_STORE, MVT::v4f16, Promote);
    AddPromotedToType(ISD::ATOMIC_STORE, MVT::v4f16, MVT::i64);
 
    setOperationAction(ISD::STORE, MVT::v4i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4bf16, MVT::v2i32);
 
    setOperationAction(ISD::LOAD, MVT::v8i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8i16, MVT::v4i32);
    setOperationAction(ISD::LOAD, MVT::v8f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8f16, MVT::v4i32);
    setOperationAction(ISD::LOAD, MVT::v8bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v8bf16, MVT::v4i32);
 
    setOperationAction(ISD::STORE, MVT::v4i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
    setOperationAction(ISD::STORE, MVT::v4f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
 
    setOperationAction(ISD::STORE, MVT::v8i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8i16, MVT::v4i32);
    setOperationAction(ISD::STORE, MVT::v8f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8f16, MVT::v4i32);
    setOperationAction(ISD::STORE, MVT::v8bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v8bf16, MVT::v4i32);
 
    setOperationAction(ISD::LOAD, MVT::v16i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16i16, MVT::v8i32);
    setOperationAction(ISD::LOAD, MVT::v16f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16f16, MVT::v8i32);
    setOperationAction(ISD::LOAD, MVT::v16bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v16bf16, MVT::v8i32);
 
    setOperationAction(ISD::STORE, MVT::v16i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16i16, MVT::v8i32);
    setOperationAction(ISD::STORE, MVT::v16f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16f16, MVT::v8i32);
    setOperationAction(ISD::STORE, MVT::v16bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v16bf16, MVT::v8i32);
 
    setOperationAction(ISD::LOAD, MVT::v32i16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v32i16, MVT::v16i32);
    setOperationAction(ISD::LOAD, MVT::v32f16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v32f16, MVT::v16i32);
    setOperationAction(ISD::LOAD, MVT::v32bf16, Promote);
    AddPromotedToType(ISD::LOAD, MVT::v32bf16, MVT::v16i32);
 
    setOperationAction(ISD::STORE, MVT::v32i16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v32i16, MVT::v16i32);
    setOperationAction(ISD::STORE, MVT::v32f16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v32f16, MVT::v16i32);
    setOperationAction(ISD::STORE, MVT::v32bf16, Promote);
    AddPromotedToType(ISD::STORE, MVT::v32bf16, MVT::v16i32);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v2i32, Expand);
    setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v4i32, Expand);
 
    setOperationAction({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND},
                       MVT::v8i32, Expand);
 
    setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16, MVT::v2bf16},
                       Subtarget->hasVOP3PInsts() ? Legal : Custom);
 
    setOperationAction(ISD::FNEG, {MVT::v2f16, MVT::v2bf16}, Legal);
    // This isn't really legal, but this avoids the legalizer unrolling it (and
    // allows matching fneg (fabs x) patterns)
    setOperationAction(ISD::FABS, {MVT::v2f16, MVT::v2bf16}, Legal);
 
    // Can do this in one BFI plus a constant materialize.
    setOperationAction(ISD::FCOPYSIGN,
                       {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16,
                        MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16,
                        MVT::v32f16, MVT::v32bf16},
                       Custom);
 
    setOperationAction(
        {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
        MVT::f16, Custom);
    setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal);
 
    setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINIMUMNUM,
                        ISD::FMAXIMUMNUM},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
                       Custom);
 
    setOperationAction({ISD::FMINNUM, ISD::FMAXNUM},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
                       Expand);
 
    for (MVT Vec16 :
         {MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::v16i16, MVT::v16f16,
          MVT::v16bf16, MVT::v32i16, MVT::v32f16, MVT::v32bf16}) {
      setOperationAction(
          {ISD::BUILD_VECTOR, ISD::EXTRACT_VECTOR_ELT, ISD::SCALAR_TO_VECTOR},
          Vec16, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, Vec16, Expand);
    }
  }
 
  if (Subtarget->hasVOP3PInsts()) {
    setOperationAction({ISD::ADD, ISD::SUB, ISD::MUL, ISD::SHL, ISD::SRL,
                        ISD::SRA, ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX,
                        ISD::UADDSAT, ISD::USUBSAT, ISD::SADDSAT, ISD::SSUBSAT},
                       MVT::v2i16, Legal);
 
    setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
                        ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
                        ISD::FCANONICALIZE},
                       MVT::v2f16, Legal);
 
    setOperationAction(ISD::EXTRACT_VECTOR_ELT,
                       {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Custom);
 
    setOperationAction(ISD::VECTOR_SHUFFLE,
                       {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16,
                        MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16,
                        MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16},
                       Custom);
 
    for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16})
      // Split vector operations.
      setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL, ISD::ADD, ISD::SUB,
                          ISD::MUL, ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN,
                          ISD::UMAX, ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT,
                          ISD::SSUBSAT},
                         VT, Custom);
 
    for (MVT VT : {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16})
      // Split vector operations.
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FABS,
                          ISD::FCANONICALIZE},
                         VT, Custom);
 
    setOperationAction(
        {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
        {MVT::v2f16, MVT::v4f16}, Custom);
 
    setOperationAction(ISD::FEXP, MVT::v2f16, Custom);
    setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16, MVT::v4bf16},
                       Custom);
 
    if (Subtarget->hasBF16PackedInsts()) {
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMAXNUM, ISD::FMINNUM,
                          ISD::FMA, ISD::FNEG, ISD::FABS, ISD::FCANONICALIZE},
                         MVT::v2bf16, Legal);
 
      for (MVT VT : {MVT::v4bf16, MVT::v8bf16, MVT::v16bf16, MVT::v32bf16})
        // Split vector operations.
        setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE,
                            ISD::FNEG, ISD::FABS},
                           VT, Custom);
    }
 
    if (Subtarget->hasAnyPackedFP32Ops()) {
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
                         MVT::v2f32, Legal);
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG},
                         {MVT::v4f32, MVT::v8f32, MVT::v16f32, MVT::v32f32},
                         Custom);
    }
    if (Subtarget->hasAnyPackedFP64Ops()) {
      setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG,
                          ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
                          ISD::FCANONICALIZE, ISD::BUILD_VECTOR},
                         MVT::v2f64, Legal);
      setOperationAction(
          {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM},
          MVT::v2f64, Custom);
      setOperationAction(
          {ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FNEG, ISD::FMINNUM_IEEE,
           ISD::FMAXNUM_IEEE, ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
           ISD::FMAXIMUMNUM, ISD::FCANONICALIZE},
          {MVT::v4f64, MVT::v8f64, MVT::v16f64, MVT::v32f64}, Custom);
    }
 
    if (Subtarget->hasAnyPackedU64Ops()) {
      setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::BUILD_VECTOR},
                         MVT::v2i64, Legal);
      setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL},
                         {MVT::v4i64, MVT::v8i64, MVT::v16i64, MVT::v32i64},
                         Custom);
    }
  }
 
  setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v4f16, Custom);
 
  if (Subtarget->has16BitInsts()) {
    setOperationAction(ISD::SELECT, MVT::v2i16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
    setOperationAction(ISD::SELECT, MVT::v2f16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
    setOperationAction(ISD::SELECT, MVT::v2bf16, Promote);
    AddPromotedToType(ISD::SELECT, MVT::v2bf16, MVT::i32);
  } else {
    // Legalization hack.
    setOperationAction(ISD::SELECT, {MVT::v2i16, MVT::v2f16}, Custom);
 
    setOperationAction({ISD::FNEG, ISD::FABS}, MVT::v2f16, Custom);
  }
 
  setOperationAction(ISD::SELECT,
                     {MVT::v4i16, MVT::v4f16, MVT::v4bf16, MVT::v2i8, MVT::v4i8,
                      MVT::v8i8, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
                      MVT::v16i16, MVT::v16f16, MVT::v16bf16, MVT::v32i16,
                      MVT::v32f16, MVT::v32bf16},
                     Custom);
 
  setOperationAction({ISD::SMULO, ISD::UMULO}, MVT::i64, Custom);
 
  if (Subtarget->hasVMulU64Inst())
    setOperationAction(ISD::MUL, MVT::i64, Legal);
  else if (Subtarget->hasScalarSMulU64())
    setOperationAction(ISD::MUL, MVT::i64, Custom);
 
  if (Subtarget->hasMad64_32())
    setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom);
 
  if (Subtarget->hasSafeSmemPrefetch() || Subtarget->hasVmemPrefInsts())
    setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
  if (Subtarget->hasIEEEMinimumMaximumInsts()) {
    setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM},
                       {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal);
  } else {
    // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum
    if (Subtarget->hasMinimum3Maximum3F32())
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal);
 
    if (Subtarget->hasMinimum3Maximum3PKF16()) {
      setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::v2f16, Legal);
 
      // If only the vector form is available, we need to widen to a vector.
      if (!Subtarget->hasMinimum3Maximum3F16())
        setOperationPromotedToType({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16,
                                   MVT::v2f16);
    }
  }
 
  if (Subtarget->hasVOP3PInsts()) {
    // We want to break these into v2f16 pieces, not scalarize.
    setOperationAction({ISD::FMINIMUM, ISD::FMAXIMUM},
                       {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16},
                       Custom);
  }
 
  if (Subtarget->hasMinMaxI64Insts())
    setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, MVT::i64,
                       Legal);
 
  setOperationAction(ISD::INTRINSIC_WO_CHAIN,
                     {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
                      MVT::bf16, MVT::v2i16, MVT::v2f16, MVT::v2bf16, MVT::i128,
                      MVT::i8},
                     Custom);
 
  setOperationAction(ISD::INTRINSIC_W_CHAIN,
                     {MVT::v2f16, MVT::v2i16, MVT::v2bf16, MVT::v3f16,
                      MVT::v3i16, MVT::v4f16, MVT::v4i16, MVT::v4bf16,
                      MVT::v8i16, MVT::v8f16, MVT::v8bf16, MVT::Other, MVT::f16,
                      MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
                     Custom);
 
  setOperationAction(ISD::INTRINSIC_VOID,
                     {MVT::Other, MVT::v2i16, MVT::v2f16, MVT::v2bf16,
                      MVT::v3i16, MVT::v3f16, MVT::v4f16, MVT::v4i16,
                      MVT::v4bf16, MVT::v8i16, MVT::v8f16, MVT::v8bf16,
                      MVT::f16, MVT::i16, MVT::bf16, MVT::i8, MVT::i128},
                     Custom);
 
  setOperationAction(ISD::STACKSAVE, MVT::Other, Custom);
  setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
  setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
  setOperationAction(ISD::GET_FPENV, MVT::i64, Custom);
  setOperationAction(ISD::SET_FPENV, MVT::i64, Custom);
 
  // TODO: Could move this to custom lowering, could benefit from combines on
  // extract of relevant bits.
  setOperationAction(ISD::GET_FPMODE, MVT::i32, Legal);
 
  setOperationAction(ISD::MUL, MVT::i1, Promote);
 
  if (Subtarget->hasBF16ConversionInsts()) {
    setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND},
                       {MVT::bf16, MVT::v2bf16}, Custom);
    setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal);
  }
 
  if (Subtarget->hasBF16TransInsts()) {
    setOperationAction({ISD::FEXP2, ISD::FLOG2, ISD::FSQRT}, MVT::bf16, Legal);
  }
 
  const bool HasE5M3ConversionInsts =
      Subtarget->hasFP8ConversionInsts() && Subtarget->hasFP8E5M3Insts();
  if (Subtarget->hasOCPFP8ConversionInsts() || HasE5M3ConversionInsts) {
    setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f32, MVT::v2f32},
                       Custom);
    setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, MVT::v2i8, Custom);
 
    // i8 result promotes to i16, wider vectors split down to v2i8, and v2i8 is
    // handled in ReplaceNodeResults before the legalizer splits it per lane.
    setOperationAction(ISD::CONVERT_TO_ARBITRARY_FP, {MVT::i16, MVT::v2i8},
                       Custom);
  }
 
  if (Subtarget->hasFP8F16ConversionInsts()) {
    setOperationAction(ISD::CONVERT_FROM_ARBITRARY_FP, {MVT::f16, MVT::v2f16},
                       Custom);
  }
 
  if (Subtarget->hasCvtPkF16F32Inst()) {
    setOperationAction(ISD::FP_ROUND,
                       {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16},
                       Custom);
  }
 
  setTargetDAGCombine({ISD::ADD,
                       ISD::PTRADD,
                       ISD::SUB,
                       ISD::MUL,
                       ISD::FADD,
                       ISD::FSUB,
                       ISD::FDIV,
                       ISD::FMUL,
                       ISD::FMINNUM,
                       ISD::FMAXNUM,
                       ISD::FMINNUM_IEEE,
                       ISD::FMAXNUM_IEEE,
                       ISD::FMINIMUM,
                       ISD::FMAXIMUM,
                       ISD::FMINIMUMNUM,
                       ISD::FMAXIMUMNUM,
                       ISD::FMA,
                       ISD::ABS,
                       ISD::SMIN,
                       ISD::SMAX,
                       ISD::UMIN,
                       ISD::UMAX,
                       ISD::SETCC,
                       ISD::SELECT,
                       ISD::SMIN,
                       ISD::SMAX,
                       ISD::UMIN,
                       ISD::UMAX,
                       ISD::USUBSAT,
                       ISD::AND,
                       ISD::OR,
                       ISD::XOR,
                       ISD::SHL,
                       ISD::SRL,
                       ISD::SRA,
                       ISD::FSHR,
                       ISD::SINT_TO_FP,
                       ISD::UINT_TO_FP,
                       ISD::FCANONICALIZE,
                       ISD::SCALAR_TO_VECTOR,
                       ISD::ZERO_EXTEND,
                       ISD::SIGN_EXTEND_INREG,
                       ISD::ANY_EXTEND,
                       ISD::EXTRACT_VECTOR_ELT,
                       ISD::INSERT_VECTOR_ELT,
                       ISD::FCOPYSIGN});
 
  if (Subtarget->has16BitInsts() && !Subtarget->hasMed3_16())
    setTargetDAGCombine(ISD::FP_ROUND);
 
  // All memory operations. Some folding on the pointer operand is done to help
  // matching the constant offsets in the addressing modes.
  setTargetDAGCombine({ISD::LOAD,
                       ISD::STORE,
                       ISD::ATOMIC_LOAD,
                       ISD::ATOMIC_STORE,
                       ISD::ATOMIC_CMP_SWAP,
                       ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
                       ISD::ATOMIC_SWAP,
                       ISD::ATOMIC_LOAD_ADD,
                       ISD::ATOMIC_LOAD_SUB,
                       ISD::ATOMIC_LOAD_AND,
                       ISD::ATOMIC_LOAD_OR,
                       ISD::ATOMIC_LOAD_XOR,
                       ISD::ATOMIC_LOAD_NAND,
                       ISD::ATOMIC_LOAD_MIN,
                       ISD::ATOMIC_LOAD_MAX,
                       ISD::ATOMIC_LOAD_UMIN,
                       ISD::ATOMIC_LOAD_UMAX,
                       ISD::ATOMIC_LOAD_FADD,
                       ISD::ATOMIC_LOAD_FMIN,
                       ISD::ATOMIC_LOAD_FMAX,
                       ISD::ATOMIC_LOAD_UINC_WRAP,
                       ISD::ATOMIC_LOAD_UDEC_WRAP,
                       ISD::ATOMIC_LOAD_USUB_COND,
                       ISD::ATOMIC_LOAD_USUB_SAT,
                       ISD::INTRINSIC_VOID,
                       ISD::INTRINSIC_W_CHAIN});
 
  // FIXME: In other contexts we pretend this is a per-function property.
  setStackPointerRegisterToSaveRestore(AMDGPU::SGPR32);
 
  setSchedulingPreference(Sched::RegPressure);
}
 
const GCNSubtarget *SITargetLowering::getSubtarget() const { return Subtarget; }
 
ArrayRef<MCPhysReg> SITargetLowering::getRoundingControlRegisters() const {
  static const MCPhysReg RCRegs[] = {AMDGPU::MODE};
  return RCRegs;
}
 
//===----------------------------------------------------------------------===//
// TargetLowering queries
//===----------------------------------------------------------------------===//
 
// v_mad_mix* support a conversion from f16 to f32.
//
// There is only one special case when denormals are enabled we don't currently,
// where this is OK to use.
bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
                                       EVT DestVT, EVT SrcVT) const {
  return DestVT.getScalarType() == MVT::f32 &&
         ((((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) ||
            (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) &&
           SrcVT.getScalarType() == MVT::f16) ||
          (Opcode == ISD::FMA && Subtarget->hasFmaMixBF16Insts() &&
           SrcVT.getScalarType() == MVT::bf16)) &&
         // TODO: This probably only requires no input flushing?
         denormalModeIsFlushAllF32(DAG.getMachineFunction());
}
 
bool SITargetLowering::isFPExtFoldable(const MachineInstr &MI, unsigned Opcode,
                                       LLT DestTy, LLT SrcTy) const {
  return ((Opcode == TargetOpcode::G_FMAD && Subtarget->hasMadMixInsts()) ||
          (Opcode == TargetOpcode::G_FMA && Subtarget->hasFmaMixInsts())) &&
         DestTy.getScalarSizeInBits() == 32 &&
         SrcTy.getScalarSizeInBits() == 16 &&
         // TODO: This probably only requires no input flushing?
         denormalModeIsFlushAllF32(*MI.getMF());
}
 
bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
  // SI has some legal vector types, but no legal vector operations. Say no
  // shuffles are legal in order to prefer scalarizing some vector operations.
  return false;
}
 
MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                    CallingConv::ID CC,
                                                    EVT VT) const {
  if (CC == CallingConv::AMDGPU_KERNEL)
    return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
 
  if (VT.isVector()) {
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
    if (Size == 16) {
      return Subtarget->has16BitInsts()
                 ? MVT::getVectorVT(ScalarVT.getSimpleVT(), 2)
                 : MVT::i32;
    }
 
    if (Size < 16)
      return Subtarget->has16BitInsts() ? MVT::i16 : MVT::i32;
    return Size == 32 ? ScalarVT.getSimpleVT() : MVT::i32;
  }
 
  if (!Subtarget->has16BitInsts() && VT.getSizeInBits() == 16)
    return MVT::i32;
 
  if (VT.getSizeInBits() > 32)
    return MVT::i32;
 
  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
 
unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                         CallingConv::ID CC,
                                                         EVT VT) const {
  if (CC == CallingConv::AMDGPU_KERNEL)
    return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
 
  if (VT.isVector()) {
    unsigned NumElts = VT.getVectorNumElements();
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
 
    // FIXME: Should probably promote 8-bit vectors to i16.
    if (Size == 16)
      return (NumElts + 1) / 2;
 
    if (Size <= 32)
      return NumElts;
 
    if (Size > 32)
      return NumElts * ((Size + 31) / 32);
  } else if (VT.getSizeInBits() > 32)
    return (VT.getSizeInBits() + 31) / 32;
 
  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
}
 
unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
    LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
    unsigned &NumIntermediates, MVT &RegisterVT) const {
  if (CC != CallingConv::AMDGPU_KERNEL && VT.isVector()) {
    unsigned NumElts = VT.getVectorNumElements();
    EVT ScalarVT = VT.getScalarType();
    unsigned Size = ScalarVT.getSizeInBits();
    // FIXME: We should fix the ABI to be the same on targets without 16-bit
    // support, but unless we can properly handle 3-vectors, it will be still be
    // inconsistent.
    if (Size == 16) {
      MVT SimpleIntermediateVT =
          MVT::getVectorVT(ScalarVT.getSimpleVT(), ElementCount::getFixed(2));
      IntermediateVT = SimpleIntermediateVT;
      RegisterVT = Subtarget->has16BitInsts() ? SimpleIntermediateVT : MVT::i32;
      NumIntermediates = (NumElts + 1) / 2;
      return (NumElts + 1) / 2;
    }
 
    if (Size == 32) {
      RegisterVT = ScalarVT.getSimpleVT();
      IntermediateVT = RegisterVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
    if (Size < 16 && Subtarget->has16BitInsts()) {
      // FIXME: Should probably form v2i16 pieces
      RegisterVT = MVT::i16;
      IntermediateVT = ScalarVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
    if (Size != 16 && Size <= 32) {
      RegisterVT = MVT::i32;
      IntermediateVT = ScalarVT;
      NumIntermediates = NumElts;
      return NumIntermediates;
    }
 
    if (Size > 32) {
      RegisterVT = MVT::i32;
      IntermediateVT = RegisterVT;
      NumIntermediates = NumElts * ((Size + 31) / 32);
      return NumIntermediates;
    }
  }
 
  return TargetLowering::getVectorTypeBreakdownForCallingConv(
      Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
}
 
static EVT memVTFromLoadIntrData(const SITargetLowering &TLI,
                                 const DataLayout &DL, Type *Ty,
                                 unsigned MaxNumLanes) {
  assert(MaxNumLanes != 0);
 
  LLVMContext &Ctx = Ty->getContext();
  if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
    unsigned NumElts = std::min(MaxNumLanes, VT->getNumElements());
    return EVT::getVectorVT(Ctx, TLI.getValueType(DL, VT->getElementType()),
                            NumElts);
  }
 
  return TLI.getValueType(DL, Ty);
}
 
// Peek through TFE struct returns to only use the data size.
static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI,
                                   const DataLayout &DL, Type *Ty,
                                   unsigned MaxNumLanes) {
  auto *ST = dyn_cast<StructType>(Ty);
  if (!ST)
    return memVTFromLoadIntrData(TLI, DL, Ty, MaxNumLanes);
 
  // TFE intrinsics return an aggregate type.
  assert(ST->getNumContainedTypes() == 2 &&
         ST->getContainedType(1)->isIntegerTy(32));
  return memVTFromLoadIntrData(TLI, DL, ST->getContainedType(0), MaxNumLanes);
}
 
/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its
/// in-memory representation. This return value is a custom type because there
/// is no MVT::i160 and adding one breaks integer promotion logic. While this
/// could cause issues during codegen, these address space 7 pointers will be
/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer
/// in order to allow pre-codegen passes that query TargetTransformInfo, often
/// for cost modeling, to work. (This also sets us up decently for doing the
/// buffer lowering in GlobalISel if SelectionDAG ever goes away.)
MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const {
  if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160)
    return MVT::amdgpuBufferFatPointer;
  if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
      DL.getPointerSizeInBits(AS) == 192)
    return MVT::amdgpuBufferStridedPointer;
  return AMDGPUTargetLowering::getPointerTy(DL, AS);
}
/// Similarly, the in-memory representation of a p7 is {p8, i32}, aka
/// v8i32 when padding is added.
/// The in-memory representation of a p9 is {p8, i32, i32}, which is
/// also v8i32 with padding.
MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const {
  if ((AMDGPUAS::BUFFER_FAT_POINTER == AS &&
       DL.getPointerSizeInBits(AS) == 160) ||
      (AMDGPUAS::BUFFER_STRIDED_POINTER == AS &&
       DL.getPointerSizeInBits(AS) == 192))
    return MVT::v8i32;
  return AMDGPUTargetLowering::getPointerMemTy(DL, AS);
}
 
static unsigned getIntrMemWidth(unsigned IntrID) {
  switch (IntrID) {
  case Intrinsic::amdgcn_global_load_async_to_lds_b8:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
  case Intrinsic::amdgcn_global_store_async_from_lds_b8:
    return 8;
  case Intrinsic::amdgcn_global_load_async_to_lds_b32:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
  case Intrinsic::amdgcn_global_store_async_from_lds_b32:
  case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
  case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
  case Intrinsic::amdgcn_flat_load_monitor_b32:
  case Intrinsic::amdgcn_global_load_monitor_b32:
    return 32;
  case Intrinsic::amdgcn_global_load_async_to_lds_b64:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
  case Intrinsic::amdgcn_global_store_async_from_lds_b64:
  case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
  case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
  case Intrinsic::amdgcn_flat_load_monitor_b64:
  case Intrinsic::amdgcn_global_load_monitor_b64:
    return 64;
  case Intrinsic::amdgcn_global_load_async_to_lds_b128:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
  case Intrinsic::amdgcn_global_store_async_from_lds_b128:
  case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
  case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
  case Intrinsic::amdgcn_flat_load_monitor_b128:
  case Intrinsic::amdgcn_global_load_monitor_b128:
    return 128;
  default:
    llvm_unreachable("Unknown width");
  }
}
 
static AtomicOrdering parseAtomicOrderingCABIArg(const CallBase &CI,
                                                 unsigned ArgIdx) {
  Value *OrderingArg = CI.getArgOperand(ArgIdx);
  unsigned Ord = cast<ConstantInt>(OrderingArg)->getZExtValue();
  switch (AtomicOrderingCABI(Ord)) {
  case AtomicOrderingCABI::acquire:
    return AtomicOrdering::Acquire;
    break;
  case AtomicOrderingCABI::release:
    return AtomicOrdering::Release;
    break;
  case AtomicOrderingCABI::seq_cst:
    return AtomicOrdering::SequentiallyConsistent;
    break;
  default:
    return AtomicOrdering::Monotonic;
  }
}
 
static unsigned parseSyncscopeMDArg(const CallBase &CI, unsigned ArgIdx) {
  MDNode *ScopeMD = cast<MDNode>(
      cast<MetadataAsValue>(CI.getArgOperand(ArgIdx))->getMetadata());
  StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
  return CI.getContext().getOrInsertSyncScopeID(Scope);
}
 
void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
                                          const CallBase &CI,
                                          MachineFunction &MF,
                                          unsigned IntrID) const {
  MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
  if (CI.hasMetadata(LLVMContext::MD_invariant_load))
    Flags |= MachineMemOperand::MOInvariant;
  if (CI.hasMetadata(LLVMContext::MD_nontemporal))
    Flags |= MachineMemOperand::MONonTemporal;
  Flags |= getTargetMMOFlags(CI);
 
  if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
          AMDGPU::lookupRsrcIntrinsic(IntrID)) {
    AttributeSet Attr =
        Intrinsic::getFnAttributes(CI.getContext(), (Intrinsic::ID)IntrID);
    MemoryEffects ME = Attr.getMemoryEffects();
    if (ME.doesNotAccessMemory())
      return;
 
    bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
    if (!IsSPrefetch) {
      auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
      if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
        Flags |= MachineMemOperand::MOVolatile;
    }
 
    Flags |= MachineMemOperand::MODereferenceable;
 
    IntrinsicInfo Info;
    // TODO: Should images get their own address space?
    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
 
    const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = nullptr;
    if (RsrcIntr->IsImage) {
      const AMDGPU::ImageDimIntrinsicInfo *Intr =
          AMDGPU::getImageDimIntrinsicInfo(IntrID);
      BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
      Info.align.reset();
    }
 
    Value *RsrcArg = CI.getArgOperand(RsrcIntr->RsrcArg);
    if (auto *RsrcPtrTy = dyn_cast<PointerType>(RsrcArg->getType())) {
      if (RsrcPtrTy->getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
        // We conservatively set the memory operand of a buffer intrinsic to the
        // base resource pointer, so that we can access alias information about
        // those pointers. Cases like "this points at the same value
        // but with a different offset" are handled in
        // areMemAccessesTriviallyDisjoint.
        Info.ptrVal = RsrcArg;
    }
 
    if (ME.onlyReadsMemory()) {
      if (RsrcIntr->IsImage) {
        unsigned MaxNumLanes = 4;
 
        if (!BaseOpcode->Gather4) {
          // If this isn't a gather, we may have excess loaded elements in the
          // IR type. Check the dmask for the real number of elements loaded.
          unsigned DMask =
              cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue();
          MaxNumLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
        }
 
        Info.memVT = memVTFromLoadIntrReturn(*this, MF.getDataLayout(),
                                             CI.getType(), MaxNumLanes);
      } else {
        Info.memVT =
            memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
                                    std::numeric_limits<unsigned>::max());
      }
 
      // FIXME: What does alignment mean for an image?
      Info.opc = ISD::INTRINSIC_W_CHAIN;
      Info.flags = Flags | MachineMemOperand::MOLoad;
    } else if (ME.onlyWritesMemory()) {
      Info.opc = ISD::INTRINSIC_VOID;
 
      Type *DataTy = CI.getArgOperand(0)->getType();
      if (RsrcIntr->IsImage) {
        unsigned DMask = cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue();
        unsigned DMaskLanes = DMask == 0 ? 1 : llvm::popcount(DMask);
        Info.memVT = memVTFromLoadIntrData(*this, MF.getDataLayout(), DataTy,
                                           DMaskLanes);
      } else
        Info.memVT = getValueType(MF.getDataLayout(), DataTy);
 
      Info.flags = Flags | MachineMemOperand::MOStore;
    } else {
      // Atomic, NoReturn Sampler or prefetch
      Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
                                          : ISD::INTRINSIC_W_CHAIN;
 
      switch (IntrID) {
      default:
        Info.flags = Flags | MachineMemOperand::MOLoad;
        if (!IsSPrefetch)
          Info.flags |= MachineMemOperand::MOStore;
 
        if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
          // Fake memory access type for no return sampler intrinsics
          Info.memVT = MVT::i32;
        } else {
          // XXX - Should this be volatile without known ordering?
          Info.flags |= MachineMemOperand::MOVolatile;
          Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
        }
        break;
      case Intrinsic::amdgcn_raw_buffer_load_lds:
      case Intrinsic::amdgcn_raw_buffer_load_async_lds:
      case Intrinsic::amdgcn_raw_ptr_buffer_load_lds:
      case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds:
      case Intrinsic::amdgcn_struct_buffer_load_lds:
      case Intrinsic::amdgcn_struct_buffer_load_async_lds:
      case Intrinsic::amdgcn_struct_ptr_buffer_load_lds:
      case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: {
        unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
 
        // Entry 0: Load from buffer.
        // Don't set an offset, since the pointer value always represents the
        // base of the buffer.
        Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
        Info.flags = Flags | MachineMemOperand::MOLoad;
        Infos.push_back(Info);
 
        // Entry 1: Store to LDS.
        // Instruction offset is applied, and an additional per-lane offset
        // which we simulate using a larger memory type.
        Info.memVT = EVT::getIntegerVT(
            CI.getContext(), Width * 8 * Subtarget->getWavefrontSize());
        Info.ptrVal = CI.getArgOperand(1); // LDS destination pointer
        Info.offset = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 2))
                          ->getZExtValue();
        Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
        Info.flags = Flags | MachineMemOperand::MOStore;
        Infos.push_back(Info);
        return;
      }
      case Intrinsic::amdgcn_raw_atomic_buffer_load:
      case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
      case Intrinsic::amdgcn_struct_atomic_buffer_load:
      case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load: {
        Info.memVT =
            memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
                                    std::numeric_limits<unsigned>::max());
        Info.flags = Flags | MachineMemOperand::MOLoad;
        Infos.push_back(Info);
        return;
      }
      }
    }
    Infos.push_back(Info);
    return;
  }
 
  IntrinsicInfo Info;
  switch (IntrID) {
  case Intrinsic::amdgcn_ds_ordered_add:
  case Intrinsic::amdgcn_ds_ordered_swap: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
    if (!Vol->isZero())
      Info.flags |= MachineMemOperand::MOVolatile;
 
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_ds_add_gs_reg_rtn:
  case Intrinsic::amdgcn_ds_sub_gs_reg_rtn: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
    Info.ptrVal = nullptr;
    Info.fallbackAddressSpace = AMDGPUAS::STREAMOUT_REGISTER;
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_ds_append:
  case Intrinsic::amdgcn_ds_consume: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
    if (!Vol->isZero())
      Info.flags |= MachineMemOperand::MOVolatile;
 
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
  case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: {
    Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64)
                   ? ISD::INTRINSIC_W_CHAIN
                   : ISD::INTRINSIC_VOID;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.memVT = MVT::i64;
    Info.size = 8;
    Info.align.reset();
    Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    Info.order = AtomicOrdering::Monotonic;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
  case Intrinsic::amdgcn_image_bvh_intersect_ray:
  case Intrinsic::amdgcn_image_bvh8_intersect_ray: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT =
        MVT::getVT(IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray
                       ? CI.getType()
                       : cast<StructType>(CI.getType())
                             ->getElementType(0)); // XXX: what is correct VT?
 
    Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
    Info.align.reset();
    Info.flags = Flags | MachineMemOperand::MOLoad |
                 MachineMemOperand::MODereferenceable;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_global_atomic_fmin_num:
  case Intrinsic::amdgcn_global_atomic_fmax_num:
  case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
  case Intrinsic::amdgcn_flat_atomic_fmin_num:
  case Intrinsic::amdgcn_flat_atomic_fmax_num: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags =
        Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
        MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_cluster_load_b32:
  case Intrinsic::amdgcn_cluster_load_b64:
  case Intrinsic::amdgcn_cluster_load_b128:
  case Intrinsic::amdgcn_ds_load_tr6_b96:
  case Intrinsic::amdgcn_ds_load_tr4_b64:
  case Intrinsic::amdgcn_ds_load_tr8_b64:
  case Intrinsic::amdgcn_ds_load_tr16_b128:
  case Intrinsic::amdgcn_global_load_tr6_b96:
  case Intrinsic::amdgcn_global_load_tr4_b64:
  case Intrinsic::amdgcn_global_load_tr_b64:
  case Intrinsic::amdgcn_global_load_tr_b128:
  case Intrinsic::amdgcn_ds_read_tr4_b64:
  case Intrinsic::amdgcn_ds_read_tr6_b96:
  case Intrinsic::amdgcn_ds_read_tr8_b64:
  case Intrinsic::amdgcn_ds_read_tr16_b64: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::getVT(CI.getType());
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags = Flags | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_flat_load_monitor_b32:
  case Intrinsic::amdgcn_flat_load_monitor_b64:
  case Intrinsic::amdgcn_flat_load_monitor_b128:
  case Intrinsic::amdgcn_global_load_monitor_b32:
  case Intrinsic::amdgcn_global_load_monitor_b64:
  case Intrinsic::amdgcn_global_load_monitor_b128: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags = MachineMemOperand::MOLoad;
    Info.order = parseAtomicOrderingCABIArg(CI, 1);
    Info.ssid = parseSyncscopeMDArg(CI, 2);
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
  case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
  case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
    Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
    Info.ptrVal = CI.getOperand(0);
    Info.align.reset();
    Info.flags = (MachineMemOperand::MOLoad | MOCooperative);
    Info.order = parseAtomicOrderingCABIArg(CI, 1);
    Info.ssid = parseSyncscopeMDArg(CI, 2);
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
  case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
  case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
    Info.ptrVal = CI.getArgOperand(0);
    Info.align.reset();
    Info.flags = (MachineMemOperand::MOStore | MOCooperative);
    Info.order = parseAtomicOrderingCABIArg(CI, 2);
    Info.ssid = parseSyncscopeMDArg(CI, 3);
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_ds_gws_init:
  case Intrinsic::amdgcn_ds_gws_barrier:
  case Intrinsic::amdgcn_ds_gws_sema_v:
  case Intrinsic::amdgcn_ds_gws_sema_br:
  case Intrinsic::amdgcn_ds_gws_sema_p:
  case Intrinsic::amdgcn_ds_gws_sema_release_all: {
    Info.opc = ISD::INTRINSIC_VOID;
 
    const GCNTargetMachine &TM =
        static_cast<const GCNTargetMachine &>(getTargetMachine());
 
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    Info.ptrVal = MFI->getGWSPSV(TM);
 
    // This is an abstract access, but we need to specify a type and size.
    Info.memVT = MVT::i32;
    Info.size = 4;
    Info.align = Align(4);
 
    if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
      Info.flags = Flags | MachineMemOperand::MOLoad;
    else
      Info.flags = Flags | MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_global_load_async_to_lds_b8:
  case Intrinsic::amdgcn_global_load_async_to_lds_b32:
  case Intrinsic::amdgcn_global_load_async_to_lds_b64:
  case Intrinsic::amdgcn_global_load_async_to_lds_b128:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b128: {
    // Entry 0: Load from source (global/flat).
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
    Info.ptrVal = CI.getArgOperand(0); // Global pointer
    Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue();
    Info.flags = Flags | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
 
    // Entry 1: Store to LDS (same offset).
    Info.flags = Flags | MachineMemOperand::MOStore;
    Info.ptrVal = CI.getArgOperand(1); // LDS pointer
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_global_store_async_from_lds_b8:
  case Intrinsic::amdgcn_global_store_async_from_lds_b32:
  case Intrinsic::amdgcn_global_store_async_from_lds_b64:
  case Intrinsic::amdgcn_global_store_async_from_lds_b128: {
    // Entry 0: Load from LDS.
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
    Info.ptrVal = CI.getArgOperand(1); // LDS pointer
    Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue();
    Info.flags = Flags | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
 
    // Entry 1: Store to global (same offset).
    Info.flags = Flags | MachineMemOperand::MOStore;
    Info.ptrVal = CI.getArgOperand(0); // Global pointer
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_av_load_b128:
  case Intrinsic::amdgcn_av_store_b128: {
    bool IsStore = IntrID == Intrinsic::amdgcn_av_store_b128;
    Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
    Info.memVT = MVT::v4i32;
    Info.ptrVal = CI.getArgOperand(0);
    Info.align = Align(16);
    Info.flags |=
        IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
    // Pretend to be atomic so that SIMemoryLegalizer::expandStore sets cache
    // flags appropriately.
    Info.order = AtomicOrdering::Monotonic;
 
    LLVMContext &Ctx = CI.getContext();
    unsigned ScopeIdx = CI.arg_size() - 1;
    MDNode *ScopeMD = cast<MDNode>(
        cast<MetadataAsValue>(CI.getArgOperand(ScopeIdx))->getMetadata());
    StringRef Scope = cast<MDString>(ScopeMD->getOperand(0))->getString();
    Info.ssid = Ctx.getOrInsertSyncScopeID(Scope);
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_load_to_lds:
  case Intrinsic::amdgcn_load_async_to_lds:
  case Intrinsic::amdgcn_global_load_lds:
  case Intrinsic::amdgcn_global_load_async_lds: {
    unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
    auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
    bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
    if (IsVolatile)
      Flags |= MachineMemOperand::MOVolatile;
 
    // Entry 0: Load from source (global/flat).
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
    Info.ptrVal = CI.getArgOperand(0); // Source pointer
    Info.offset = cast<ConstantInt>(CI.getArgOperand(3))->getSExtValue();
    Info.flags = Flags | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
 
    // Entry 1: Store to LDS.
    // Same offset from the instruction, but an additional per-lane offset is
    // added. Represent that using a wider memory type.
    Info.memVT = EVT::getIntegerVT(CI.getContext(),
                                   Width * 8 * Subtarget->getWavefrontSize());
    Info.ptrVal = CI.getArgOperand(1); // LDS destination pointer
    Info.flags = Flags | MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_ds_bvh_stack_rtn:
  case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn:
  case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
  case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: {
    Info.opc = ISD::INTRINSIC_W_CHAIN;
 
    const GCNTargetMachine &TM =
        static_cast<const GCNTargetMachine &>(getTargetMachine());
 
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    Info.ptrVal = MFI->getGWSPSV(TM);
 
    // This is an abstract access, but we need to specify a type and size.
    Info.memVT = MVT::i32;
    Info.size = 4;
    Info.align = Align(4);
 
    Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
    Infos.push_back(Info);
    return;
  }
  case Intrinsic::amdgcn_s_prefetch_data:
  case Intrinsic::amdgcn_s_prefetch_inst:
  case Intrinsic::amdgcn_flat_prefetch:
  case Intrinsic::amdgcn_global_prefetch: {
    Info.opc = ISD::INTRINSIC_VOID;
    Info.memVT = EVT::getIntegerVT(CI.getContext(), 8);
    Info.ptrVal = CI.getArgOperand(0);
    Info.flags = Flags | MachineMemOperand::MOLoad;
    Infos.push_back(Info);
    return;
  }
  default:
    return;
  }
}
 
void SITargetLowering::CollectTargetIntrinsicOperands(
    const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
  switch (cast<IntrinsicInst>(I).getIntrinsicID()) {
  case Intrinsic::amdgcn_addrspacecast_nonnull: {
    // The DAG's ValueType loses the addrspaces.
    // Add them as 2 extra Constant operands "from" and "to".
    unsigned SrcAS = I.getOperand(0)->getType()->getPointerAddressSpace();
    unsigned DstAS = I.getType()->getPointerAddressSpace();
    Ops.push_back(DAG.getTargetConstant(SrcAS, SDLoc(), MVT::i32));
    Ops.push_back(DAG.getTargetConstant(DstAS, SDLoc(), MVT::i32));
    break;
  }
  default:
    break;
  }
}
 
bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II,
                                            SmallVectorImpl<Value *> &Ops,
                                            Type *&AccessTy) const {
  Value *Ptr = nullptr;
  switch (II->getIntrinsicID()) {
  case Intrinsic::amdgcn_cluster_load_b128:
  case Intrinsic::amdgcn_cluster_load_b64:
  case Intrinsic::amdgcn_cluster_load_b32:
  case Intrinsic::amdgcn_ds_append:
  case Intrinsic::amdgcn_ds_consume:
  case Intrinsic::amdgcn_ds_load_tr8_b64:
  case Intrinsic::amdgcn_ds_load_tr16_b128:
  case Intrinsic::amdgcn_ds_load_tr4_b64:
  case Intrinsic::amdgcn_ds_load_tr6_b96:
  case Intrinsic::amdgcn_ds_read_tr4_b64:
  case Intrinsic::amdgcn_ds_read_tr6_b96:
  case Intrinsic::amdgcn_ds_read_tr8_b64:
  case Intrinsic::amdgcn_ds_read_tr16_b64:
  case Intrinsic::amdgcn_ds_ordered_add:
  case Intrinsic::amdgcn_ds_ordered_swap:
  case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64:
  case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64:
  case Intrinsic::amdgcn_flat_atomic_fmax_num:
  case Intrinsic::amdgcn_flat_atomic_fmin_num:
  case Intrinsic::amdgcn_global_atomic_fmax_num:
  case Intrinsic::amdgcn_global_atomic_fmin_num:
  case Intrinsic::amdgcn_global_atomic_ordered_add_b64:
  case Intrinsic::amdgcn_global_load_tr_b64:
  case Intrinsic::amdgcn_global_load_tr_b128:
  case Intrinsic::amdgcn_global_load_tr4_b64:
  case Intrinsic::amdgcn_global_load_tr6_b96:
  case Intrinsic::amdgcn_global_store_async_from_lds_b8:
  case Intrinsic::amdgcn_global_store_async_from_lds_b32:
  case Intrinsic::amdgcn_global_store_async_from_lds_b64:
  case Intrinsic::amdgcn_global_store_async_from_lds_b128:
  case Intrinsic::amdgcn_av_load_b128:
  case Intrinsic::amdgcn_av_store_b128:
    Ptr = II->getArgOperand(0);
    break;
  case Intrinsic::amdgcn_load_to_lds:
  case Intrinsic::amdgcn_load_async_to_lds:
  case Intrinsic::amdgcn_global_load_lds:
  case Intrinsic::amdgcn_global_load_async_lds:
  case Intrinsic::amdgcn_global_load_async_to_lds_b8:
  case Intrinsic::amdgcn_global_load_async_to_lds_b32:
  case Intrinsic::amdgcn_global_load_async_to_lds_b64:
  case Intrinsic::amdgcn_global_load_async_to_lds_b128:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
  case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
    Ptr = II->getArgOperand(1);
    break;
  default:
    return false;
  }
  AccessTy = II->getType();
  Ops.push_back(Ptr);
  return true;
}
 
bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM,
                                                 unsigned AddrSpace) const {
  if (!Subtarget->hasFlatInstOffsets()) {
    // Flat instructions do not have offsets, and only have the register
    // address.
    return AM.BaseOffs == 0 && AM.Scale == 0;
  }
 
  using AMDGPU::FlatAddrSpace;
  FlatAddrSpace FlatVariant =
      AddrSpace == AMDGPUAS::GLOBAL_ADDRESS    ? FlatAddrSpace::FlatGlobal
      : AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ? FlatAddrSpace::FlatScratch
                                               : FlatAddrSpace::FLAT;
 
  return AM.Scale == 0 &&
         (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset(
                                  AM.BaseOffs, AddrSpace, FlatVariant));
}
 
bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
  if (Subtarget->hasFlatGlobalInsts())
    return isLegalFlatAddressingMode(AM, AMDGPUAS::GLOBAL_ADDRESS);
 
  if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) {
    // Assume the we will use FLAT for all global memory accesses
    // on VI.
    // FIXME: This assumption is currently wrong.  On VI we still use
    // MUBUF instructions for the r + i addressing mode.  As currently
    // implemented, the MUBUF instructions only work on buffer < 4GB.
    // It may be possible to support > 4GB buffers with MUBUF instructions,
    // by setting the stride value in the resource descriptor which would
    // increase the size limit to (stride * 4GB).  However, this is risky,
    // because it has never been validated.
    return isLegalFlatAddressingMode(AM, AMDGPUAS::FLAT_ADDRESS);
  }
 
  return isLegalMUBUFAddressingMode(AM);
}
 
bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
  // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
  // additionally can do r + r + i with addr64. 32-bit has more addressing
  // mode options. Depending on the resource constant, it can also do
  // (i64 r0) + (i32 r1) * (i14 i).
  //
  // Private arrays end up using a scratch buffer most of the time, so also
  // assume those use MUBUF instructions. Scratch loads / stores are currently
  // implemented as mubuf instructions with offen bit set, so slightly
  // different than the normal addr64.
  const SIInstrInfo *TII = Subtarget->getInstrInfo();
  if (!TII->isLegalMUBUFImmOffset(AM.BaseOffs))
    return false;
 
  // FIXME: Since we can split immediate into soffset and immediate offset,
  // would it make sense to allow any immediate?
 
  switch (AM.Scale) {
  case 0: // r + i or just i, depending on HasBaseReg.
    return true;
  case 1:
    return true; // We have r + r or r + i.
  case 2:
    if (AM.HasBaseReg) {
      // Reject 2 * r + r.
      return false;
    }
 
    // Allow 2 * r as r + r
    // Or  2 * r + i is allowed as r + r + i.
    return true;
  default: // Don't allow n * r
    return false;
  }
}
 
bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
                                             const AddrMode &AM, Type *Ty,
                                             unsigned AS,
                                             Instruction *I) const {
  // No global is ever allowed as a base.
  if (AM.BaseGV)
    return false;
 
  if (AS == AMDGPUAS::GLOBAL_ADDRESS)
    return isLegalGlobalAddressingMode(AM);
 
  if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
      AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
      AS == AMDGPUAS::BUFFER_FAT_POINTER || AS == AMDGPUAS::BUFFER_RESOURCE ||
      AS == AMDGPUAS::BUFFER_STRIDED_POINTER) {
    // If the offset isn't a multiple of 4, it probably isn't going to be
    // correctly aligned.
    // FIXME: Can we get the real alignment here?
    if (AM.BaseOffs % 4 != 0)
      return isLegalMUBUFAddressingMode(AM);
 
    if (!Subtarget->hasScalarSubwordLoads()) {
      // There are no SMRD extloads, so if we have to do a small type access we
      // will use a MUBUF load.
      // FIXME?: We also need to do this if unaligned, but we don't know the
      // alignment here.
      if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
        return isLegalGlobalAddressingMode(AM);
    }
 
    if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
      // SMRD instructions have an 8-bit, dword offset on SI.
      if (!isUInt<8>(AM.BaseOffs / 4))
        return false;
    } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
      // On CI+, this can also be a 32-bit literal constant offset. If it fits
      // in 8-bits, it can use a smaller encoding.
      if (!isUInt<32>(AM.BaseOffs / 4))
        return false;
    } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX9) {
      // On VI, these use the SMEM format and the offset is 20-bit in bytes.
      if (!isUInt<20>(AM.BaseOffs))
        return false;
    } else if (Subtarget->getGeneration() < AMDGPUSubtarget::GFX12) {
      // On GFX9 the offset is signed 21-bit in bytes (but must not be negative
      // for S_BUFFER_* instructions).
      if (!isInt<21>(AM.BaseOffs))
        return false;
    } else {
      // On GFX12, all offsets are signed 24-bit in bytes.
      if (!isInt<24>(AM.BaseOffs))
        return false;
    }
 
    if ((AS == AMDGPUAS::CONSTANT_ADDRESS ||
         AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
        AM.BaseOffs < 0) {
      // Scalar (non-buffer) loads can only use a negative offset if
      // soffset+offset is non-negative. Since the compiler can only prove that
      // in a few special cases, it is safer to claim that negative offsets are
      // not supported.
      return false;
    }
 
    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
      return true;
 
    if (AM.Scale == 1 && AM.HasBaseReg)
      return true;
 
    return false;
  }
 
  if (AS == AMDGPUAS::PRIVATE_ADDRESS)
    return Subtarget->hasFlatScratchEnabled()
               ? isLegalFlatAddressingMode(AM, AMDGPUAS::PRIVATE_ADDRESS)
               : isLegalMUBUFAddressingMode(AM);
 
  if (AS == AMDGPUAS::LOCAL_ADDRESS ||
      (AS == AMDGPUAS::REGION_ADDRESS && Subtarget->hasGDS())) {
    // Basic, single offset DS instructions allow a 16-bit unsigned immediate
    // field.
    // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
    // an 8-bit dword offset but we don't know the alignment here.
    if (!isUInt<16>(AM.BaseOffs))
      return false;
 
    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
      return true;
 
    if (AM.Scale == 1 && AM.HasBaseReg)
      return true;
 
    return false;
  }
 
  if (AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE) {
    // For an unknown address space, this usually means that this is for some
    // reason being used for pure arithmetic, and not based on some addressing
    // computation. We don't have instructions that compute pointers with any
    // addressing modes, so treat them as having no offset like flat
    // instructions.
    return isLegalFlatAddressingMode(AM, AMDGPUAS::FLAT_ADDRESS);
  }
 
  // Assume a user alias of global for unknown address spaces.
  return isLegalGlobalAddressingMode(AM);
}
 
bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
                                        const MachineFunction &MF) const {
  if (AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS)
    return (MemVT.getSizeInBits() <= 4 * 32);
  if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
    unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
    return (MemVT.getSizeInBits() <= MaxPrivateBits);
  }
  if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
    return (MemVT.getSizeInBits() <= 2 * 32);
  return true;
}
 
bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
    unsigned Size, unsigned AddrSpace, Align Alignment,
    MachineMemOperand::Flags Flags, unsigned *IsFast) const {
  if (IsFast)
    *IsFast = 0;
 
  if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
      AddrSpace == AMDGPUAS::REGION_ADDRESS) {
    // Check if alignment requirements for ds_read/write instructions are
    // disabled.
    if (!Subtarget->hasUnalignedDSAccessEnabled() && Alignment < Align(4))
      return false;
 
    Align RequiredAlignment(
        PowerOf2Ceil(divideCeil(Size, 8))); // Natural alignment.
    if (Subtarget->hasLDSMisalignedBugInWGPMode() && Size > 32 &&
        Alignment < RequiredAlignment)
      return false;
 
    // Either, the alignment requirements are "enabled", or there is an
    // unaligned LDS access related hardware bug though alignment requirements
    // are "disabled". In either case, we need to check for proper alignment
    // requirements.
    //
    switch (Size) {
    case 64:
      // SI has a hardware bug in the LDS / GDS bounds checking: if the base
      // address is negative, then the instruction is incorrectly treated as
      // out-of-bounds even if base + offsets is in bounds. Split vectorized
      // loads here to avoid emitting ds_read2_b32. We may re-combine the
      // load later in the SILoadStoreOptimizer.
      if (!Subtarget->hasUsableDSOffset() && Alignment < Align(8))
        return false;
 
      // 8 byte accessing via ds_read/write_b64 require 8-byte alignment, but we
      // can do a 4 byte aligned, 8 byte access in a single operation using
      // ds_read2/write2_b32 with adjacent offsets.
      RequiredAlignment = Align(4);
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // We will either select ds_read_b64/ds_write_b64 or ds_read2_b32/
        // ds_write2_b32 depending on the alignment. In either case with either
        // alignment there is no faster way of doing this.
 
        // The numbers returned here and below are not additive, it is a 'speed
        // rank'. They are just meant to be compared to decide if a certain way
        // of lowering an operation is faster than another. For that purpose
        // naturally aligned operation gets it bitsize to indicate that "it
        // operates with a speed comparable to N-bit wide load". With the full
        // alignment ds128 is slower than ds96 for example. If underaligned it
        // is comparable to a speed of a single dword access, which would then
        // mean 32 < 128 and it is faster to issue a wide load regardless.
        // 1 is simply "slow, don't do it". I.e. comparing an aligned load to a
        // wider load which will not be aligned anymore the latter is slower.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 64
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    case 96:
      if (!Subtarget->hasDS96AndDS128())
        return false;
 
      // 12 byte accessing via ds_read/write_b96 require 16-byte alignment on
      // gfx8 and older.
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // Naturally aligned access is fastest. However, also report it is Fast
        // if memory is aligned less than DWORD. A narrow load or store will be
        // be equally slow as a single ds_read_b96/ds_write_b96, but there will
        // be more of them, so overall we will pay less penalty issuing a single
        // instruction.
 
        // See comment on the values above.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 96
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    case 128:
      if (!Subtarget->hasDS96AndDS128() || !Subtarget->useDS128())
        return false;
 
      // 16 byte accessing via ds_read/write_b128 require 16-byte alignment on
      // gfx8 and older, but  we can do a 8 byte aligned, 16 byte access in a
      // single operation using ds_read2/write2_b64.
      RequiredAlignment = Align(8);
 
      if (Subtarget->hasUnalignedDSAccessEnabled()) {
        // Naturally aligned access is fastest. However, also report it is Fast
        // if memory is aligned less than DWORD. A narrow load or store will be
        // be equally slow as a single ds_read_b128/ds_write_b128, but there
        // will be more of them, so overall we will pay less penalty issuing a
        // single instruction.
 
        // See comment on the values above.
        if (IsFast)
          *IsFast = (Alignment >= RequiredAlignment) ? 128
                    : (Alignment < Align(4))         ? 32
                                                     : 1;
        return true;
      }
 
      break;
    default:
      if (Size > 32)
        return false;
 
      break;
    }
 
    // See comment on the values above.
    // Note that we have a single-dword or sub-dword here, so if underaligned
    // it is a slowest possible access, hence returned value is 0.
    if (IsFast)
      *IsFast = (Alignment >= RequiredAlignment) ? Size : 0;
 
    return Alignment >= RequiredAlignment ||
           Subtarget->hasUnalignedDSAccessEnabled();
  }
 
  // FIXME: We have to be conservative here and assume that flat operations
  // will access scratch.  If we had access to the IR function, then we
  // could determine if any private memory was used in the function.
  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
      AddrSpace == AMDGPUAS::FLAT_ADDRESS) {
    bool AlignedBy4 = Alignment >= Align(4);
    if (Subtarget->hasUnalignedScratchAccessEnabled()) {
      if (IsFast)
        *IsFast = AlignedBy4 ? Size : 1;
      return true;
    }
 
    if (IsFast)
      *IsFast = AlignedBy4;
 
    return AlignedBy4;
  }
 
  // So long as they are correct, wide global memory operations perform better
  // than multiple smaller memory ops -- even when misaligned
  if (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace)) {
    if (IsFast)
      *IsFast = Size;
 
    return Alignment >= Align(4) ||
           Subtarget->hasUnalignedBufferAccessEnabled();
  }
 
  // Ensure robust out-of-bounds guarantees for buffer accesses are met when the
  // "amdgpu.buffer.oob.mode" module flag has not enabled relaxed untyped-buffer
  // OOB semantics. Normally hardware will ensure proper
  // out-of-bounds behavior, but in the edge case where an access starts
  // out-of-bounds and then enters in-bounds, the entire access would be treated
  // as out-of-bounds. Prevent misaligned memory accesses by requiring the
  // natural alignment of buffer accesses.
  if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
      AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
      AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
    if (!Subtarget->hasRelaxedBufferOOBMode() &&
        Alignment < Align(PowerOf2Ceil(divideCeil(Size, 8))))
      return false;
  }
 
  // Smaller than dword value must be aligned.
  if (Size < 32)
    return false;
 
  // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
  // byte-address are ignored, thus forcing Dword alignment.
  // This applies to private, global, and constant memory.
  if (IsFast)
    *IsFast = 1;
 
  return Size >= 32 && Alignment >= Align(4);
}
 
bool SITargetLowering::allowsMisalignedMemoryAccesses(
    EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
    unsigned *IsFast) const {
  return allowsMisalignedMemoryAccessesImpl(VT.getSizeInBits(), AddrSpace,
                                            Alignment, Flags, IsFast);
}
 
EVT SITargetLowering::getOptimalMemOpType(
    LLVMContext &Context, const MemOp &Op,
    const AttributeList &FuncAttributes) const {
  // FIXME: Should account for address space here.
 
  // The default fallback uses the private pointer size as a guess for a type to
  // use. Make sure we switch these to 64-bit accesses.
 
  if (Op.size() >= 16 &&
      Op.isDstAligned(Align(4))) // XXX: Should only do for global
    return MVT::v4i32;
 
  if (Op.size() >= 8 && Op.isDstAligned(Align(4)))
    return MVT::v2i32;
 
  // Use the default.
  return MVT::Other;
}
 
bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
  const MemSDNode *MemNode = cast<MemSDNode>(N);
  return MemNode->getMemOperand()->getFlags() & MONoClobber;
}
 
bool SITargetLowering::isNonGlobalAddrSpace(unsigned AS) {
  return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS ||
         AS == AMDGPUAS::PRIVATE_ADDRESS;
}
 
bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
                                           unsigned DestAS) const {
  if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
    if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
        Subtarget->hasGloballyAddressableScratch()) {
      // Flat -> private requires subtracting src_flat_scratch_base_lo.
      return false;
    }
 
    // Flat -> private/local is a simple truncate.
    // Flat -> global is no-op
    return true;
  }
 
  const GCNTargetMachine &TM =
      static_cast<const GCNTargetMachine &>(getTargetMachine());
  return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
}
 
TargetLoweringBase::LegalizeTypeAction
SITargetLowering::getPreferredVectorAction(MVT VT) const {
  if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
      VT.getScalarType().bitsLE(MVT::i16))
    return VT.isPow2VectorType() ? TypeSplitVector : TypeWidenVector;
  return TargetLoweringBase::getPreferredVectorAction(VT);
}
 
bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                                         Type *Ty) const {
  // FIXME: Could be smarter if called for vector constants.
  return true;
}
 
TargetLowering::ExtractSubvectorCost
SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
                                          unsigned Index) const {
  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
    return ExtractSubvectorCost::Expensive;
 
  // TODO: Add more cases that are cheap.
  if (Index == 0)
    return ExtractSubvectorCost::Free;
  return ExtractSubvectorCost::Expensive;
}
 
bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const {
  // TODO: This should be more aggressive, particular for 16-bit element
  // vectors. However there are some mixed improvements and regressions.
  EVT EltTy = VT.getVectorElementType();
  unsigned MinAlign = Subtarget->useRealTrue16Insts() ? 16 : 32;
  return EltTy.getSizeInBits() % MinAlign == 0;
}
 
bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
  if (Subtarget->has16BitInsts() && VT == MVT::i16) {
    switch (Op) {
    case ISD::LOAD:
    case ISD::STORE:
      return true;
    default:
      return false;
    }
  }
 
  // SimplifySetCC uses this function to determine whether or not it should
  // create setcc with i1 operands.  We don't have instructions for i1 setcc.
  if (VT == MVT::i1 && Op == ISD::SETCC)
    return false;
 
  return TargetLowering::isTypeDesirableForOp(Op, VT);
}
 
MachinePointerInfo
SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
  // This isn't really a constant pool but close enough.
  MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
  PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
  return PtrInfo;
}
 
SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
                                                   const SDLoc &SL,
                                                   SDValue Chain,
                                                   uint64_t Offset) const {
  const DataLayout &DL = DAG.getDataLayout();
  MachineFunction &MF = DAG.getMachineFunction();
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
 
  auto [InputPtrReg, RC, ArgTy] =
      Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
 
  // We may not have the kernarg segment argument if we have no kernel
  // arguments.
  if (!InputPtrReg)
    return DAG.getConstant(Offset, SL, PtrVT);
 
  MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
  SDValue BasePtr = DAG.getCopyFromReg(
      Chain, SL, MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
 
  return DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::getFixed(Offset));
}
 
SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
                                            const SDLoc &SL) const {
  uint64_t Offset =
      getImplicitParameterOffset(DAG.getMachineFunction(), FIRST_IMPLICIT);
  return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
}
 
SDValue SITargetLowering::getLDSKernelId(SelectionDAG &DAG,
                                         const SDLoc &SL) const {
 
  Function &F = DAG.getMachineFunction().getFunction();
  std::optional<uint32_t> KnownSize =
      AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
  if (KnownSize.has_value())
    return DAG.getConstant(*KnownSize, SL, MVT::i32);
  return SDValue();
}
 
SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
                                         const SDLoc &SL, SDValue Val,
                                         bool Signed,
                                         const ISD::InputArg *Arg) const {
  // First, if it is a widened vector, narrow it.
  if (VT.isVector() &&
      VT.getVectorNumElements() != MemVT.getVectorNumElements()) {
    EVT NarrowedVT =
        EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(),
                         VT.getVectorNumElements());
    Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
                      DAG.getConstant(0, SL, MVT::i32));
  }
 
  // Then convert the vector elements or scalar value.
  if (Arg && (Arg->Flags.isSExt() || Arg->Flags.isZExt()) && VT.bitsLT(MemVT)) {
    unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
    Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
  }
 
  if (MemVT.isFloatingPoint()) {
    if (VT.isFloatingPoint()) {
      Val = getFPExtOrFPRound(DAG, Val, SL, VT);
    } else {
      assert(!MemVT.isVector());
      EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
      SDValue Cast = DAG.getBitcast(IntVT, Val);
      Val = DAG.getAnyExtOrTrunc(Cast, SL, VT);
    }
  } else if (Signed)
    Val = DAG.getSExtOrTrunc(Val, SL, VT);
  else
    Val = DAG.getZExtOrTrunc(Val, SL, VT);
 
  return Val;
}
 
SDValue SITargetLowering::lowerKernargMemParameter(
    SelectionDAG &DAG, EVT VT, EVT MemVT, const SDLoc &SL, SDValue Chain,
    uint64_t Offset, Align Alignment, bool Signed,
    const ISD::InputArg *Arg) const {
 
  MachinePointerInfo PtrInfo =
      getKernargSegmentPtrInfo(DAG.getMachineFunction());
 
  // Try to avoid using an extload by loading earlier than the argument address,
  // and extracting the relevant bits. The load should hopefully be merged with
  // the previous argument.
  if (MemVT.getStoreSize() < 4 && Alignment < 4) {
    // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
    int64_t AlignDownOffset = alignDown(Offset, 4);
    int64_t OffsetDiff = Offset - AlignDownOffset;
 
    EVT IntVT = MemVT.changeTypeToInteger();
 
    // TODO: If we passed in the base kernel offset we could have a better
    // alignment than 4, but we don't really need it.
    SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
    SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr,
                               PtrInfo.getWithOffset(AlignDownOffset), Align(4),
                               MachineMemOperand::MODereferenceable |
                                   MachineMemOperand::MOInvariant);
 
    SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
    SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
 
    SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
    ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
    ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
 
    return DAG.getMergeValues({ArgVal, Load.getValue(1)}, SL);
  }
 
  SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
  SDValue Load = DAG.getLoad(
      MemVT, SL, Chain, Ptr, PtrInfo.getWithOffset(Offset), Alignment,
      MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant);
 
  SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
  return DAG.getMergeValues({Val, Load.getValue(1)}, SL);
}
 
/// Coerce an argument which was passed in a different ABI type to the original
/// expected value type.
SDValue SITargetLowering::convertABITypeToValueType(SelectionDAG &DAG,
                                                    SDValue Val,
                                                    CCValAssign &VA,
                                                    const SDLoc &SL) const {
  EVT ValVT = VA.getValVT();
 
  // If this is an 8 or 16-bit value, it is really passed promoted
  // to 32 bits. Insert an assert[sz]ext to capture this, then
  // truncate to the right size.
  switch (VA.getLocInfo()) {
  case CCValAssign::Full:
    return Val;
  case CCValAssign::BCvt:
    return DAG.getNode(ISD::BITCAST, SL, ValVT, Val);
  case CCValAssign::SExt:
    Val = DAG.getNode(ISD::AssertSext, SL, VA.getLocVT(), Val,
                      DAG.getValueType(ValVT));
    return DAG.getNode(ISD::TRUNCATE, SL, ValVT, Val);
  case CCValAssign::ZExt:
    Val = DAG.getNode(ISD::AssertZext, SL, VA.getLocVT(), Val,
                      DAG.getValueType(ValVT));
    return DAG.getNode(ISD::TRUNCATE, SL, ValVT, Val);
  case CCValAssign::AExt:
    return DAG.getNode(ISD::TRUNCATE, SL, ValVT, Val);
  default:
    llvm_unreachable("Unknown loc info!");
  }
}
 
SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG,
                                              CCValAssign &VA, const SDLoc &SL,
                                              SDValue Chain,
                                              const ISD::InputArg &Arg) const {
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  if (Arg.Flags.isByVal()) {
    unsigned Size = Arg.Flags.getByValSize();
    int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false);
    return DAG.getFrameIndex(FrameIdx, MVT::i32);
  }
 
  unsigned ArgOffset = VA.getLocMemOffset();
  unsigned ArgSize = VA.getValVT().getStoreSize();
 
  int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true);
 
  // Create load nodes to retrieve arguments from the stack.
  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
 
  // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
  ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
  MVT MemVT = VA.getValVT();
 
  switch (VA.getLocInfo()) {
  default:
    break;
  case CCValAssign::BCvt:
    MemVT = VA.getLocVT();
    break;
  case CCValAssign::SExt:
    ExtType = ISD::SEXTLOAD;
    break;
  case CCValAssign::ZExt:
    ExtType = ISD::ZEXTLOAD;
    break;
  case CCValAssign::AExt:
    ExtType = ISD::EXTLOAD;
    break;
  }
 
  SDValue ArgValue = DAG.getExtLoad(
      ExtType, SL, VA.getLocVT(), Chain, FIN,
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), MemVT);
 
  SDValue ConvertedVal = convertABITypeToValueType(DAG, ArgValue, VA, SL);
  if (ConvertedVal == ArgValue)
    return ConvertedVal;
 
  return DAG.getMergeValues({ConvertedVal, ArgValue.getValue(1)}, SL);
}
 
SDValue SITargetLowering::lowerWorkGroupId(
    SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
    AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
    AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
    AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
  if (!Subtarget->hasClusters())
    return getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
 
  // Clusters are supported. Return the global position in the grid. If clusters
  // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
 
  // WorkGroupIdXYZ = ClusterId == 0 ?
  //   ClusterIdXYZ :
  //   ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
  SDValue ClusterIdXYZ = getPreloadedValue(DAG, MFI, VT, WorkGroupIdPV);
  SDLoc SL(ClusterIdXYZ);
  SDValue ClusterMaxIdXYZ = getPreloadedValue(DAG, MFI, VT, ClusterMaxIdPV);
  SDValue One = DAG.getConstant(1, SL, VT);
  SDValue ClusterSizeXYZ = DAG.getNode(ISD::ADD, SL, VT, ClusterMaxIdXYZ, One);
  SDValue ClusterWorkGroupIdXYZ =
      getPreloadedValue(DAG, MFI, VT, ClusterWorkGroupIdPV);
  SDValue GlobalIdXYZ =
      DAG.getNode(ISD::ADD, SL, VT, ClusterWorkGroupIdXYZ,
                  DAG.getNode(ISD::MUL, SL, VT, ClusterIdXYZ, ClusterSizeXYZ));
 
  switch (MFI.getClusterDims().getKind()) {
  case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
  case AMDGPU::ClusterDimsAttr::Kind::VariableDims:
    return GlobalIdXYZ;
  case AMDGPU::ClusterDimsAttr::Kind::NoCluster:
    return ClusterIdXYZ;
  case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
    using namespace AMDGPU::Hwreg;
    SDValue ClusterIdField =
        DAG.getTargetConstant(HwregEncoding::encode(ID_IB_STS2, 6, 4), SL, VT);
    SDNode *GetReg =
        DAG.getMachineNode(AMDGPU::S_GETREG_B32_const, SL, VT, ClusterIdField);
    SDValue ClusterId(GetReg, 0);
    SDValue Zero = DAG.getConstant(0, SL, VT);
    return DAG.getNode(ISD::SELECT_CC, SL, VT, ClusterId, Zero, ClusterIdXYZ,
                       GlobalIdXYZ, DAG.getCondCode(ISD::SETEQ));
  }
  }
 
  llvm_unreachable("nothing should reach here");
}
 
SDValue SITargetLowering::getPreloadedValue(
    SelectionDAG &DAG, const SIMachineFunctionInfo &MFI, EVT VT,
    AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
  const ArgDescriptor *Reg = nullptr;
  const TargetRegisterClass *RC = nullptr;
  LLT Ty;
 
  CallingConv::ID CC = DAG.getMachineFunction().getFunction().getCallingConv();
  const ArgDescriptor WorkGroupIDX =
      ArgDescriptor::createRegister(AMDGPU::TTMP9);
  // If GridZ is not programmed in an entry function then the hardware will set
  // it to all zeros, so there is no need to mask the GridY value in the low
  // order bits.
  const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
      AMDGPU::TTMP7,
      AMDGPU::isEntryFunctionCC(CC) && !MFI.hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
  const ArgDescriptor WorkGroupIDZ =
      ArgDescriptor::createRegister(AMDGPU::TTMP7, 0xFFFF0000u);
  const ArgDescriptor ClusterWorkGroupIDX =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000000Fu);
  const ArgDescriptor ClusterWorkGroupIDY =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000000F0u);
  const ArgDescriptor ClusterWorkGroupIDZ =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00000F00u);
  const ArgDescriptor ClusterWorkGroupMaxIDX =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000F000u);
  const ArgDescriptor ClusterWorkGroupMaxIDY =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000F0000u);
  const ArgDescriptor ClusterWorkGroupMaxIDZ =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00F00000u);
  const ArgDescriptor ClusterWorkGroupMaxFlatID =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0F000000u);
 
  auto LoadConstant = [&](unsigned N) {
    return DAG.getConstant(N, SDLoc(), VT);
  };
 
  if (Subtarget->hasArchitectedSGPRs() &&
      (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
    AMDGPU::ClusterDimsAttr ClusterDims = MFI.getClusterDims();
    bool HasFixedDims = ClusterDims.isFixedDims();
 
    switch (PVID) {
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
      Reg = &WorkGroupIDX;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
      Reg = &WorkGroupIDY;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
      Reg = &WorkGroupIDZ;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
      if (HasFixedDims && ClusterDims.getDims()[0] == 1)
        return LoadConstant(0);
      Reg = &ClusterWorkGroupIDX;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
      if (HasFixedDims && ClusterDims.getDims()[1] == 1)
        return LoadConstant(0);
      Reg = &ClusterWorkGroupIDY;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
      if (HasFixedDims && ClusterDims.getDims()[2] == 1)
        return LoadConstant(0);
      Reg = &ClusterWorkGroupIDZ;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
      if (HasFixedDims)
        return LoadConstant(ClusterDims.getDims()[0] - 1);
      Reg = &ClusterWorkGroupMaxIDX;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
      if (HasFixedDims)
        return LoadConstant(ClusterDims.getDims()[1] - 1);
      Reg = &ClusterWorkGroupMaxIDY;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
      if (HasFixedDims)
        return LoadConstant(ClusterDims.getDims()[2] - 1);
      Reg = &ClusterWorkGroupMaxIDZ;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
      Reg = &ClusterWorkGroupMaxFlatID;
      RC = &AMDGPU::SReg_32RegClass;
      Ty = LLT::scalar(32);
      break;
    default:
      break;
    }
  }
 
  if (!Reg)
    std::tie(Reg, RC, Ty) = MFI.getPreloadedValue(PVID);
  if (!Reg) {
    if (PVID == AMDGPUFunctionArgInfo::PreloadedValue::KERNARG_SEGMENT_PTR) {
      // It's possible for a kernarg intrinsic call to appear in a kernel with
      // no allocated segment, in which case we do not add the user sgpr
      // argument, so just return null.
      return DAG.getConstant(0, SDLoc(), VT);
    }
 
    // It's undefined behavior if a function marked with the amdgpu-no-*
    // attributes uses the corresponding intrinsic.
    return DAG.getPOISON(VT);
  }
 
  return loadInputValue(DAG, RC, VT, SDLoc(DAG.getEntryNode()), *Reg);
}
 
static void processPSInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
                               CallingConv::ID CallConv,
                               ArrayRef<ISD::InputArg> Ins, BitVector &Skipped,
                               FunctionType *FType,
                               SIMachineFunctionInfo *Info) {
  for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; ++I) {
    const ISD::InputArg *Arg = &Ins[I];
 
    assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
           "vector type argument should have been split");
 
    // First check if it's a PS input addr.
    if (CallConv == CallingConv::AMDGPU_PS && !Arg->Flags.isInReg() &&
        PSInputNum <= 15) {
      bool SkipArg = !Arg->Used && !Info->isPSInputAllocated(PSInputNum);
 
      // Inconveniently only the first part of the split is marked as isSplit,
      // so skip to the end. We only want to increment PSInputNum once for the
      // entire split argument.
      if (Arg->Flags.isSplit()) {
        while (!Arg->Flags.isSplitEnd()) {
          assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
                 "unexpected vector split in ps argument type");
          if (!SkipArg)
            Splits.push_back(*Arg);
          Arg = &Ins[++I];
        }
      }
 
      if (SkipArg) {
        // We can safely skip PS inputs.
        Skipped.set(Arg->getOrigArgIndex());
        ++PSInputNum;
        continue;
      }
 
      Info->markPSInputAllocated(PSInputNum);
      if (Arg->Used)
        Info->markPSInputEnabled(PSInputNum);
 
      ++PSInputNum;
    }
 
    Splits.push_back(*Arg);
  }
}
 
// Allocate special inputs passed in VGPRs.
void SITargetLowering::allocateSpecialEntryInputVGPRs(
    CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
    SIMachineFunctionInfo &Info) const {
  const LLT S32 = LLT::scalar(32);
  MachineRegisterInfo &MRI = MF.getRegInfo();
 
  if (Info.hasWorkItemIDX()) {
    Register Reg = AMDGPU::VGPR0;
    MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
    CCInfo.AllocateReg(Reg);
    unsigned Mask =
        (Subtarget->hasPackedTID() && Info.hasWorkItemIDY()) ? 0x3ff : ~0u;
    Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
  }
 
  if (Info.hasWorkItemIDY()) {
    assert(Info.hasWorkItemIDX());
    if (Subtarget->hasPackedTID()) {
      Info.setWorkItemIDY(
          ArgDescriptor::createRegister(AMDGPU::VGPR0, 0x3ff << 10));
    } else {
      unsigned Reg = AMDGPU::VGPR1;
      MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
      CCInfo.AllocateReg(Reg);
      Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
    }
  }
 
  if (Info.hasWorkItemIDZ()) {
    assert(Info.hasWorkItemIDX() && Info.hasWorkItemIDY());
    if (Subtarget->hasPackedTID()) {
      Info.setWorkItemIDZ(
          ArgDescriptor::createRegister(AMDGPU::VGPR0, 0x3ff << 20));
    } else {
      unsigned Reg = AMDGPU::VGPR2;
      MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
 
      CCInfo.AllocateReg(Reg);
      Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
    }
  }
}
 
// Try to allocate a VGPR at the end of the argument list, or if no argument
// VGPRs are left allocating a stack slot.
// If \p Mask is given it indicates bitfield position in the register.
// If \p Arg is given use it with new ]p Mask instead of allocating new.
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
                                         ArgDescriptor Arg = ArgDescriptor()) {
  if (Arg.isSet())
    return ArgDescriptor::createArg(Arg, Mask);
 
  ArrayRef<MCPhysReg> ArgVGPRs = ArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs);
  if (RegIdx == ArgVGPRs.size()) {
    // Spill to stack required.
    int64_t Offset = CCInfo.AllocateStack(4, Align(4));
 
    return ArgDescriptor::createStack(Offset, Mask);
  }
 
  unsigned Reg = ArgVGPRs[RegIdx];
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
 
  MachineFunction &MF = CCInfo.getMachineFunction();
  Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
  MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32));
  return ArgDescriptor::createRegister(Reg, Mask);
}
 
static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
                                             const TargetRegisterClass *RC,
                                             unsigned NumArgRegs) {
  ArrayRef<MCPhysReg> ArgSGPRs = ArrayRef(RC->begin(), 32);
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
  if (RegIdx == ArgSGPRs.size())
    report_fatal_error("ran out of SGPRs for arguments");
 
  unsigned Reg = ArgSGPRs[RegIdx];
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
 
  MachineFunction &MF = CCInfo.getMachineFunction();
  MF.addLiveIn(Reg, RC);
  return ArgDescriptor::createRegister(Reg);
}
 
// If this has a fixed position, we still should allocate the register in the
// CCInfo state. Technically we could get away with this for values passed
// outside of the normal argument range.
static void allocateFixedSGPRInputImpl(CCState &CCInfo,
                                       const TargetRegisterClass *RC,
                                       MCRegister Reg) {
  Reg = CCInfo.AllocateReg(Reg);
  assert(Reg != AMDGPU::NoRegister);
  MachineFunction &MF = CCInfo.getMachineFunction();
  MF.addLiveIn(Reg, RC);
}
 
static void allocateSGPR32Input(CCState &CCInfo, ArgDescriptor &Arg) {
  if (Arg) {
    allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_32RegClass,
                               Arg.getRegister());
  } else
    Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32);
}
 
static void allocateSGPR64Input(CCState &CCInfo, ArgDescriptor &Arg) {
  if (Arg) {
    allocateFixedSGPRInputImpl(CCInfo, &AMDGPU::SGPR_64RegClass,
                               Arg.getRegister());
  } else
    Arg = allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
}
 
/// Allocate implicit function VGPR arguments at the end of allocated user
/// arguments.
void SITargetLowering::allocateSpecialInputVGPRs(
    CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
    SIMachineFunctionInfo &Info) const {
  const unsigned Mask = 0x3ff;
  ArgDescriptor Arg;
 
  if (Info.hasWorkItemIDX()) {
    Arg = allocateVGPR32Input(CCInfo, Mask);
    Info.setWorkItemIDX(Arg);
  }
 
  if (Info.hasWorkItemIDY()) {
    Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg);
    Info.setWorkItemIDY(Arg);
  }
 
  if (Info.hasWorkItemIDZ())
    Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg));
}
 
/// Allocate implicit function VGPR arguments in fixed registers.
void SITargetLowering::allocateSpecialInputVGPRsFixed(
    CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
    SIMachineFunctionInfo &Info) const {
  Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31);
  if (!Reg)
    report_fatal_error("failed to allocate VGPR for implicit arguments");
 
  const unsigned Mask = 0x3ff;
  Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask));
  Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg, Mask << 10));
  Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg, Mask << 20));
}
 
void SITargetLowering::allocateSpecialInputSGPRs(
    CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI,
    SIMachineFunctionInfo &Info) const {
  auto &ArgInfo = Info.getArgInfo();
  const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
 
  // TODO: Unify handling with private memory pointers.
  if (UserSGPRInfo.hasDispatchPtr())
    allocateSGPR64Input(CCInfo, ArgInfo.DispatchPtr);
 
  if (UserSGPRInfo.hasQueuePtr())
    allocateSGPR64Input(CCInfo, ArgInfo.QueuePtr);
 
  // Implicit arg ptr takes the place of the kernarg segment pointer. This is a
  // constant offset from the kernarg segment.
  if (Info.hasImplicitArgPtr())
    allocateSGPR64Input(CCInfo, ArgInfo.ImplicitArgPtr);
 
  if (UserSGPRInfo.hasDispatchID())
    allocateSGPR64Input(CCInfo, ArgInfo.DispatchID);
 
  // flat_scratch_init is not applicable for non-kernel functions.
 
  if (Info.hasWorkGroupIDX())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDX);
 
  if (Info.hasWorkGroupIDY())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDY);
 
  if (Info.hasWorkGroupIDZ())
    allocateSGPR32Input(CCInfo, ArgInfo.WorkGroupIDZ);
 
  if (Info.hasLDSKernelId())
    allocateSGPR32Input(CCInfo, ArgInfo.LDSKernelId);
}
 
// Allocate special inputs passed in user SGPRs.
void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
                                            MachineFunction &MF,
                                            const SIRegisterInfo &TRI,
                                            SIMachineFunctionInfo &Info) const {
  const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();
  if (UserSGPRInfo.hasImplicitBufferPtr()) {
    Register ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
    MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(ImplicitBufferPtrReg);
  }
 
  // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
  if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
    Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
    MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
    CCInfo.AllocateReg(PrivateSegmentBufferReg);
  }
 
  if (UserSGPRInfo.hasDispatchPtr()) {
    Register DispatchPtrReg = Info.addDispatchPtr(TRI);
    MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(DispatchPtrReg);
  }
 
  if (UserSGPRInfo.hasQueuePtr()) {
    Register QueuePtrReg = Info.addQueuePtr(TRI);
    MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(QueuePtrReg);
  }
 
  if (UserSGPRInfo.hasKernargSegmentPtr()) {
    MachineRegisterInfo &MRI = MF.getRegInfo();
    Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
    CCInfo.AllocateReg(InputPtrReg);
 
    Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass);
    MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
  }
 
  if (UserSGPRInfo.hasDispatchID()) {
    Register DispatchIDReg = Info.addDispatchID(TRI);
    MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(DispatchIDReg);
  }
 
  if (UserSGPRInfo.hasFlatScratchInit() && !getSubtarget()->isAmdPalOS()) {
    Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);
    MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
    CCInfo.AllocateReg(FlatScratchInitReg);
  }
 
  if (UserSGPRInfo.hasPrivateSegmentSize()) {
    Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);
    MF.addLiveIn(PrivateSegmentSizeReg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(PrivateSegmentSizeReg);
  }
 
  // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
  // these from the dispatch pointer.
}
 
// Allocate pre-loaded kernel arguemtns. Arguments to be preloading must be
// sequential starting from the first argument.
void SITargetLowering::allocatePreloadKernArgSGPRs(
    CCState &CCInfo, SmallVectorImpl<CCValAssign> &ArgLocs,
    const SmallVectorImpl<ISD::InputArg> &Ins, MachineFunction &MF,
    const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info) const {
  Function &F = MF.getFunction();
  unsigned LastExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
  GCNUserSGPRUsageInfo &SGPRInfo = Info.getUserSGPRInfo();
  bool InPreloadSequence = true;
  unsigned InIdx = 0;
  bool AlignedForImplictArgs = false;
  unsigned ImplicitArgOffset = 0;
  for (auto &Arg : F.args()) {
    if (!InPreloadSequence || !Arg.hasInRegAttr())
      break;
 
    unsigned ArgIdx = Arg.getArgNo();
    // Don't preload non-original args or parts not in the current preload
    // sequence.
    if (InIdx < Ins.size() &&
        (!Ins[InIdx].isOrigArg() || Ins[InIdx].getOrigArgIndex() != ArgIdx))
      break;
 
    for (; InIdx < Ins.size() && Ins[InIdx].isOrigArg() &&
           Ins[InIdx].getOrigArgIndex() == ArgIdx;
         InIdx++) {
      assert(ArgLocs[ArgIdx].isMemLoc());
      auto &ArgLoc = ArgLocs[InIdx];
      const Align KernelArgBaseAlign = Align(16);
      unsigned ArgOffset = ArgLoc.getLocMemOffset();
      Align Alignment = commonAlignment(KernelArgBaseAlign, ArgOffset);
      unsigned NumAllocSGPRs =
          alignTo(ArgLoc.getLocVT().getFixedSizeInBits(), 32) / 32;
 
      // Fix alignment for hidden arguments.
      if (Arg.hasAttribute("amdgpu-hidden-argument")) {
        if (!AlignedForImplictArgs) {
          ImplicitArgOffset =
              alignTo(LastExplicitArgOffset,
                      Subtarget->getAlignmentForImplicitArgPtr()) -
              LastExplicitArgOffset;
          AlignedForImplictArgs = true;
        }
        ArgOffset += ImplicitArgOffset;
      }
 
      // Arg is preloaded into the previous SGPR.
      if (ArgLoc.getLocVT().getStoreSize() < 4 && Alignment < 4) {
        assert(InIdx >= 1 && "No previous SGPR");
        Info.getArgInfo().PreloadKernArgs[InIdx].Regs.push_back(
            Info.getArgInfo().PreloadKernArgs[InIdx - 1].Regs[0]);
        continue;
      }
 
      unsigned Padding = ArgOffset - LastExplicitArgOffset;
      unsigned PaddingSGPRs = alignTo(Padding, 4) / 4;
      // Check for free user SGPRs for preloading.
      if (PaddingSGPRs + NumAllocSGPRs > SGPRInfo.getNumFreeUserSGPRs()) {
        InPreloadSequence = false;
        break;
      }
 
      // Preload this argument.
      const TargetRegisterClass *RC =
          TRI.getSGPRClassForBitWidth(NumAllocSGPRs * 32);
      SmallVectorImpl<MCRegister> *PreloadRegs =
          Info.addPreloadedKernArg(TRI, RC, NumAllocSGPRs, InIdx, PaddingSGPRs);
 
      if (PreloadRegs->size() > 1)
        RC = &AMDGPU::SGPR_32RegClass;
      for (auto &Reg : *PreloadRegs) {
        assert(Reg);
        MF.addLiveIn(Reg, RC);
        CCInfo.AllocateReg(Reg);
      }
 
      LastExplicitArgOffset = NumAllocSGPRs * 4 + ArgOffset;
    }
  }
}
 
void SITargetLowering::allocateLDSKernelId(CCState &CCInfo, MachineFunction &MF,
                                           const SIRegisterInfo &TRI,
                                           SIMachineFunctionInfo &Info) const {
  // Always allocate this last since it is a synthetic preload.
  if (Info.hasLDSKernelId()) {
    Register Reg = Info.addLDSKernelId();
    MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(Reg);
  }
}
 
// Allocate special input registers that are initialized per-wave.
void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo, MachineFunction &MF,
                                           SIMachineFunctionInfo &Info,
                                           CallingConv::ID CallConv,
                                           bool IsShader) const {
  bool HasArchitectedSGPRs = Subtarget->hasArchitectedSGPRs();
  if (Subtarget->hasUserSGPRInit16BugInWave32() && !IsShader) {
    // Note: user SGPRs are handled by the front-end for graphics shaders
    // Pad up the used user SGPRs with dead inputs.
 
    // TODO: NumRequiredSystemSGPRs computation should be adjusted appropriately
    // before enabling architected SGPRs for workgroup IDs.
    assert(!HasArchitectedSGPRs && "Unhandled feature for the subtarget");
 
    unsigned CurrentUserSGPRs = Info.getNumUserSGPRs();
    // Note we do not count the PrivateSegmentWaveByteOffset. We do not want to
    // rely on it to reach 16 since if we end up having no stack usage, it will
    // not really be added.
    unsigned NumRequiredSystemSGPRs =
        Info.hasWorkGroupIDX() + Info.hasWorkGroupIDY() +
        Info.hasWorkGroupIDZ() + Info.hasWorkGroupInfo();
    for (unsigned i = NumRequiredSystemSGPRs + CurrentUserSGPRs; i < 16; ++i) {
      Register Reg = Info.addReservedUserSGPR();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
  }
 
  if (!HasArchitectedSGPRs) {
    if (Info.hasWorkGroupIDX()) {
      Register Reg = Info.addWorkGroupIDX();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
 
    if (Info.hasWorkGroupIDY()) {
      Register Reg = Info.addWorkGroupIDY();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
 
    if (Info.hasWorkGroupIDZ()) {
      Register Reg = Info.addWorkGroupIDZ();
      MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
      CCInfo.AllocateReg(Reg);
    }
  }
 
  if (Info.hasWorkGroupInfo()) {
    Register Reg = Info.addWorkGroupInfo();
    MF.addLiveIn(Reg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(Reg);
  }
 
  if (Info.hasPrivateSegmentWaveByteOffset()) {
    // Scratch wave offset passed in system SGPR.
    unsigned PrivateSegmentWaveByteOffsetReg;
 
    if (IsShader) {
      PrivateSegmentWaveByteOffsetReg =
          Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
 
      // This is true if the scratch wave byte offset doesn't have a fixed
      // location.
      if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
        PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
        Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
      }
    } else
      PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
 
    MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
    CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
  }
 
  assert(!Subtarget->hasUserSGPRInit16BugInWave32() || IsShader ||
         Info.getNumPreloadedSGPRs() >= 16);
}
 
static void reservePrivateMemoryRegs(const TargetMachine &TM,
                                     MachineFunction &MF,
                                     const SIRegisterInfo &TRI,
                                     SIMachineFunctionInfo &Info) {
  // Now that we've figured out where the scratch register inputs are, see if
  // should reserve the arguments and use them directly.
  MachineFrameInfo &MFI = MF.getFrameInfo();
  bool HasStackObjects = MFI.hasStackObjects();
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
  // Record that we know we have non-spill stack objects so we don't need to
  // check all stack objects later.
  if (HasStackObjects)
    Info.setHasNonSpillStackObjects(true);
 
  // Everything live out of a block is spilled with fast regalloc, so it's
  // almost certain that spilling will be required.
  if (TM.getOptLevel() == CodeGenOptLevel::None)
    HasStackObjects = true;
 
  // For now assume stack access is needed in any callee functions, so we need
  // the scratch registers to pass in.
  bool RequiresStackAccess = HasStackObjects || MFI.hasCalls();
 
  if (!ST.hasFlatScratchEnabled()) {
    if (RequiresStackAccess && ST.isAmdHsaOrMesa(MF.getFunction())) {
      // If we have stack objects, we unquestionably need the private buffer
      // resource. For the Code Object V2 ABI, this will be the first 4 user
      // SGPR inputs. We can reserve those and use them directly.
 
      Register PrivateSegmentBufferReg =
          Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
      Info.setScratchRSrcReg(PrivateSegmentBufferReg);
    } else {
      unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
      // We tentatively reserve the last registers (skipping the last registers
      // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
      // we'll replace these with the ones immediately after those which were
      // really allocated. In the prologue copies will be inserted from the
      // argument to these reserved registers.
 
      // Without HSA, relocations are used for the scratch pointer and the
      // buffer resource setup is always inserted in the prologue. Scratch wave
      // offset is still in an input SGPR.
      Info.setScratchRSrcReg(ReservedBufferReg);
    }
  }
 
  MachineRegisterInfo &MRI = MF.getRegInfo();
 
  // For entry functions we have to set up the stack pointer if we use it,
  // whereas non-entry functions get this "for free". This means there is no
  // intrinsic advantage to using S32 over S34 in cases where we do not have
  // calls but do need a frame pointer (i.e. if we are requested to have one
  // because frame pointer elimination is disabled). To keep things simple we
  // only ever use S32 as the call ABI stack pointer, and so using it does not
  // imply we need a separate frame pointer.
  //
  // Try to use s32 as the SP, but move it if it would interfere with input
  // arguments. This won't work with calls though.
  //
  // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
  // registers.
  if (!MRI.isLiveIn(AMDGPU::SGPR32)) {
    Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
  } else {
    assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
 
    if (MFI.hasCalls())
      report_fatal_error("call in graphics shader with too many input SGPRs");
 
    for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
      if (!MRI.isLiveIn(Reg)) {
        Info.setStackPtrOffsetReg(Reg);
        break;
      }
    }
 
    if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
      report_fatal_error("failed to find register for SP");
  }
 
  // hasFP should be accurate for entry functions even before the frame is
  // finalized, because it does not rely on the known stack size, only
  // properties like whether variable sized objects are present.
  if (ST.getFrameLowering()->hasFP(MF)) {
    Info.setFrameOffsetReg(AMDGPU::SGPR33);
  }
}
 
bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
  const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
  return !Info->isEntryFunction();
}
 
void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {}
 
void SITargetLowering::insertCopiesSplitCSR(
    MachineBasicBlock *Entry,
    const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
  const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
  if (!IStart)
    return;
 
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
  MachineBasicBlock::iterator MBBI = Entry->begin();
  for (const MCPhysReg *I = IStart; *I; ++I) {
    const TargetRegisterClass *RC = nullptr;
    if (AMDGPU::SReg_64RegClass.contains(*I))
      RC = &AMDGPU::SGPR_64RegClass;
    else if (AMDGPU::SReg_32RegClass.contains(*I))
      RC = &AMDGPU::SGPR_32RegClass;
    else
      llvm_unreachable("Unexpected register class in CSRsViaCopy!");
 
    Register NewVR = MRI->createVirtualRegister(RC);
    // Create copy from CSR to a virtual register.
    Entry->addLiveIn(*I);
    BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
        .addReg(*I);
 
    // Insert the copy-back instructions right before the terminator.
    for (auto *Exit : Exits)
      BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
              TII->get(TargetOpcode::COPY), *I)
          .addReg(NewVR);
  }
}
 
SDValue SITargetLowering::LowerFormalArguments(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &Fn = MF.getFunction();
  FunctionType *FType = MF.getFunction().getFunctionType();
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  bool IsError = false;
 
  if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) {
    DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
        Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()));
    IsError = true;
  }
 
  SmallVector<ISD::InputArg, 16> Splits;
  SmallVector<CCValAssign, 16> ArgLocs;
  BitVector Skipped(Fn.arg_size());
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                 *DAG.getContext());
 
  bool IsGraphics = AMDGPU::isGraphics(CallConv);
  bool IsKernel = AMDGPU::isKernel(CallConv);
  bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv);
 
  if (IsGraphics) {
    const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();
    assert(!UserSGPRInfo.hasDispatchPtr() &&
           !UserSGPRInfo.hasKernargSegmentPtr() && !Info->hasWorkGroupInfo() &&
           !Info->hasLDSKernelId() && !Info->hasWorkItemIDX() &&
           !Info->hasWorkItemIDY() && !Info->hasWorkItemIDZ());
    (void)UserSGPRInfo;
    if (!Subtarget->hasFlatScratchEnabled())
      assert(!UserSGPRInfo.hasFlatScratchInit());
    if ((CallConv != CallingConv::AMDGPU_CS &&
         CallConv != CallingConv::AMDGPU_Gfx &&
         CallConv != CallingConv::AMDGPU_Gfx_WholeWave) ||
        !Subtarget->hasArchitectedSGPRs())
      assert(!Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
             !Info->hasWorkGroupIDZ());
  }
 
  bool IsWholeWaveFunc = Info->isWholeWaveFunction();
 
  if (CallConv == CallingConv::AMDGPU_PS) {
    processPSInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
 
    // At least one interpolation mode must be enabled or else the GPU will
    // hang.
    //
    // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
    // set PSInputAddr, the user wants to enable some bits after the compilation
    // based on run-time states. Since we can't know what the final PSInputEna
    // will look like, so we shouldn't do anything here and the user should take
    // responsibility for the correct programming.
    //
    // Otherwise, the following restrictions apply:
    // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
    // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
    //   enabled too.
    if ((Info->getPSInputAddr() & 0x7F) == 0 ||
        ((Info->getPSInputAddr() & 0xF) == 0 && Info->isPSInputAllocated(11))) {
      CCInfo.AllocateReg(AMDGPU::VGPR0);
      CCInfo.AllocateReg(AMDGPU::VGPR1);
      Info->markPSInputAllocated(0);
      Info->markPSInputEnabled(0);
    }
    if (Subtarget->isAmdPalOS()) {
      // For isAmdPalOS, the user does not enable some bits after compilation
      // based on run-time states; the register values being generated here are
      // the final ones set in hardware. Therefore we need to apply the
      // workaround to PSInputAddr and PSInputEnable together.  (The case where
      // a bit is set in PSInputAddr but not PSInputEnable is where the
      // frontend set up an input arg for a particular interpolation mode, but
      // nothing uses that input arg. Really we should have an earlier pass
      // that removes such an arg.)
      unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
      if ((PsInputBits & 0x7F) == 0 ||
          ((PsInputBits & 0xF) == 0 && (PsInputBits >> 11 & 1)))
        Info->markPSInputEnabled(llvm::countr_zero(Info->getPSInputAddr()));
    }
  } else if (IsKernel) {
    assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
  } else {
    Splits.append(IsWholeWaveFunc ? std::next(Ins.begin()) : Ins.begin(),
                  Ins.end());
  }
 
  if (IsKernel)
    analyzeFormalArgumentsCompute(CCInfo, Ins);
 
  if (IsEntryFunc) {
    allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
    allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
    if (IsKernel && Subtarget->hasKernargPreload())
      allocatePreloadKernArgSGPRs(CCInfo, ArgLocs, Ins, MF, *TRI, *Info);
 
    allocateLDSKernelId(CCInfo, MF, *TRI, *Info);
  } else if (!IsGraphics) {
    // For the fixed ABI, pass workitem IDs in the last argument register.
    allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);
 
    // FIXME: Sink this into allocateSpecialInputSGPRs
    if (!Subtarget->hasFlatScratchEnabled())
      CCInfo.AllocateReg(Info->getScratchRSrcReg());
 
    allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
  }
 
  if (!IsKernel) {
    CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
    CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
 
    // This assumes the registers are allocated by CCInfo in ascending order
    // with no gaps.
    Info->setNumWaveDispatchSGPRs(
        CCInfo.getFirstUnallocated(AMDGPU::SGPR_32RegClass.getRegisters()));
    Info->setNumWaveDispatchVGPRs(
        CCInfo.getFirstUnallocated(AMDGPU::VGPR_32RegClass.getRegisters()));
  } else if (Info->getNumKernargPreloadedSGPRs()) {
    Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
  }
 
  SmallVector<SDValue, 16> Chains;
 
  if (IsWholeWaveFunc) {
    SDValue Setup = DAG.getNode(AMDGPUISD::WHOLE_WAVE_SETUP, DL,
                                {MVT::i1, MVT::Other}, Chain);
    InVals.push_back(Setup.getValue(0));
    Chains.push_back(Setup.getValue(1));
  }
 
  // FIXME: This is the minimum kernel argument alignment. We should improve
  // this to the maximum alignment of the arguments.
  //
  // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
  // kern arg offset.
  const Align KernelArgBaseAlign = Align(16);
 
  for (unsigned i = IsWholeWaveFunc ? 1 : 0, e = Ins.size(), ArgIdx = 0; i != e;
       ++i) {
    const ISD::InputArg &Arg = Ins[i];
    if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) {
      InVals.push_back(DAG.getPOISON(Arg.VT));
      continue;
    }
 
    CCValAssign &VA = ArgLocs[ArgIdx++];
    MVT VT = VA.getLocVT();
 
    if (IsEntryFunc && VA.isMemLoc()) {
      VT = Ins[i].VT;
      EVT MemVT = VA.getLocVT();
 
      const uint64_t Offset = VA.getLocMemOffset();
      Align Alignment = commonAlignment(KernelArgBaseAlign, Offset);
 
      if (Arg.Flags.isByRef()) {
        SDValue Ptr = lowerKernArgParameterPtr(DAG, DL, Chain, Offset);
 
        const GCNTargetMachine &TM =
            static_cast<const GCNTargetMachine &>(getTargetMachine());
        if (!TM.isNoopAddrSpaceCast(AMDGPUAS::CONSTANT_ADDRESS,
                                    Arg.Flags.getPointerAddrSpace())) {
          Ptr = DAG.getAddrSpaceCast(DL, VT, Ptr, AMDGPUAS::CONSTANT_ADDRESS,
                                     Arg.Flags.getPointerAddrSpace());
        }
 
        InVals.push_back(Ptr);
        continue;
      }
 
      SDValue NewArg;
      if (Arg.isOrigArg() && Info->getArgInfo().PreloadKernArgs.count(i)) {
        if (MemVT.getStoreSize() < 4 && Alignment < 4) {
          // In this case the argument is packed into the previous preload SGPR.
          int64_t AlignDownOffset = alignDown(Offset, 4);
          int64_t OffsetDiff = Offset - AlignDownOffset;
          EVT IntVT = MemVT.changeTypeToInteger();
 
          const SIMachineFunctionInfo *Info =
              MF.getInfo<SIMachineFunctionInfo>();
          MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
          Register Reg =
              Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs[0];
 
          assert(Reg);
          Register VReg = MRI.getLiveInVirtReg(Reg);
          SDValue Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
 
          SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, DL, MVT::i32);
          SDValue Extract = DAG.getNode(ISD::SRL, DL, MVT::i32, Copy, ShiftAmt);
 
          SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, DL, IntVT, Extract);
          ArgVal = DAG.getNode(ISD::BITCAST, DL, MemVT, ArgVal);
          NewArg = convertArgType(DAG, VT, MemVT, DL, ArgVal,
                                  Ins[i].Flags.isSExt(), &Ins[i]);
 
          NewArg = DAG.getMergeValues({NewArg, Copy.getValue(1)}, DL);
        } else {
          const SIMachineFunctionInfo *Info =
              MF.getInfo<SIMachineFunctionInfo>();
          MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
          const SmallVectorImpl<MCRegister> &PreloadRegs =
              Info->getArgInfo().PreloadKernArgs.find(i)->getSecond().Regs;
 
          SDValue Copy;
          if (PreloadRegs.size() == 1) {
            Register VReg = MRI.getLiveInVirtReg(PreloadRegs[0]);
            const TargetRegisterClass *RC = MRI.getRegClass(VReg);
            NewArg = DAG.getCopyFromReg(
                Chain, DL, VReg,
                EVT::getIntegerVT(*DAG.getContext(),
                                  TRI->getRegSizeInBits(*RC)));
 
          } else {
            // If the kernarg alignment does not match the alignment of the SGPR
            // tuple RC that can accommodate this argument, it will be built up
            // via copies from from the individual SGPRs that the argument was
            // preloaded to.
            SmallVector<SDValue, 4> Elts;
            for (auto Reg : PreloadRegs) {
              Register VReg = MRI.getLiveInVirtReg(Reg);
              Copy = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i32);
              Elts.push_back(Copy);
            }
            NewArg =
                DAG.getBuildVector(EVT::getVectorVT(*DAG.getContext(), MVT::i32,
                                                    PreloadRegs.size()),
                                   DL, Elts);
          }
 
          // If the argument was preloaded to multiple consecutive 32-bit
          // registers because of misalignment between addressable SGPR tuples
          // and the argument size, we can still assume that because of kernarg
          // segment alignment restrictions that NewArg's size is the same as
          // MemVT and just do a bitcast. If MemVT is less than 32-bits we add a
          // truncate since we cannot preload to less than a single SGPR and the
          // MemVT may be smaller.
          EVT MemVTInt =
              EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits());
          if (MemVT.bitsLT(NewArg.getSimpleValueType()))
            NewArg = DAG.getNode(ISD::TRUNCATE, DL, MemVTInt, NewArg);
 
          NewArg = DAG.getBitcast(MemVT, NewArg);
          NewArg = convertArgType(DAG, VT, MemVT, DL, NewArg,
                                  Ins[i].Flags.isSExt(), &Ins[i]);
          NewArg = DAG.getMergeValues({NewArg, Chain}, DL);
        }
      } else {
        // Hidden arguments that are in the kernel signature must be preloaded
        // to user SGPRs. Print a diagnostic error if a hidden argument is in
        // the argument list and is not preloaded.
        if (Arg.isOrigArg()) {
          Argument *OrigArg = Fn.getArg(Arg.getOrigArgIndex());
          if (OrigArg->hasAttribute("amdgpu-hidden-argument")) {
            DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
                *OrigArg->getParent(),
                "hidden argument in kernel signature was not preloaded",
                DL.getDebugLoc()));
          }
        }
 
        NewArg =
            lowerKernargMemParameter(DAG, VT, MemVT, DL, Chain, Offset,
                                     Alignment, Ins[i].Flags.isSExt(), &Ins[i]);
      }
      Chains.push_back(NewArg.getValue(1));
 
      auto *ParamTy =
          dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
      if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
          ParamTy &&
          (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
           ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
        // On SI local pointers are just offsets into LDS, so they are always
        // less than 16-bits.  On CI and newer they could potentially be
        // real pointers, so we can't guarantee their size.
        NewArg = DAG.getNode(ISD::AssertZext, DL, NewArg.getValueType(), NewArg,
                             DAG.getValueType(MVT::i16));
      }
 
      InVals.push_back(NewArg);
      continue;
    }
    if (!IsEntryFunc && VA.isMemLoc()) {
      SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
      InVals.push_back(Val);
      if (!Arg.Flags.isByVal())
        Chains.push_back(Val.getValue(1));
      continue;
    }
 
    assert(VA.isRegLoc() && "Parameter must be in a register!");
 
    Register Reg = VA.getLocReg();
    const TargetRegisterClass *RC = nullptr;
    if (AMDGPU::VGPR_32RegClass.contains(Reg))
      RC = &AMDGPU::VGPR_32RegClass;
    else if (AMDGPU::SGPR_32RegClass.contains(Reg))
      RC = &AMDGPU::SGPR_32RegClass;
    else
      llvm_unreachable("Unexpected register class in LowerFormalArguments!");
 
    Reg = MF.addLiveIn(Reg, RC);
    SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
    if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
      // FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
      // they will read physical regs before any side effect instructions.
      SDValue ReadFirstLane =
          DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
      Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Val.getValueType(),
                        ReadFirstLane, Val);
    }
 
    if (Arg.Flags.isSRet()) {
      // The return object should be reasonably addressable.
      Val = annotateStackObjectPointer(Val, DAG, DL,
                                       Arg.Flags.getNonZeroMemAlign());
    }
 
    Val = convertABITypeToValueType(DAG, Val, VA, DL);
    InVals.push_back(Val);
  }
 
  // Start adding system SGPRs.
  if (IsEntryFunc)
    allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsGraphics);
 
  unsigned StackArgSize = CCInfo.getStackSize();
  Info->setBytesInStackArgArea(StackArgSize);
 
  return Chains.empty() ? Chain
                        : DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
}
 
// TODO: If return values can't fit in registers, we should return as many as
// possible in registers before passing on stack.
bool SITargetLowering::CanLowerReturn(
    CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
    const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
    const Type *RetTy) const {
  // Replacing returns with sret/stack usage doesn't make sense for shaders.
  // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
  // for shaders. Vector types should be explicitly handled by CC.
  if (AMDGPU::isEntryFunctionCC(CallConv))
    return true;
 
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
  if (!CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg)))
    return false;
 
  // We must use the stack if return would require unavailable registers.
  unsigned MaxNumVGPRs = Subtarget->getMaxNumVGPRs(MF);
  unsigned TotalNumVGPRs = Subtarget->getAddressableNumArchVGPRs();
  for (unsigned i = MaxNumVGPRs; i < TotalNumVGPRs; ++i)
    if (CCInfo.isAllocated(AMDGPU::VGPR_32RegClass.getRegister(i)))
      return false;
 
  return true;
}
 
SDValue
SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                              bool isVarArg,
                              const SmallVectorImpl<ISD::OutputArg> &Outs,
                              const SmallVectorImpl<SDValue> &OutVals,
                              const SDLoc &DL, SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
 
  if (AMDGPU::isKernel(CallConv)) {
    return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
                                             OutVals, DL, DAG);
  }
 
  bool IsShader = AMDGPU::isShader(CallConv);
 
  Info->setIfReturnsVoid(Outs.empty());
  bool IsWaveEnd = Info->returnsVoid() && IsShader;
 
  // CCValAssign - represent the assignment of the return value to a location.
  SmallVector<CCValAssign, 48> RVLocs;
 
  // CCState - Info about the registers and stack slots.
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
 
  // Analyze outgoing return values.
  CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
 
  SDValue Glue;
  SmallVector<SDValue, 48> RetOps;
  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
 
  SDValue ReadFirstLane =
      DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
  // Copy the result values into the output registers.
  for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
       ++I, ++RealRVLocIdx) {
    CCValAssign &VA = RVLocs[I];
    assert(VA.isRegLoc() && "Can only return in registers!");
    // TODO: Partially return in registers if return values don't fit.
    SDValue Arg = OutVals[RealRVLocIdx];
 
    // Copied from other backends.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
    if (TRI->isSGPRPhysReg(VA.getLocReg()))
      Arg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Arg.getValueType(),
                        ReadFirstLane, Arg);
    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Glue);
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
  }
 
  // FIXME: Does sret work properly?
  if (!Info->isEntryFunction()) {
    const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
    const MCPhysReg *I =
        TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
    if (I) {
      for (; *I; ++I) {
        if (AMDGPU::SReg_64RegClass.contains(*I))
          RetOps.push_back(DAG.getRegister(*I, MVT::i64));
        else if (AMDGPU::SReg_32RegClass.contains(*I))
          RetOps.push_back(DAG.getRegister(*I, MVT::i32));
        else
          llvm_unreachable("Unexpected register class in CSRsViaCopy!");
      }
    }
  }
 
  // Update chain and glue.
  RetOps[0] = Chain;
  if (Glue.getNode())
    RetOps.push_back(Glue);
 
  unsigned Opc = AMDGPUISD::ENDPGM;
  if (!IsWaveEnd)
    Opc = Info->isWholeWaveFunction() ? AMDGPUISD::WHOLE_WAVE_RETURN
          : IsShader                  ? AMDGPUISD::RETURN_TO_EPILOG
                                      : AMDGPUISD::RET_GLUE;
  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
}
 
SDValue SITargetLowering::LowerCallResult(
    SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool IsVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
    SDValue ThisVal) const {
  CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg);
 
  // Assign locations to each value returned by this call.
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
  CCInfo.AnalyzeCallResult(Ins, RetCC);
 
  // Copy all of the result registers out of their specified physreg.
  for (CCValAssign VA : RVLocs) {
    SDValue Val;
 
    if (VA.isRegLoc()) {
      Val =
          DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InGlue);
      Chain = Val.getValue(1);
      InGlue = Val.getValue(2);
    } else if (VA.isMemLoc()) {
      report_fatal_error("TODO: return values in memory");
    } else
      llvm_unreachable("unknown argument location type");
 
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::ZExt:
      Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::SExt:
      Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    case CCValAssign::AExt:
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    InVals.push_back(Val);
  }
 
  return Chain;
}
 
// Add code to pass special inputs required depending on used features separate
// from the explicit user arguments present in the IR.
void SITargetLowering::passSpecialInputs(
    CallLoweringInfo &CLI, CCState &CCInfo, const SIMachineFunctionInfo &Info,
    SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
    SmallVectorImpl<SDValue> &MemOpChains, SDValue Chain) const {
  // If we don't have a call site, this was a call inserted by
  // legalization. These can never use special inputs.
  if (!CLI.CB)
    return;
 
  SelectionDAG &DAG = CLI.DAG;
  const SDLoc &DL = CLI.DL;
  const Function &F = DAG.getMachineFunction().getFunction();
 
  const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
  const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
 
  const AMDGPUFunctionArgInfo &CalleeArgInfo =
      AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
 
  // TODO: Unify with private memory register handling. This is complicated by
  // the fact that at least in kernels, the input argument is not necessarily
  // in the same location as the input.
  // clang-format off
  static constexpr std::pair<AMDGPUFunctionArgInfo::PreloadedValue,
      std::array<StringLiteral, 2>> ImplicitAttrs[] = {
    {AMDGPUFunctionArgInfo::DISPATCH_PTR, {"amdgpu-no-dispatch-ptr", ""}},
    {AMDGPUFunctionArgInfo::QUEUE_PTR, {"amdgpu-no-queue-ptr", ""}},
    {AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, {"amdgpu-no-implicitarg-ptr", ""}},
    {AMDGPUFunctionArgInfo::DISPATCH_ID, {"amdgpu-no-dispatch-id", ""}},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_X, {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"}},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"}},
    {AMDGPUFunctionArgInfo::WORKGROUP_ID_Z, {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"}},
    {AMDGPUFunctionArgInfo::LDS_KERNEL_ID, {"amdgpu-no-lds-kernel-id", ""}},
  };
  // clang-format on
 
  for (auto [InputID, Attrs] : ImplicitAttrs) {
    // If the callee does not use the attribute value, skip copying the value.
    if (all_of(Attrs, [&](StringRef Attr) {
          return Attr.empty() || CLI.CB->hasFnAttr(Attr);
        }))
      continue;
 
    const auto [OutgoingArg, ArgRC, ArgTy] =
        CalleeArgInfo.getPreloadedValue(InputID);
    if (!OutgoingArg)
      continue;
 
    const auto [IncomingArg, IncomingArgRC, Ty] =
        CallerArgInfo.getPreloadedValue(InputID);
    assert(IncomingArgRC == ArgRC);
 
    // All special arguments are ints for now.
    EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? MVT::i64 : MVT::i32;
    SDValue InputReg;
 
    if (IncomingArg) {
      InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg);
    } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {
      // The implicit arg ptr is special because it doesn't have a corresponding
      // input for kernels, and is computed from the kernarg segment pointer.
      InputReg = getImplicitArgPtr(DAG, DL);
    } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {
      std::optional<uint32_t> Id =
          AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
      if (Id.has_value()) {
        InputReg = DAG.getConstant(*Id, DL, ArgVT);
      } else {
        InputReg = DAG.getPOISON(ArgVT);
      }
    } else {
      // We may have proven the input wasn't needed, although the ABI is
      // requiring it. We just need to allocate the register appropriately.
      InputReg = DAG.getPOISON(ArgVT);
    }
 
    if (OutgoingArg->isRegister()) {
      RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
      if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))
        report_fatal_error("failed to allocate implicit input argument");
    } else {
      unsigned SpecialArgOffset =
          CCInfo.AllocateStack(ArgVT.getStoreSize(), Align(4));
      SDValue ArgStore =
          storeStackInputValue(DAG, DL, Chain, InputReg, SpecialArgOffset);
      MemOpChains.push_back(ArgStore);
    }
  }
 
  // Pack workitem IDs into a single register or pass it as is if already
  // packed.
 
  auto [OutgoingArg, ArgRC, Ty] =
      CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
  if (!OutgoingArg)
    std::tie(OutgoingArg, ArgRC, Ty) =
        CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
  if (!OutgoingArg)
    std::tie(OutgoingArg, ArgRC, Ty) =
        CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
  if (!OutgoingArg)
    return;
 
  const ArgDescriptor *IncomingArgX = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X));
  const ArgDescriptor *IncomingArgY = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y));
  const ArgDescriptor *IncomingArgZ = std::get<0>(
      CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z));
 
  SDValue InputReg;
  SDLoc SL;
 
  const bool NeedWorkItemIDX = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-x");
  const bool NeedWorkItemIDY = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-y");
  const bool NeedWorkItemIDZ = !CLI.CB->hasFnAttr("amdgpu-no-workitem-id-z");
 
  // If incoming ids are not packed we need to pack them.
  if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo.WorkItemIDX &&
      NeedWorkItemIDX) {
    if (Subtarget->getMaxWorkitemID(F, 0) != 0) {
      InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
    } else {
      InputReg = DAG.getConstant(0, DL, MVT::i32);
    }
  }
 
  if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo.WorkItemIDY &&
      NeedWorkItemIDY && Subtarget->getMaxWorkitemID(F, 1) != 0) {
    SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
    Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
                    DAG.getShiftAmountConstant(10, MVT::i32, SL));
    InputReg = InputReg.getNode()
                   ? DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y)
                   : Y;
  }
 
  if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo.WorkItemIDZ &&
      NeedWorkItemIDZ && Subtarget->getMaxWorkitemID(F, 2) != 0) {
    SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
    Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
                    DAG.getShiftAmountConstant(20, MVT::i32, SL));
    InputReg = InputReg.getNode()
                   ? DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z)
                   : Z;
  }
 
  if (!InputReg && (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {
    if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {
      // We're in a situation where the outgoing function requires the workitem
      // ID, but the calling function does not have it (e.g a graphics function
      // calling a C calling convention function). This is illegal, but we need
      // to produce something.
      InputReg = DAG.getPOISON(MVT::i32);
    } else {
      // Workitem ids are already packed, any of present incoming arguments
      // will carry all required fields.
      ArgDescriptor IncomingArg =
          ArgDescriptor::createArg(IncomingArgX   ? *IncomingArgX
                                   : IncomingArgY ? *IncomingArgY
                                                  : *IncomingArgZ,
                                   ~0u);
      InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
    }
  }
 
  if (OutgoingArg->isRegister()) {
    if (InputReg)
      RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
 
    CCInfo.AllocateReg(OutgoingArg->getRegister());
  } else {
    unsigned SpecialArgOffset = CCInfo.AllocateStack(4, Align(4));
    if (InputReg) {
      SDValue ArgStore =
          storeStackInputValue(DAG, DL, Chain, InputReg, SpecialArgOffset);
      MemOpChains.push_back(ArgStore);
    }
  }
}
 
bool SITargetLowering::isEligibleForTailCallOptimization(
    SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<SDValue> &OutVals,
    const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
  if (AMDGPU::isChainCC(CalleeCC))
    return true;
 
  if (!AMDGPU::mayTailCallThisCC(CalleeCC))
    return false;
 
  // For a divergent call target, we need to do a waterfall loop over the
  // possible callees which precludes us from using a simple jump.
  if (Callee->isDivergent())
    return false;
 
  MachineFunction &MF = DAG.getMachineFunction();
  const Function &CallerF = MF.getFunction();
  CallingConv::ID CallerCC = CallerF.getCallingConv();
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
 
  // Kernels aren't callable, and don't have a live in return address so it
  // doesn't make sense to do a tail call with entry functions.
  if (!CallerPreserved)
    return false;
 
  bool CCMatch = CallerCC == CalleeCC;
 
  if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
    if (AMDGPU::canGuaranteeTCO(CalleeCC) && CCMatch)
      return true;
    return false;
  }
 
  // TODO: Can we handle var args?
  if (IsVarArg)
    return false;
 
  for (const Argument &Arg : CallerF.args()) {
    if (Arg.hasByValAttr())
      return false;
  }
 
  LLVMContext &Ctx = *DAG.getContext();
 
  // Check that the call results are passed in the same way.
  if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins,
                                  CCAssignFnForCall(CalleeCC, IsVarArg),
                                  CCAssignFnForCall(CallerCC, IsVarArg)))
    return false;
 
  // The callee has to preserve all registers the caller needs to preserve.
  if (!CCMatch) {
    const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
      return false;
  }
 
  // Nothing more to check if the callee is taking no arguments.
  if (Outs.empty())
    return true;
 
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
 
  // FIXME: We are not allocating special input registers, so we will be
  // deciding based on incorrect register assignments.
  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
 
  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
  // If the stack arguments for this call do not fit into our own save area then
  // the call cannot be made tail.
  // TODO: Is this really necessary?
  if (CCInfo.getStackSize() > FuncInfo->getBytesInStackArgArea())
    return false;
 
  for (const auto &[CCVA, ArgVal] : zip_equal(ArgLocs, OutVals)) {
    // FIXME: What about inreg arguments that end up passed in memory?
    if (!CCVA.isRegLoc())
      continue;
 
    // If we are passing an argument in an SGPR, and the value is divergent,
    // this call requires a waterfall loop.
    if (ArgVal->isDivergent() && TRI->isSGPRPhysReg(CCVA.getLocReg())) {
      LLVM_DEBUG(
          dbgs() << "Cannot tail call due to divergent outgoing argument in "
                 << printReg(CCVA.getLocReg(), TRI) << '\n');
      return false;
    }
  }
 
  const MachineRegisterInfo &MRI = MF.getRegInfo();
  return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
}
 
bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
  if (!CI->isTailCall())
    return false;
 
  const Function *ParentFn = CI->getFunction();
  if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv()))
    return false;
  return true;
}
 
namespace {
// Chain calls have special arguments that we need to handle. These are
// tagging along at the end of the arguments list(s), after the SGPR and VGPR
// arguments (index 0 and 1 respectively).
enum ChainCallArgIdx {
  Exec = 2,
  Flags,
  NumVGPRs,
  FallbackExec,
  FallbackCallee
};
} // anonymous namespace
 
// The wave scratch offset register is used as the global base pointer.
SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
                                    SmallVectorImpl<SDValue> &InVals) const {
  CallingConv::ID CallConv = CLI.CallConv;
  bool IsChainCallConv = AMDGPU::isChainCC(CallConv);
 
  SelectionDAG &DAG = CLI.DAG;
 
  const SDLoc &DL = CLI.DL;
  SDValue Chain = CLI.Chain;
  SDValue Callee = CLI.Callee;
 
  llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs;
  bool UsesDynamicVGPRs = false;
  if (IsChainCallConv) {
    // The last arguments should be the value that we need to put in EXEC,
    // followed by the flags and any other arguments with special meanings.
    // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so
    // we don't treat them like the "real" arguments.
    auto RequestedExecIt =
        llvm::find_if(CLI.Outs, [](const ISD::OutputArg &Arg) {
          return Arg.OrigArgIndex == 2;
        });
    assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC");
 
    size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin();
    CLI.OutVals.erase(CLI.OutVals.begin() + SpecialArgsBeginIdx,
                      CLI.OutVals.end());
    CLI.Outs.erase(RequestedExecIt, CLI.Outs.end());
 
    assert(CLI.Outs.back().OrigArgIndex < 2 &&
           "Haven't popped all the special args");
 
    TargetLowering::ArgListEntry RequestedExecArg =
        CLI.Args[ChainCallArgIdx::Exec];
    if (!RequestedExecArg.Ty->isIntegerTy(Subtarget->getWavefrontSize()))
      return lowerUnhandledCall(CLI, InVals, "Invalid value for EXEC");
 
    // Convert constants into TargetConstants, so they become immediate operands
    // instead of being selected into S_MOV.
    auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) {
      if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Arg.Node)) {
        ChainCallSpecialArgs.push_back(DAG.getTargetConstant(
            ArgNode->getAPIntValue(), DL, ArgNode->getValueType(0)));
      } else
        ChainCallSpecialArgs.push_back(Arg.Node);
    };
 
    PushNodeOrTargetConstant(RequestedExecArg);
 
    // Process any other special arguments depending on the value of the flags.
    TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags];
 
    const APInt &FlagsValue = cast<ConstantSDNode>(Flags.Node)->getAPIntValue();
    if (FlagsValue.isZero()) {
      if (CLI.Args.size() > ChainCallArgIdx::Flags + 1)
        return lowerUnhandledCall(CLI, InVals,
                                  "no additional args allowed if flags == 0");
    } else if (FlagsValue.isOneBitSet(0)) {
      if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) {
        return lowerUnhandledCall(CLI, InVals, "expected 3 additional args");
      }
 
      if (!Subtarget->isWave32()) {
        return lowerUnhandledCall(
            CLI, InVals, "dynamic VGPR mode is only supported for wave32");
      }
 
      UsesDynamicVGPRs = true;
      std::for_each(CLI.Args.begin() + ChainCallArgIdx::NumVGPRs,
                    CLI.Args.end(), PushNodeOrTargetConstant);
    }
  }
 
  SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
  SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
  SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
  bool &IsTailCall = CLI.IsTailCall;
  bool IsVarArg = CLI.IsVarArg;
  bool IsSibCall = false;
  MachineFunction &MF = DAG.getMachineFunction();
 
  if (Callee.isUndef() || isNullConstant(Callee)) {
    if (!CLI.IsTailCall) {
      for (ISD::InputArg &Arg : CLI.Ins)
        InVals.push_back(DAG.getPOISON(Arg.VT));
    }
 
    return Chain;
  }
 
  if (IsVarArg) {
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported call to variadic function ");
  }
 
  if (!CLI.CB)
    return lowerUnhandledCall(CLI, InVals, "unsupported libcall legalization");
 
  if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
    return lowerUnhandledCall(CLI, InVals,
                              "unsupported required tail call to function ");
  }
 
  if (IsTailCall) {
    IsTailCall = isEligibleForTailCallOptimization(Callee, CallConv, IsVarArg,
                                                   Outs, OutVals, Ins, DAG);
    if (!IsTailCall &&
        ((CLI.CB && CLI.CB->isMustTailCall()) || IsChainCallConv)) {
      report_fatal_error("failed to perform tail call elimination on a call "
                         "site marked musttail or on llvm.amdgcn.cs.chain");
    }
 
    bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
 
    // A sibling call is one where we're under the usual C ABI and not planning
    // to change that but can still do a tail call:
    if (!TailCallOpt && IsTailCall)
      IsSibCall = true;
 
    if (IsTailCall)
      ++NumTailCalls;
  }
 
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
  SmallVector<SDValue, 8> MemOpChains;
 
  // Analyze operands of the call, assigning locations to each operand.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
  CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
 
  if (CallConv != CallingConv::AMDGPU_Gfx && !AMDGPU::isChainCC(CallConv) &&
      CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {
    // With a fixed ABI, allocate fixed registers before user arguments.
    passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
  }
 
  // Mark the scratch resource descriptor as allocated so the CC analysis
  // does not assign user arguments to these registers, matching the callee.
  if (!Subtarget->hasFlatScratchEnabled())
    CCInfo.AllocateReg(Info->getScratchRSrcReg());
 
  CCInfo.AnalyzeCallOperands(Outs, AssignFn);
 
  // Get a count of how many bytes are to be pushed on the stack.
  unsigned NumBytes = CCInfo.getStackSize();
 
  if (IsSibCall) {
    // Since we're not changing the ABI to make this a tail call, the memory
    // operands are already available in the caller's incoming argument space.
    NumBytes = 0;
  }
 
  // FPDiff is the byte offset of the call's argument area from the callee's.
  // Stores to callee stack arguments will be placed in FixedStackSlots offset
  // by this amount for a tail call. In a sibling call it must be 0 because the
  // caller will deallocate the entire stack and the callee still expects its
  // arguments to begin at SP+0. Completely unused for non-tail calls.
  int32_t FPDiff = 0;
  MachineFrameInfo &MFI = MF.getFrameInfo();
  auto *TRI = Subtarget->getRegisterInfo();
 
  // Adjust the stack pointer for the new arguments...
  // These operations are automatically eliminated by the prolog/epilog pass
  if (!IsSibCall)
    Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
 
  if (!IsSibCall || IsChainCallConv) {
    if (!Subtarget->hasFlatScratchEnabled()) {
      SmallVector<SDValue, 4> CopyFromChains;
 
      // In the HSA case, this should be an identity copy.
      SDValue ScratchRSrcReg =
          DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
      RegsToPass.emplace_back(IsChainCallConv
                                  ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
                                  : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3,
                              ScratchRSrcReg);
      CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
      Chain = DAG.getTokenFactor(DL, CopyFromChains);
    }
  }
 
  const unsigned NumSpecialInputs = RegsToPass.size();
 
  MVT PtrVT = MVT::i32;
 
  // Walk the register/memloc assignments, inserting copies/loads.
  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
    CCValAssign &VA = ArgLocs[i];
    SDValue Arg = OutVals[i];
 
    // Promote the value if needed.
    switch (VA.getLocInfo()) {
    case CCValAssign::Full:
      break;
    case CCValAssign::BCvt:
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    case CCValAssign::FPExt:
      Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
      break;
    default:
      llvm_unreachable("Unknown loc info!");
    }
 
    if (VA.isRegLoc()) {
      RegsToPass.push_back(std::pair(VA.getLocReg(), Arg));
    } else {
      assert(VA.isMemLoc());
 
      SDValue DstAddr;
      MachinePointerInfo DstInfo;
 
      unsigned LocMemOffset = VA.getLocMemOffset();
      int32_t Offset = LocMemOffset;
 
      SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
      MaybeAlign Alignment;
 
      if (IsTailCall) {
        ISD::ArgFlagsTy Flags = Outs[i].Flags;
        unsigned OpSize = Flags.isByVal() ? Flags.getByValSize()
                                          : VA.getValVT().getStoreSize();
 
        // FIXME: We can have better than the minimum byval required alignment.
        Alignment =
            Flags.isByVal()
                ? Flags.getNonZeroByValAlign()
                : commonAlignment(Subtarget->getStackAlignment(), Offset);
 
        Offset = Offset + FPDiff;
        int FI = MFI.CreateFixedObject(OpSize, Offset, true);
 
        DstAddr = DAG.getFrameIndex(FI, PtrVT);
        DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
 
        // Make sure any stack arguments overlapping with where we're storing
        // are loaded before this eventual operation. Otherwise they'll be
        // clobbered.
 
        // FIXME: Why is this really necessary? This seems to just result in a
        // lot of code to copy the stack and write them back to the same
        // locations, which are supposed to be immutable?
        Chain = addTokenForArgument(Chain, DAG, MFI, FI);
      } else {
        // Stores to the argument stack area are relative to the stack pointer.
        SDValue SP = DAG.getCopyFromReg(Chain, DL, Info->getStackPtrOffsetReg(),
                                        MVT::i32);
        DstAddr = DAG.getNode(ISD::ADD, DL, MVT::i32, SP, PtrOff);
        DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
        Alignment =
            commonAlignment(Subtarget->getStackAlignment(), LocMemOffset);
      }
 
      if (Outs[i].Flags.isByVal()) {
        SDValue SizeNode =
            DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);
        SDValue Cpy =
            DAG.getMemcpy(Chain, DL, DstAddr, Arg, SizeNode,
                          Outs[i].Flags.getNonZeroByValAlign(),
                          Outs[i].Flags.getNonZeroByValAlign(),
                          /*isVol = */ false, /*AlwaysInline = */ true,
                          /*CI=*/nullptr, std::nullopt, DstInfo,
                          MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS));
 
        MemOpChains.push_back(Cpy);
      } else {
        SDValue Store =
            DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Alignment);
        MemOpChains.push_back(Store);
      }
    }
  }
 
  if (!MemOpChains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
 
  SDValue ReadFirstLaneID =
      DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
 
  SDValue TokenGlue;
  if (CLI.ConvergenceControlToken) {
    TokenGlue = DAG.getNode(ISD::CONVERGENCECTRL_GLUE, DL, MVT::Glue,
                            CLI.ConvergenceControlToken);
  }
 
  // Build a sequence of copy-to-reg nodes chained together with token chain
  // and flag operands which copy the outgoing args into the appropriate regs.
  SDValue InGlue;
 
  unsigned ArgIdx = 0;
  for (auto [Reg, Val] : RegsToPass) {
    if (ArgIdx++ >= NumSpecialInputs &&
        (IsChainCallConv || !Val->isDivergent()) && TRI->isSGPRPhysReg(Reg)) {
      // For chain calls, the inreg arguments are required to be
      // uniform. Speculatively Insert a readfirstlane in case we cannot prove
      // they are uniform.
      //
      // For other calls, if an inreg arguments is known to be uniform,
      // speculatively insert a readfirstlane in case it is in a VGPR.
      //
      // FIXME: We need to execute this in a waterfall loop if it is a divergent
      // value, so let that continue to produce invalid code.
 
      SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Val});
      if (TokenGlue)
        ReadfirstlaneArgs.push_back(TokenGlue);
      Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Val.getValueType(),
                        ReadfirstlaneArgs);
    }
 
    Chain = DAG.getCopyToReg(Chain, DL, Reg, Val, InGlue);
    InGlue = Chain.getValue(1);
  }
 
  // We don't usually want to end the call-sequence here because we would tidy
  // the frame up *after* the call, however in the ABI-changing tail-call case
  // we've carefully laid out the parameters so that when sp is reset they'll be
  // in the correct location.
  if (IsTailCall && !IsSibCall) {
    Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InGlue, DL);
    InGlue = Chain.getValue(1);
  }
 
  std::vector<SDValue> Ops({Chain});
 
  // Add a redundant copy of the callee global which will not be legalized, as
  // we need direct access to the callee later.
  if (GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(Callee)) {
    const GlobalValue *GV = GSD->getGlobal();
    Ops.push_back(Callee);
    Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
  } else {
    if (IsTailCall) {
      // isEligibleForTailCallOptimization considered whether the call target is
      // divergent, but we may still end up with a uniform value in a VGPR.
      // Insert a readfirstlane just in case.
      SDValue ReadFirstLaneID =
          DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32);
 
      SmallVector<SDValue, 3> ReadfirstlaneArgs({ReadFirstLaneID, Callee});
      if (TokenGlue)
        ReadfirstlaneArgs.push_back(TokenGlue); // Wire up convergence token.
      Callee = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Callee.getValueType(),
                           ReadfirstlaneArgs);
    }
 
    Ops.push_back(Callee);
    Ops.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
  }
 
  if (IsTailCall) {
    // Each tail call may have to adjust the stack by a different amount, so
    // this information must travel along with the operation for eventual
    // consumption by emitEpilogue.
    Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
  }
 
  if (IsChainCallConv)
    llvm::append_range(Ops, ChainCallSpecialArgs);
 
  // Add argument registers to the end of the list so that they are known live
  // into the call.
  for (auto &[Reg, Val] : RegsToPass)
    Ops.push_back(DAG.getRegister(Reg, Val.getValueType()));
 
  // Add a register mask operand representing the call-preserved registers.
  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
  assert(Mask && "Missing call preserved mask for calling convention");
  Ops.push_back(DAG.getRegisterMask(Mask));
 
  if (SDValue Token = CLI.ConvergenceControlToken) {
    SmallVector<SDValue, 2> GlueOps;
    GlueOps.push_back(Token);
    if (InGlue)
      GlueOps.push_back(InGlue);
 
    InGlue = SDValue(DAG.getMachineNode(TargetOpcode::CONVERGENCECTRL_GLUE, DL,
                                        MVT::Glue, GlueOps),
                     0);
  }
 
  if (InGlue)
    Ops.push_back(InGlue);
 
  // If we're doing a tall call, use a TC_RETURN here rather than an
  // actual call instruction.
  if (IsTailCall) {
    MFI.setHasTailCall();
    unsigned OPC = AMDGPUISD::TC_RETURN;
    switch (CallConv) {
    case CallingConv::AMDGPU_Gfx:
      OPC = AMDGPUISD::TC_RETURN_GFX;
      break;
    case CallingConv::AMDGPU_CS_Chain:
    case CallingConv::AMDGPU_CS_ChainPreserve:
      OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR
                             : AMDGPUISD::TC_RETURN_CHAIN;
      break;
    }
 
    // If the caller is a whole wave function, we need to use a special opcode
    // so we can patch up EXEC.
    if (Info->isWholeWaveFunction())
      OPC = AMDGPUISD::TC_RETURN_GFX_WholeWave;
 
    return DAG.getNode(OPC, DL, MVT::Other, Ops);
  }
 
  // Returns a chain and a flag for retval copy to use.
  SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, {MVT::Other, MVT::Glue}, Ops);
  Chain = Call.getValue(0);
  InGlue = Call.getValue(1);
 
  uint64_t CalleePopBytes = NumBytes;
  Chain = DAG.getCALLSEQ_END(Chain, 0, CalleePopBytes, InGlue, DL);
  if (!Ins.empty())
    InGlue = Chain.getValue(1);
 
  // Handle result values, copying them out of physregs into vregs that we
  // return.
  return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, DL, DAG,
                         InVals, /*IsThisReturn=*/false, SDValue());
}
 
// This is similar to the default implementation in ExpandDYNAMIC_STACKALLOC,
// except for:
// 1. Stack growth direction(default: downwards, AMDGPU: upwards), and
// 2. Scale size where, scale = wave-reduction(alloca-size) * wave-size
SDValue SITargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                                  SelectionDAG &DAG) const {
  const MachineFunction &MF = DAG.getMachineFunction();
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
 
  SDLoc dl(Op);
  EVT VT = Op.getValueType();
  SDValue Chain = Op.getOperand(0);
  Register SPReg = Info->getStackPtrOffsetReg();
 
  // Chain the dynamic stack allocation so that it doesn't modify the stack
  // pointer when other instructions are using the stack.
  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
 
  SDValue Size = Op.getOperand(1);
  SDValue BaseAddr = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
  Align Alignment = cast<ConstantSDNode>(Op.getOperand(2))->getAlignValue();
 
  const TargetFrameLowering *TFL = Subtarget->getFrameLowering();
  assert(TFL->getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp &&
         "Stack grows upwards for AMDGPU");
 
  Chain = BaseAddr.getValue(1);
  // When using flat-scratch, the stack offset is unscaled.
  const bool HasFlatScratch = Subtarget->hasFlatScratchEnabled();
  const unsigned WavefrontSizeLog2 = Subtarget->getWavefrontSizeLog2();
 
  Align StackAlign = TFL->getStackAlign();
  if (Alignment > StackAlign) {
    uint64_t ScaledAlignment = Alignment.value()
                               << (HasFlatScratch ? 0 : WavefrontSizeLog2);
    uint64_t StackAlignMask = ScaledAlignment - 1;
    SDValue TmpAddr = DAG.getNode(ISD::ADD, dl, VT, BaseAddr,
                                  DAG.getConstant(StackAlignMask, dl, VT));
    BaseAddr = DAG.getNode(ISD::AND, dl, VT, TmpAddr,
                           DAG.getSignedConstant(-ScaledAlignment, dl, VT));
  }
 
  assert(Size.getValueType() == MVT::i32 && "Size must be 32-bit");
  SDValue NewSP;
  if (isa<ConstantSDNode>(Size)) {
    // Increase the stack pointer by the size of the alloca.
    // If not using flat-scratch, we have to scale the size by the wave-size.
    SDValue ScaledSize =
        HasFlatScratch
            ? Size
            : DAG.getNode(ISD::SHL, dl, VT, Size,
                          DAG.getConstant(WavefrontSizeLog2, dl, MVT::i32));
    NewSP = DAG.getNode(ISD::ADD, dl, VT, BaseAddr, ScaledSize); // Value
  } else {
    // For dynamic sized alloca, perform wave-wide reduction to get max of
    // alloca size(divergent), and then scale it (when not using flat-scratch)
    // by wave-size.
    SDValue WaveReduction =
        DAG.getTargetConstant(Intrinsic::amdgcn_wave_reduce_umax, dl, MVT::i32);
    Size = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32, WaveReduction,
                       Size, DAG.getTargetConstant(0, dl, MVT::i32));
    SDValue ScaledSize = Size;
    if (!HasFlatScratch) {
      ScaledSize =
          DAG.getNode(ISD::SHL, dl, VT, Size,
                      DAG.getConstant(WavefrontSizeLog2, dl, MVT::i32));
    }
    NewSP =
        DAG.getNode(ISD::ADD, dl, VT, BaseAddr, ScaledSize); // Value in vgpr.
    SDValue ReadFirstLaneID =
        DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, dl, MVT::i32);
    NewSP = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32, ReadFirstLaneID,
                        NewSP);
  }
 
  Chain = DAG.getCopyToReg(Chain, dl, SPReg, NewSP); // Output chain
  SDValue CallSeqEnd = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
 
  return DAG.getMergeValues({BaseAddr, CallSeqEnd}, dl);
}
 
SDValue SITargetLowering::LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const {
  if (Op.getValueType() != MVT::i32)
    return Op; // Defer to cannot select error.
 
  Register SP = getStackPointerRegisterToSaveRestore();
  SDLoc SL(Op);
 
  SDValue CopyFromSP = DAG.getCopyFromReg(Op->getOperand(0), SL, SP, MVT::i32);
 
  // Convert from wave uniform to swizzled vector address. This should protect
  // from any edge cases where the stacksave result isn't directly used with
  // stackrestore.
  SDValue VectorAddress =
      DAG.getNode(AMDGPUISD::WAVE_ADDRESS, SL, MVT::i32, CopyFromSP);
  return DAG.getMergeValues({VectorAddress, CopyFromSP.getValue(1)}, SL);
}
 
SDValue SITargetLowering::lowerGET_ROUNDING(SDValue Op,
                                            SelectionDAG &DAG) const {
  SDLoc SL(Op);
  assert(Op.getValueType() == MVT::i32);
 
  uint32_t BothRoundHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 4);
  SDValue GetRoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
 
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
  SDValue GetReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, Op->getVTList(),
                               Op.getOperand(0), IntrinID, GetRoundBothImm);
 
  // There are two rounding modes, one for f32 and one for f64/f16. We only
  // report in the standard value range if both are the same.
  //
  // The raw values also differ from the expected FLT_ROUNDS values. Nearest
  // ties away from zero is not supported, and the other values are rotated by
  // 1.
  //
  // If the two rounding modes are not the same, report a target defined value.
 
  // Mode register rounding mode fields:
  //
  // [1:0] Single-precision round mode.
  // [3:2] Double/Half-precision round mode.
  //
  // 0=nearest even; 1= +infinity; 2= -infinity, 3= toward zero.
  //
  //             Hardware   Spec
  // Toward-0        3        0
  // Nearest Even    0        1
  // +Inf            1        2
  // -Inf            2        3
  //  NearestAway0  N/A       4
  //
  // We have to handle 16 permutations of a 4-bit value, so we create a 64-bit
  // table we can index by the raw hardware mode.
  //
  // (trunc (FltRoundConversionTable >> MODE.fp_round)) & 0xf
 
  SDValue BitTable =
      DAG.getConstant(AMDGPU::FltRoundConversionTable, SL, MVT::i64);
 
  SDValue Two = DAG.getConstant(2, SL, MVT::i32);
  SDValue RoundModeTimesNumBits =
      DAG.getNode(ISD::SHL, SL, MVT::i32, GetReg, Two);
 
  // TODO: We could possibly avoid a 64-bit shift and use a simpler table if we
  // knew only one mode was demanded.
  SDValue TableValue =
      DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
  SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
 
  SDValue EntryMask = DAG.getConstant(0xf, SL, MVT::i32);
  SDValue TableEntry =
      DAG.getNode(ISD::AND, SL, MVT::i32, TruncTable, EntryMask);
 
  // There's a gap in the 4-bit encoded table and actual enum values, so offset
  // if it's an extended value.
  SDValue Four = DAG.getConstant(4, SL, MVT::i32);
  SDValue IsStandardValue =
      DAG.getSetCC(SL, MVT::i1, TableEntry, Four, ISD::SETULT);
  SDValue EnumOffset = DAG.getNode(ISD::ADD, SL, MVT::i32, TableEntry, Four);
  SDValue Result = DAG.getNode(ISD::SELECT, SL, MVT::i32, IsStandardValue,
                               TableEntry, EnumOffset);
 
  return DAG.getMergeValues({Result, GetReg.getValue(1)}, SL);
}
 
SDValue SITargetLowering::lowerSET_ROUNDING(SDValue Op,
                                            SelectionDAG &DAG) const {
  SDLoc SL(Op);
 
  SDValue NewMode = Op.getOperand(1);
  assert(NewMode.getValueType() == MVT::i32);
 
  // Index a table of 4-bit entries mapping from the C FLT_ROUNDS values to the
  // hardware MODE.fp_round values.
  if (auto *ConstMode = dyn_cast<ConstantSDNode>(NewMode)) {
    uint32_t ClampedVal = std::min(
        static_cast<uint32_t>(ConstMode->getZExtValue()),
        static_cast<uint32_t>(AMDGPU::TowardZeroF32_TowardNegativeF64));
    NewMode = DAG.getConstant(
        AMDGPU::decodeFltRoundToHWConversionTable(ClampedVal), SL, MVT::i32);
  } else {
    // If we know the input can only be one of the supported standard modes in
    // the range 0-3, we can use a simplified mapping to hardware values.
    KnownBits KB = DAG.computeKnownBits(NewMode);
    const bool UseReducedTable = KB.countMinLeadingZeros() >= 30;
    // The supported standard values are 0-3. The extended values start at 8. We
    // need to offset by 4 if the value is in the extended range.
 
    if (UseReducedTable) {
      // Truncate to the low 32-bits.
      SDValue BitTable = DAG.getConstant(
          AMDGPU::FltRoundToHWConversionTable & 0xffff, SL, MVT::i32);
 
      SDValue Two = DAG.getConstant(2, SL, MVT::i32);
      SDValue RoundModeTimesNumBits =
          DAG.getNode(ISD::SHL, SL, MVT::i32, NewMode, Two);
 
      NewMode =
          DAG.getNode(ISD::SRL, SL, MVT::i32, BitTable, RoundModeTimesNumBits);
 
      // TODO: SimplifyDemandedBits on the setreg source here can likely reduce
      // the table extracted bits into inline immediates.
    } else {
      // table_index = umin(value, value - 4)
      // MODE.fp_round = (bit_table >> (table_index << 2)) & 0xf
      SDValue BitTable =
          DAG.getConstant(AMDGPU::FltRoundToHWConversionTable, SL, MVT::i64);
 
      SDValue Four = DAG.getConstant(4, SL, MVT::i32);
      SDValue OffsetEnum = DAG.getNode(ISD::SUB, SL, MVT::i32, NewMode, Four);
      SDValue IndexVal =
          DAG.getNode(ISD::UMIN, SL, MVT::i32, NewMode, OffsetEnum);
 
      SDValue Two = DAG.getConstant(2, SL, MVT::i32);
      SDValue RoundModeTimesNumBits =
          DAG.getNode(ISD::SHL, SL, MVT::i32, IndexVal, Two);
 
      SDValue TableValue =
          DAG.getNode(ISD::SRL, SL, MVT::i64, BitTable, RoundModeTimesNumBits);
      SDValue TruncTable = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, TableValue);
 
      // No need to mask out the high bits since the setreg will ignore them
      // anyway.
      NewMode = TruncTable;
    }
 
    // Insert a readfirstlane in case the value is a VGPR. We could do this
    // earlier and keep more operations scalar, but that interferes with
    // combining the source.
    SDValue ReadFirstLaneID =
        DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
    NewMode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
                          ReadFirstLaneID, NewMode);
  }
 
  // N.B. The setreg will be later folded into s_round_mode on supported
  // targets.
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
  uint32_t BothRoundHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 4);
  SDValue RoundBothImm = DAG.getTargetConstant(BothRoundHwReg, SL, MVT::i32);
 
  SDValue SetReg =
      DAG.getNode(ISD::INTRINSIC_VOID, SL, Op->getVTList(), Op.getOperand(0),
                  IntrinID, RoundBothImm, NewMode);
 
  return SetReg;
}
 
SDValue SITargetLowering::lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const {
  if (Op->isDivergent() &&
      (!Subtarget->hasVmemPrefInsts() || !Op.getConstantOperandVal(4)))
    // Cannot do I$ prefetch with divergent pointer.
    return SDValue();
 
  switch (cast<MemSDNode>(Op)->getAddressSpace()) {
  case AMDGPUAS::FLAT_ADDRESS:
  case AMDGPUAS::GLOBAL_ADDRESS:
  case AMDGPUAS::CONSTANT_ADDRESS:
    break;
  case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
    if (Subtarget->hasSafeSmemPrefetch())
      break;
    [[fallthrough]];
  default:
    return SDValue();
  }
 
  // I$ prefetch
  if (!Subtarget->hasSafeSmemPrefetch() && !Op.getConstantOperandVal(4))
    return SDValue();
 
  return Op;
}
 
// Work around DAG legality rules only based on the result type.
SDValue SITargetLowering::lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
  bool IsStrict = Op.getOpcode() == ISD::STRICT_FP_EXTEND;
  SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
  EVT SrcVT = Src.getValueType();
 
  if (SrcVT.getScalarType() != MVT::bf16)
    return Op;
 
  SDLoc SL(Op);
  SDValue BitCast =
      DAG.getNode(ISD::BITCAST, SL, SrcVT.changeTypeToInteger(), Src);
 
  EVT DstVT = Op.getValueType();
  if (IsStrict)
    llvm_unreachable("Need STRICT_BF16_TO_FP");
 
  return DAG.getNode(ISD::BF16_TO_FP, SL, DstVT, BitCast);
}
 
SDValue SITargetLowering::lowerGET_FPENV(SDValue Op, SelectionDAG &DAG) const {
  SDLoc SL(Op);
  if (Op.getValueType() != MVT::i64)
    return Op;
 
  uint32_t ModeHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23);
  SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
  uint32_t TrapHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5);
  SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
 
  SDVTList VTList = DAG.getVTList(MVT::i32, MVT::Other);
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_getreg, SL, MVT::i32);
  SDValue GetModeReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
                                   Op.getOperand(0), IntrinID, ModeHwRegImm);
  SDValue GetTrapReg = DAG.getNode(ISD::INTRINSIC_W_CHAIN, SL, VTList,
                                   Op.getOperand(0), IntrinID, TrapHwRegImm);
  SDValue TokenReg =
      DAG.getNode(ISD::TokenFactor, SL, MVT::Other, GetModeReg.getValue(1),
                  GetTrapReg.getValue(1));
 
  SDValue CvtPtr =
      DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, GetModeReg, GetTrapReg);
  SDValue Result = DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr);
 
  return DAG.getMergeValues({Result, TokenReg}, SL);
}
 
SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const {
  SDLoc SL(Op);
  if (Op.getOperand(1).getValueType() != MVT::i64)
    return Op;
 
  SDValue Input = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op.getOperand(1));
  SDValue NewModeReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
                                   DAG.getConstant(0, SL, MVT::i32));
  SDValue NewTrapReg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Input,
                                   DAG.getConstant(1, SL, MVT::i32));
 
  SDValue ReadFirstLaneID =
      DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, SL, MVT::i32);
  NewModeReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
                           ReadFirstLaneID, NewModeReg);
  NewTrapReg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
                           ReadFirstLaneID, NewTrapReg);
 
  unsigned ModeHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23);
  SDValue ModeHwRegImm = DAG.getTargetConstant(ModeHwReg, SL, MVT::i32);
  unsigned TrapHwReg =
      AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5);
  SDValue TrapHwRegImm = DAG.getTargetConstant(TrapHwReg, SL, MVT::i32);
 
  SDValue IntrinID =
      DAG.getTargetConstant(Intrinsic::amdgcn_s_setreg, SL, MVT::i32);
  SDValue SetModeReg =
      DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
                  IntrinID, ModeHwRegImm, NewModeReg);
  SDValue SetTrapReg =
      DAG.getNode(ISD::INTRINSIC_VOID, SL, MVT::Other, Op.getOperand(0),
                  IntrinID, TrapHwRegImm, NewTrapReg);
  return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, SetTrapReg, SetModeReg);
}
 
Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT,
                                             const MachineFunction &MF) const {
  const Function &Fn = MF.getFunction();
 
  Register Reg = StringSwitch<Register>(RegName)
                     .Case("m0", AMDGPU::M0)
                     .Case("exec", AMDGPU::EXEC)
                     .Case("exec_lo", AMDGPU::EXEC_LO)
                     .Case("exec_hi", AMDGPU::EXEC_HI)
                     .Case("flat_scratch", AMDGPU::FLAT_SCR)
                     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
                     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
                     .Default(Register());
  if (!Reg)
    return Reg;
 
  if (!Subtarget->hasFlatScrRegister() &&
      Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) {
    Fn.getContext().emitError(Twine("invalid register \"" + StringRef(RegName) +
                                    "\" for subtarget."));
  }
 
  switch (Reg) {
  case AMDGPU::M0:
  case AMDGPU::EXEC_LO:
  case AMDGPU::EXEC_HI:
  case AMDGPU::FLAT_SCR_LO:
  case AMDGPU::FLAT_SCR_HI:
    if (VT.getSizeInBits() == 32)
      return Reg;
    break;
  case AMDGPU::EXEC:
  case AMDGPU::FLAT_SCR:
    if (VT.getSizeInBits() == 64)
      return Reg;
    break;
  default:
    llvm_unreachable("missing register type checking");
  }
 
  report_fatal_error(
      Twine("invalid type for register \"" + StringRef(RegName) + "\"."));
}
 
// If kill is not the last instruction, split the block so kill is always a
// proper terminator.
MachineBasicBlock *
SITargetLowering::splitKillBlock(MachineInstr &MI,
                                 MachineBasicBlock *BB) const {
  MachineBasicBlock *SplitBB = BB->splitAt(MI, /*UpdateLiveIns=*/true);
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
  MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode()));
  return SplitBB;
}
 
// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
// \p MI will be the only instruction in the loop body block. Otherwise, it will
// be the first instruction in the remainder block.
//
/// \returns { LoopBody, Remainder }
static std::pair<MachineBasicBlock *, MachineBasicBlock *>
splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
  MachineFunction *MF = MBB.getParent();
  MachineBasicBlock::iterator I(&MI);
 
  // To insert the loop we need to split the block. Move everything after this
  // point to a new block, and insert a new empty block between the two.
  MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
  MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
  MachineFunction::iterator MBBI(MBB);
  ++MBBI;
 
  MF->insert(MBBI, LoopBB);
  MF->insert(MBBI, RemainderBB);
 
  LoopBB->addSuccessor(LoopBB);
  LoopBB->addSuccessor(RemainderBB);
 
  // Move the rest of the block into a new block.
  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
 
  if (InstInLoop) {
    auto Next = std::next(I);
 
    // Move instruction to loop body.
    LoopBB->splice(LoopBB->begin(), &MBB, I, Next);
 
    // Move the rest of the block.
    RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end());
  } else {
    RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
  }
 
  MBB.addSuccessor(LoopBB);
 
  return std::pair(LoopBB, RemainderBB);
}
 
/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
  MachineBasicBlock *MBB = MI.getParent();
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
  auto I = MI.getIterator();
  auto E = std::next(I);
 
  // clang-format off
  BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
      .addImm(0);
  // clang-format on
 
  MIBundleBuilder Bundler(*MBB, I, E);
  finalizeBundle(*MBB, Bundler.begin());
}
 
MachineBasicBlock *
SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
                                         MachineBasicBlock *BB) const {
  const DebugLoc &DL = MI.getDebugLoc();
 
  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
 
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
 
  // Apparently kill flags are only valid if the def is in the same block?
  if (MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0))
    Src->setIsKill(false);
 
  auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, *BB, true);
 
  MachineBasicBlock::iterator I = LoopBB->end();
 
  const unsigned EncodedReg = AMDGPU::Hwreg::HwregEncoding::encode(
      AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1);
 
  // Clear TRAP_STS.MEM_VIOL
  BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
      .addImm(0)
      .addImm(EncodedReg);
 
  bundleInstWithWaitcnt(MI);
 
  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
 
  // Load and check TRAP_STS.MEM_VIOL
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
      .addImm(EncodedReg);
 
  // FIXME: Do we need to use an isel pseudo that may clobber scc?
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32))
      .addReg(Reg, RegState::Kill)
      .addImm(0);
  // clang-format off
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
      .addMBB(LoopBB);
  // clang-format on
 
  return RemainderBB;
}
 
// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
// wavefront. If the value is uniform and just happens to be in a VGPR, this
// will only do one iteration. In the worst case, this will loop 64 times.
//
// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
static MachineBasicBlock::iterator
emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
                       MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
                       const DebugLoc &DL, const MachineOperand &Idx,
                       unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
                       unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
                       Register &SGPRIdxReg) {
 
  MachineFunction *MF = OrigBB.getParent();
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
  MachineBasicBlock::iterator I = LoopBB.begin();
 
  const TargetRegisterClass *BoolRC = TRI->getBoolRC();
  Register PhiExec = MRI.createVirtualRegister(BoolRC);
  Register NewExec = MRI.createVirtualRegister(BoolRC);
  Register CurrentIdxReg =
      MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
  Register CondReg = MRI.createVirtualRegister(BoolRC);
 
  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg)
      .addReg(InitReg)
      .addMBB(&OrigBB)
      .addReg(ResultReg)
      .addMBB(&LoopBB);
 
  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec)
      .addReg(InitSaveExecReg)
      .addMBB(&OrigBB)
      .addReg(NewExec)
      .addMBB(&LoopBB);
 
  // Read the next variant <- also loop target.
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
      .addReg(Idx.getReg(), getUndefRegState(Idx.isUndef()));
 
  // Compare the just read M0 value to all possible Idx values.
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
      .addReg(CurrentIdxReg)
      .addReg(Idx.getReg(), {}, Idx.getSubReg());
 
  // Update EXEC, save the original EXEC value to VCC.
  BuildMI(LoopBB, I, DL, TII->get(LMC.AndSaveExecOpc), NewExec)
      .addReg(CondReg, RegState::Kill);
 
  MRI.setSimpleHint(NewExec, CondReg);
 
  if (UseGPRIdxMode) {
    if (Offset == 0) {
      SGPRIdxReg = CurrentIdxReg;
    } else {
      SGPRIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg)
          .addReg(CurrentIdxReg, RegState::Kill)
          .addImm(Offset);
    }
  } else {
    // Move index from VCC into M0
    if (Offset == 0) {
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::M0)
          .addReg(CurrentIdxReg, RegState::Kill);
    } else {
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
          .addReg(CurrentIdxReg, RegState::Kill)
          .addImm(Offset);
    }
  }
 
  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
  MachineInstr *InsertPt =
      BuildMI(LoopBB, I, DL, TII->get(LMC.XorTermOpc), LMC.ExecReg)
          .addReg(LMC.ExecReg)
          .addReg(NewExec);
 
  // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
  // s_cbranch_scc0?
 
  // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
  // clang-format off
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
      .addMBB(&LoopBB);
  // clang-format on
 
  return InsertPt->getIterator();
}
 
// This has slightly sub-optimal regalloc when the source vector is killed by
// the read. The register allocator does not understand that the kill is
// per-workitem, so is kept alive for the whole loop so we end up not re-using a
// subregister from it, using 1 more VGPR than necessary. This was saved when
// this was expanded after register allocation.
static MachineBasicBlock::iterator
loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
               unsigned InitResultReg, unsigned PhiReg, int Offset,
               bool UseGPRIdxMode, Register &SGPRIdxReg) {
  MachineFunction *MF = MBB.getParent();
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  MachineRegisterInfo &MRI = MF->getRegInfo();
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  const auto *BoolXExecRC = TRI->getWaveMaskRegClass();
  Register DstReg = MI.getOperand(0).getReg();
  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
  Register TmpExec = MRI.createVirtualRegister(BoolXExecRC);
  const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
 
  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec);
 
  // Save the EXEC mask
  // clang-format off
  BuildMI(MBB, I, DL, TII->get(LMC.MovOpc), SaveExec)
      .addReg(LMC.ExecReg);
  // clang-format on
 
  auto [LoopBB, RemainderBB] = splitBlockForLoop(MI, MBB, false);
 
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
  auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx,
                                      InitResultReg, DstReg, PhiReg, TmpExec,
                                      Offset, UseGPRIdxMode, SGPRIdxReg);
 
  MachineBasicBlock *LandingPad = MF->CreateMachineBasicBlock();
  MachineFunction::iterator MBBI(LoopBB);
  ++MBBI;
  MF->insert(MBBI, LandingPad);
  LoopBB->removeSuccessor(RemainderBB);
  LandingPad->addSuccessor(RemainderBB);
  LoopBB->addSuccessor(LandingPad);
  MachineBasicBlock::iterator First = LandingPad->begin();
  // clang-format off
  BuildMI(*LandingPad, First, DL, TII->get(LMC.MovOpc), LMC.ExecReg)
      .addReg(SaveExec);
  // clang-format on
 
  return InsPt;
}
 
// Returns subreg index, offset
static std::pair<unsigned, int>
computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
                            const TargetRegisterClass *SuperRC, unsigned VecReg,
                            int Offset) {
  int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32;
 
  // Skip out of bounds offsets, or else we would end up using an undefined
  // register.
  if (Offset >= NumElts || Offset < 0)
    return std::pair(AMDGPU::sub0, Offset);
 
  return std::pair(SIRegisterInfo::getSubRegFromChannel(Offset), 0);
}
 
static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
                                 MachineRegisterInfo &MRI, MachineInstr &MI,
                                 int Offset) {
  MachineBasicBlock *MBB = MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
  assert(Idx->getReg() != AMDGPU::NoRegister);
 
  if (Offset == 0) {
    // clang-format off
    BuildMI(*MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::M0)
        .add(*Idx);
    // clang-format on
  } else {
    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
        .add(*Idx)
        .addImm(Offset);
  }
}
 
static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
                                   MachineRegisterInfo &MRI, MachineInstr &MI,
                                   int Offset) {
  MachineBasicBlock *MBB = MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
  if (Offset == 0)
    return Idx->getReg();
 
  Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
      .add(*Idx)
      .addImm(Offset);
  return Tmp;
}
 
static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
                                          MachineBasicBlock &MBB,
                                          const GCNSubtarget &ST) {
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo &TRI = TII->getRegisterInfo();
  MachineFunction *MF = MBB.getParent();
  MachineRegisterInfo &MRI = MF->getRegInfo();
 
  Register Dst = MI.getOperand(0).getReg();
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
  Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
 
  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
 
  unsigned SubReg;
  std::tie(SubReg, Offset) =
      computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset);
 
  const bool UseGPRIdxMode = ST.useVGPRIndexMode();
 
  // Check for a SGPR index.
  if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
    MachineBasicBlock::iterator I(&MI);
    const DebugLoc &DL = MI.getDebugLoc();
 
    if (UseGPRIdxMode) {
      // TODO: Look at the uses to avoid the copy. This may require rescheduling
      // to avoid interfering with other uses, so probably requires a new
      // optimization pass.
      Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
 
      const MCInstrDesc &GPRIDXDesc =
          TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
      BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
          .addReg(SrcReg)
          .addReg(Idx)
          .addImm(SubReg);
    } else {
      setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
 
      BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
          .addReg(SrcReg, {}, SubReg)
          .addReg(SrcReg, RegState::Implicit);
    }
 
    MI.eraseFromParent();
 
    return &MBB;
  }
 
  // Control flow needs to be inserted if indexing with a VGPR.
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock::iterator I(&MI);
 
  Register PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
  Register InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
 
  Register SGPRIdxReg;
  auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg, Offset,
                              UseGPRIdxMode, SGPRIdxReg);
 
  MachineBasicBlock *LoopBB = InsPt->getParent();
 
  if (UseGPRIdxMode) {
    const MCInstrDesc &GPRIDXDesc =
        TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
 
    BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
        .addReg(SrcReg)
        .addReg(SGPRIdxReg)
        .addImm(SubReg);
  } else {
    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
        .addReg(SrcReg, {}, SubReg)
        .addReg(SrcReg, RegState::Implicit);
  }
 
  MI.eraseFromParent();
 
  return LoopBB;
}
 
static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
                                          MachineBasicBlock &MBB,
                                          const GCNSubtarget &ST) {
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo &TRI = TII->getRegisterInfo();
  MachineFunction *MF = MBB.getParent();
  MachineRegisterInfo &MRI = MF->getRegInfo();
 
  Register Dst = MI.getOperand(0).getReg();
  const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src);
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
  const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val);
  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg());
  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
 
  // This can be an immediate, but will be folded later.
  assert(Val->getReg());
 
  unsigned SubReg;
  std::tie(SubReg, Offset) =
      computeIndirectRegAndOffset(TRI, VecRC, SrcVec->getReg(), Offset);
  const bool UseGPRIdxMode = ST.useVGPRIndexMode();
 
  if (Idx->getReg() == AMDGPU::NoRegister) {
    MachineBasicBlock::iterator I(&MI);
    const DebugLoc &DL = MI.getDebugLoc();
 
    assert(Offset == 0);
 
    BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst)
        .add(*SrcVec)
        .add(*Val)
        .addImm(SubReg);
 
    MI.eraseFromParent();
    return &MBB;
  }
 
  // Check for a SGPR index.
  if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
    MachineBasicBlock::iterator I(&MI);
    const DebugLoc &DL = MI.getDebugLoc();
 
    if (UseGPRIdxMode) {
      Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
 
      const MCInstrDesc &GPRIDXDesc =
          TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
      BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
          .addReg(SrcVec->getReg())
          .add(*Val)
          .addReg(Idx)
          .addImm(SubReg);
    } else {
      setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
 
      const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
          TRI.getRegSizeInBits(*VecRC), 32, false);
      BuildMI(MBB, I, DL, MovRelDesc, Dst)
          .addReg(SrcVec->getReg())
          .add(*Val)
          .addImm(SubReg);
    }
    MI.eraseFromParent();
    return &MBB;
  }
 
  // Control flow needs to be inserted if indexing with a VGPR.
  if (Val->isReg())
    MRI.clearKillFlags(Val->getReg());
 
  const DebugLoc &DL = MI.getDebugLoc();
 
  Register PhiReg = MRI.createVirtualRegister(VecRC);
 
  Register SGPRIdxReg;
  auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg, Offset,
                              UseGPRIdxMode, SGPRIdxReg);
  MachineBasicBlock *LoopBB = InsPt->getParent();
 
  if (UseGPRIdxMode) {
    const MCInstrDesc &GPRIDXDesc =
        TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
 
    BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
        .addReg(PhiReg)
        .add(*Val)
        .addReg(SGPRIdxReg)
        .addImm(SubReg);
  } else {
    const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
        TRI.getRegSizeInBits(*VecRC), 32, false);
    BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst)
        .addReg(PhiReg)
        .add(*Val)
        .addImm(SubReg);
  }
 
  MI.eraseFromParent();
  return LoopBB;
}
 
static MachineBasicBlock *expand64BitScalarArithmetic(MachineInstr &MI,
                                                      MachineBasicBlock *BB) {
  // For targets older than GFX12, we emit a sequence of 32-bit operations.
  // For GFX12, we emit s_add_u64 and s_sub_u64.
  MachineFunction *MF = BB->getParent();
  const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
  const DebugLoc &DL = MI.getDebugLoc();
  MachineOperand &Dest = MI.getOperand(0);
  MachineOperand &Src0 = MI.getOperand(1);
  MachineOperand &Src1 = MI.getOperand(2);
  bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
  if (ST.hasScalarAddSub64()) {
    unsigned Opc = IsAdd ? AMDGPU::S_ADD_U64 : AMDGPU::S_SUB_U64;
    // clang-format off
    BuildMI(*BB, MI, DL, TII->get(Opc), Dest.getReg())
        .add(Src0)
        .add(Src1);
    // clang-format on
  } else {
    const SIRegisterInfo *TRI = ST.getRegisterInfo();
    const TargetRegisterClass *BoolRC = TRI->getBoolRC();
 
    Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
    Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
 
    MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
    MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
 
    MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, BoolRC, AMDGPU::sub0, &AMDGPU::SReg_32RegClass);
    MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, BoolRC, AMDGPU::sub1, &AMDGPU::SReg_32RegClass);
 
    unsigned LoOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
    unsigned HiOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
    BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0).add(Src0Sub0).add(Src1Sub0);
    BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1).add(Src0Sub1).add(Src1Sub1);
    BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
        .addReg(DestSub0)
        .addImm(AMDGPU::sub0)
        .addReg(DestSub1)
        .addImm(AMDGPU::sub1);
  }
  MI.eraseFromParent();
  return BB;
}
 
static void expand64BitV_CNDMASK(MachineInstr &MI, MachineBasicBlock *BB) {
  MachineFunction *MF = BB->getParent();
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  MachineRegisterInfo &MRI = MF->getRegInfo();
  const DebugLoc &DL = MI.getDebugLoc();
  Register Dst = MI.getOperand(0).getReg();
  const MachineOperand &Src0 = MI.getOperand(1);
  const MachineOperand &Src1 = MI.getOperand(2);
  Register SrcCond = MI.getOperand(3).getReg();
 
  Register DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
  Register DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
  const TargetRegisterClass *CondRC = TRI->getWaveMaskRegClass();
  Register SrcCondCopy = MRI.createVirtualRegister(CondRC);
 
  int Src0Idx =
      AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
  int Src1Idx =
      AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
  const TargetRegisterClass *Src0RC =
      TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src0Idx));
  const TargetRegisterClass *Src1RC =
      TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), Src1Idx));
 
  const TargetRegisterClass *Src0SubRC =
      TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
  const TargetRegisterClass *Src1SubRC =
      TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
 
  MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(
      MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
  MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(
      MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
 
  MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(
      MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
  MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(
      MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
 
  BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy).addReg(SrcCond);
  BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
      .addImm(0)
      .add(Src0Sub0)
      .addImm(0)
      .add(Src1Sub0)
      .addReg(SrcCondCopy);
 
  BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
      .addImm(0)
      .add(Src0Sub1)
      .addImm(0)
      .add(Src1Sub1)
      .addReg(SrcCondCopy);
 
  BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
      .addReg(DstLo)
      .addImm(AMDGPU::sub0)
      .addReg(DstHi)
      .addImm(AMDGPU::sub1);
  MI.eraseFromParent();
}
 
static uint64_t getIdentityValueForWaveReduction(unsigned Opc) {
  switch (Opc) {
  case AMDGPU::S_MIN_U32:
    return std::numeric_limits<uint32_t>::max();
  case AMDGPU::S_MIN_I32:
    return std::numeric_limits<int32_t>::max();
  case AMDGPU::S_MAX_U32:
    return std::numeric_limits<uint32_t>::min();
  case AMDGPU::S_MAX_I32:
    return std::numeric_limits<int32_t>::min();
  case AMDGPU::V_ADD_F32_e64: // -0.0
    return 0x80000000;
  case AMDGPU::V_SUB_F32_e64: // +0.0
    return 0x0;
  case AMDGPU::S_ADD_I32:
  case AMDGPU::S_SUB_I32:
  case AMDGPU::S_OR_B32:
  case AMDGPU::S_XOR_B32:
    return std::numeric_limits<uint32_t>::min();
  case AMDGPU::S_AND_B32:
    return std::numeric_limits<uint32_t>::max();
  case AMDGPU::V_MIN_F32_e64:
  case AMDGPU::V_MAX_F32_e64:
    return 0x7fc00000;           // qNAN
  case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
    return std::numeric_limits<uint64_t>::max();
  case AMDGPU::V_CMP_LT_I64_e64: // min.i64
    return std::numeric_limits<int64_t>::max();
  case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
    return std::numeric_limits<uint64_t>::min();
  case AMDGPU::V_CMP_GT_I64_e64: // max.i64
    return std::numeric_limits<int64_t>::min();
  case AMDGPU::V_MIN_F64_e64:
  case AMDGPU::V_MAX_F64_e64:
  case AMDGPU::V_MIN_NUM_F64_e64:
  case AMDGPU::V_MAX_NUM_F64_e64:
    return 0x7FF8000000000000; // qNAN
  case AMDGPU::S_ADD_U64_PSEUDO:
  case AMDGPU::S_SUB_U64_PSEUDO:
  case AMDGPU::S_OR_B64:
  case AMDGPU::S_XOR_B64:
    return std::numeric_limits<uint64_t>::min();
  case AMDGPU::S_AND_B64:
    return std::numeric_limits<uint64_t>::max();
  case AMDGPU::V_ADD_F64_e64:
  case AMDGPU::V_ADD_F64_pseudo_e64:
    return 0x8000000000000000; // -0.0
  default:
    llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction");
  }
}
 
static bool is32bitWaveReduceOperation(unsigned Opc) {
  return Opc == AMDGPU::S_MIN_U32 || Opc == AMDGPU::S_MIN_I32 ||
         Opc == AMDGPU::S_MAX_U32 || Opc == AMDGPU::S_MAX_I32 ||
         Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32 ||
         Opc == AMDGPU::S_AND_B32 || Opc == AMDGPU::S_OR_B32 ||
         Opc == AMDGPU::S_XOR_B32 || Opc == AMDGPU::V_MIN_F32_e64 ||
         Opc == AMDGPU::V_MAX_F32_e64 || Opc == AMDGPU::V_ADD_F32_e64 ||
         Opc == AMDGPU::V_SUB_F32_e64;
}
 
static bool isFloatingPointWaveReduceOperation(unsigned Opc) {
  return Opc == AMDGPU::V_MIN_F32_e64 || Opc == AMDGPU::V_MAX_F32_e64 ||
         Opc == AMDGPU::V_ADD_F32_e64 || Opc == AMDGPU::V_SUB_F32_e64 ||
         Opc == AMDGPU::V_MIN_F64_e64 || Opc == AMDGPU::V_MAX_F64_e64 ||
         Opc == AMDGPU::V_MIN_NUM_F64_e64 || Opc == AMDGPU::V_MAX_NUM_F64_e64 ||
         Opc == AMDGPU::V_ADD_F64_e64 || Opc == AMDGPU::V_ADD_F64_pseudo_e64;
}
 
static std::tuple<unsigned, unsigned>
getDPPOpcForWaveReduction(unsigned Opc, const GCNSubtarget &ST) {
  unsigned DPPOpc;
  switch (Opc) {
  case AMDGPU::S_MIN_U32:
    DPPOpc = AMDGPU::V_MIN_U32_dpp;
    break;
  case AMDGPU::S_MIN_I32:
    DPPOpc = AMDGPU::V_MIN_I32_dpp;
    break;
  case AMDGPU::S_MAX_U32:
    DPPOpc = AMDGPU::V_MAX_U32_dpp;
    break;
  case AMDGPU::S_MAX_I32:
    DPPOpc = AMDGPU::V_MAX_I32_dpp;
    break;
  case AMDGPU::S_ADD_I32:
  case AMDGPU::S_SUB_I32:
    DPPOpc = ST.hasAddNoCarryInsts() ? AMDGPU::V_ADD_U32_dpp
                                     : AMDGPU::V_ADD_CO_U32_dpp;
    break;
  case AMDGPU::S_AND_B32:
    DPPOpc = AMDGPU::V_AND_B32_dpp;
    break;
  case AMDGPU::S_OR_B32:
    DPPOpc = AMDGPU::V_OR_B32_dpp;
    break;
  case AMDGPU::S_XOR_B32:
    DPPOpc = AMDGPU::V_XOR_B32_dpp;
    break;
  case AMDGPU::V_ADD_F32_e64:
  case AMDGPU::V_SUB_F32_e64:
    DPPOpc = AMDGPU::V_ADD_F32_dpp;
    break;
  case AMDGPU::V_MIN_F32_e64:
    DPPOpc = AMDGPU::V_MIN_F32_dpp;
    break;
  case AMDGPU::V_MAX_F32_e64:
    DPPOpc = AMDGPU::V_MAX_F32_dpp;
    break;
  case AMDGPU::V_CMP_LT_U64_e64: // umin.u64
  case AMDGPU::V_CMP_LT_I64_e64: // min.i64
  case AMDGPU::V_CMP_GT_U64_e64: // umax.u64
  case AMDGPU::V_CMP_GT_I64_e64: // max.i64
  case AMDGPU::S_ADD_U64_PSEUDO:
  case AMDGPU::S_SUB_U64_PSEUDO:
  case AMDGPU::S_AND_B64:
  case AMDGPU::S_OR_B64:
  case AMDGPU::S_XOR_B64:
  case AMDGPU::V_MIN_NUM_F64_e64:
  case AMDGPU::V_MIN_F64_e64:
  case AMDGPU::V_MAX_NUM_F64_e64:
  case AMDGPU::V_MAX_F64_e64:
  case AMDGPU::V_ADD_F64_pseudo_e64:
  case AMDGPU::V_ADD_F64_e64:
    DPPOpc = AMDGPU::V_MOV_B64_DPP_PSEUDO;
    break;
  default:
    llvm_unreachable("unhandled lane op");
  }
  unsigned ClampOpc = Opc;
  if (!ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/true)) {
    if (Opc == AMDGPU::S_SUB_I32)
      ClampOpc = AMDGPU::S_ADD_I32;
    if (Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO)
      ClampOpc = AMDGPU::V_ADD_CO_U32_e64;
    else if (Opc == AMDGPU::S_AND_B64)
      ClampOpc = AMDGPU::V_AND_B32_e64;
    else if (Opc == AMDGPU::S_OR_B64)
      ClampOpc = AMDGPU::V_OR_B32_e64;
    else if (Opc == AMDGPU::S_XOR_B64)
      ClampOpc = AMDGPU::V_XOR_B32_e64;
    else
      ClampOpc = ST.getInstrInfo()->getVALUOp(ClampOpc);
  }
  return {DPPOpc, ClampOpc};
}
 
static std::pair<Register, Register>
ExtractSubRegs(MachineInstr &MI, MachineOperand &Op,
               const TargetRegisterClass *SrcRC, const GCNSubtarget &ST,
               MachineRegisterInfo &MRI) {
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const SIInstrInfo *TII = ST.getInstrInfo();
  const TargetRegisterClass *SrcSubRC =
      TRI->getSubRegisterClass(SrcRC, AMDGPU::sub0);
  Register Op1L =
      TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub0, SrcSubRC);
  Register Op1H =
      TII->buildExtractSubReg(MI, MRI, Op, SrcRC, AMDGPU::sub1, SrcSubRC);
  return {Op1L, Op1H};
}
 
static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
                                          MachineBasicBlock &BB,
                                          const GCNSubtarget &ST,
                                          unsigned Opc) {
  MachineRegisterInfo &MRI = BB.getParent()->getRegInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const DebugLoc &DL = MI.getDebugLoc();
  const SIInstrInfo *TII = ST.getInstrInfo();
 
  // Reduction operations depend on whether the input operand is SGPR or VGPR.
  Register SrcReg = MI.getOperand(1).getReg();
  bool isSGPR = TRI->isSGPRClass(MRI.getRegClass(SrcReg));
  Register DstReg = MI.getOperand(0).getReg();
  unsigned Stratergy = static_cast<unsigned>(MI.getOperand(2).getImm());
  enum WAVE_REDUCE_STRATEGY : unsigned { DEFAULT = 0, ITERATIVE = 1, DPP = 2 };
  MachineBasicBlock *RetBB = nullptr;
  unsigned MIOpc = MI.getOpcode();
  auto BuildRegSequence = [&](MachineBasicBlock &BB,
                              MachineBasicBlock::iterator MI, Register Dst,
                              Register Src0, Register Src1) {
    auto RegSequence =
        BuildMI(BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dst)
            .addReg(Src0)
            .addImm(AMDGPU::sub0)
            .addReg(Src1)
            .addImm(AMDGPU::sub1);
    return RegSequence;
  };
  if (isSGPR) {
    switch (Opc) {
    case AMDGPU::S_MIN_U32:
    case AMDGPU::S_MIN_I32:
    case AMDGPU::V_MIN_F32_e64:
    case AMDGPU::S_MAX_U32:
    case AMDGPU::S_MAX_I32:
    case AMDGPU::V_MAX_F32_e64:
    case AMDGPU::S_AND_B32:
    case AMDGPU::S_OR_B32: {
      // Idempotent operations.
      BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg).addReg(SrcReg);
      RetBB = &BB;
      break;
    }
    case AMDGPU::V_CMP_LT_U64_e64: // umin
    case AMDGPU::V_CMP_LT_I64_e64: // min
    case AMDGPU::V_CMP_GT_U64_e64: // umax
    case AMDGPU::V_CMP_GT_I64_e64: // max
    case AMDGPU::V_MIN_F64_e64:
    case AMDGPU::V_MIN_NUM_F64_e64:
    case AMDGPU::V_MAX_F64_e64:
    case AMDGPU::V_MAX_NUM_F64_e64:
    case AMDGPU::S_AND_B64:
    case AMDGPU::S_OR_B64: {
      // Idempotent operations.
      BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B64), DstReg).addReg(SrcReg);
      RetBB = &BB;
      break;
    }
    case AMDGPU::S_XOR_B32:
    case AMDGPU::S_XOR_B64:
    case AMDGPU::S_ADD_I32:
    case AMDGPU::S_ADD_U64_PSEUDO:
    case AMDGPU::V_ADD_F32_e64:
    case AMDGPU::V_ADD_F64_e64:
    case AMDGPU::V_ADD_F64_pseudo_e64:
    case AMDGPU::S_SUB_I32:
    case AMDGPU::S_SUB_U64_PSEUDO:
    case AMDGPU::V_SUB_F32_e64: {
      const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
      const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg);
      Register ExecMask = MRI.createVirtualRegister(WaveMaskRegClass);
      Register NumActiveLanes =
          MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
 
      bool IsWave32 = ST.isWave32();
      unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
      MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
      unsigned BitCountOpc =
          IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64;
 
      BuildMI(BB, MI, DL, TII->get(MovOpc), ExecMask).addReg(ExecReg);
 
      auto NewAccumulator =
          BuildMI(BB, MI, DL, TII->get(BitCountOpc), NumActiveLanes)
              .addReg(ExecMask);
 
      switch (Opc) {
      case AMDGPU::S_XOR_B32:
      case AMDGPU::S_XOR_B64: {
        // Performing an XOR operation on a uniform value
        // depends on the parity of the number of active lanes.
        // For even parity, the result will be 0, for odd
        // parity the result will be the same as the input value.
        Register ParityRegister =
            MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
 
        BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister)
            .addReg(NewAccumulator->getOperand(0).getReg())
            .addImm(1)
            .setOperandDead(3); // Dead scc
        if (Opc == AMDGPU::S_XOR_B32) {
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
              .addReg(SrcReg)
              .addReg(ParityRegister);
        } else {
          Register DestSub0 =
              MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
          Register DestSub1 =
              MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
          auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
                                             MRI.getRegClass(SrcReg), ST, MRI);
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
              .addReg(Op1L)
              .addReg(ParityRegister);
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub1)
              .addReg(Op1H)
              .addReg(ParityRegister);
          BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
        }
        break;
      }
      case AMDGPU::S_SUB_I32: {
        Register NegatedVal = MRI.createVirtualRegister(DstRegClass);
 
        // Take the negation of the source operand.
        BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedVal)
            .addImm(0)
            .addReg(SrcReg);
        BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
            .addReg(NegatedVal)
            .addReg(NewAccumulator->getOperand(0).getReg());
        break;
      }
      case AMDGPU::S_ADD_I32: {
        BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
            .addReg(SrcReg)
            .addReg(NewAccumulator->getOperand(0).getReg());
        break;
      }
      case AMDGPU::S_ADD_U64_PSEUDO:
      case AMDGPU::S_SUB_U64_PSEUDO: {
        Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
        Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
        Register Op1H_Op0L_Reg =
            MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
        Register Op1L_Op0H_Reg =
            MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
        Register CarryReg =
            MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
        Register AddReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
        Register NegatedValLo =
            MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
        Register NegatedValHi =
            MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
        auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
                                           MRI.getRegClass(SrcReg), ST, MRI);
        if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
              .addImm(0)
              .addReg(NewAccumulator->getOperand(0).getReg())
              .setOperandDead(3); // Dead scc
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ASHR_I32), NegatedValHi)
              .addReg(NegatedValLo)
              .addImm(31)
              .setOperandDead(3); // Dead scc
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1L_Op0H_Reg)
              .addReg(Op1L)
              .addReg(NegatedValHi);
        }
        Register LowOpcode = Opc == AMDGPU::S_SUB_U64_PSEUDO
                                 ? NegatedValLo
                                 : NewAccumulator->getOperand(0).getReg();
        BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DestSub0)
            .addReg(Op1L)
            .addReg(LowOpcode);
        if (ST.hasScalarMulHiInsts()) {
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_HI_U32), CarryReg)
              .addReg(Op1L)
              .addReg(LowOpcode);
        } else {
          Register VCarryReg =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          Register LowOpVGPR =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), LowOpVGPR)
              .addReg(LowOpcode);
          BuildMI(BB, MI, DL, TII->get(AMDGPU::V_MUL_HI_U32_e64), VCarryReg)
              .addReg(Op1L)
              .addReg(LowOpVGPR);
          BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CarryReg)
              .addReg(VCarryReg);
        }
        BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), Op1H_Op0L_Reg)
            .addReg(Op1H)
            .addReg(LowOpcode);
 
        Register HiVal = Opc == AMDGPU::S_SUB_U64_PSEUDO ? AddReg : DestSub1;
        BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ADD_U32), HiVal)
            .addReg(CarryReg)
            .addReg(Op1H_Op0L_Reg)
            .setOperandDead(3); // Dead scc
 
        if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
          BuildMI(BB, MI, DL, TII->get(AMDGPU::S_ADD_U32), DestSub1)
              .addReg(HiVal)
              .addReg(Op1L_Op0H_Reg)
              .setOperandDead(3); // Dead scc
        }
        BuildRegSequence(BB, MI, DstReg, DestSub0, DestSub1);
        break;
      }
      case AMDGPU::V_ADD_F32_e64:
      case AMDGPU::V_ADD_F64_e64:
      case AMDGPU::V_ADD_F64_pseudo_e64:
      case AMDGPU::V_SUB_F32_e64: {
        bool is32BitOpc = is32bitWaveReduceOperation(Opc);
        const TargetRegisterClass *VregRC = TII->getRegClass(TII->get(Opc), 0);
        Register ActiveLanesVreg = MRI.createVirtualRegister(VregRC);
        Register DstVreg = MRI.createVirtualRegister(VregRC);
        // Get number of active lanes as a float val.
        BuildMI(BB, MI, DL,
                TII->get(is32BitOpc ? AMDGPU::V_CVT_F32_I32_e64
                                    : AMDGPU::V_CVT_F64_I32_e64),
                ActiveLanesVreg)
            .addReg(NewAccumulator->getOperand(0).getReg())
            .addImm(0)  // clamp
            .addImm(0); // output-modifier
 
        // Take negation of input for SUB reduction
        unsigned srcMod = (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
                           MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64)
                              ? SISrcMods::NEG
                              : SISrcMods::NONE;
        unsigned MulOpc = is32BitOpc ? AMDGPU::V_MUL_F32_e64
                          : ST.getGeneration() >= AMDGPUSubtarget::GFX12
                              ? AMDGPU::V_MUL_F64_pseudo_e64
                              : AMDGPU::V_MUL_F64_e64;
        auto DestVregInst = BuildMI(BB, MI, DL, TII->get(MulOpc),
                                    DstVreg)
                                .addImm(srcMod) // src0 modifier
                                .addReg(SrcReg)
                                .addImm(SISrcMods::NONE) // src1 modifier
                                .addReg(ActiveLanesVreg)
                                .addImm(SISrcMods::NONE)  // clamp
                                .addImm(SISrcMods::NONE); // output-mod
        if (is32BitOpc) {
          BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
              .addReg(DstVreg);
        } else {
          Register LaneValueLoReg =
              MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
          Register LaneValueHiReg =
              MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
          auto [Op1L, Op1H] =
              ExtractSubRegs(MI, DestVregInst->getOperand(0), VregRC, ST, MRI);
          // lane value input should be in an sgpr
          BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
                  LaneValueLoReg)
              .addReg(Op1L);
          BuildMI(BB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
                  LaneValueHiReg)
              .addReg(Op1H);
          NewAccumulator =
              BuildRegSequence(BB, MI, DstReg, LaneValueLoReg, LaneValueHiReg);
        }
      }
      }
      RetBB = &BB;
    }
    }
  } else {
    MachineBasicBlock::iterator I = BB.end();
    Register SrcReg = MI.getOperand(1).getReg();
    bool is32BitOpc = is32bitWaveReduceOperation(Opc);
    bool isFPOp = isFloatingPointWaveReduceOperation(Opc);
    bool NeedsMovDPP = !is32BitOpc;
    // Create virtual registers required for lowering.
    const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass();
    const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg);
    const TargetRegisterClass *SrcRegClass = MRI.getRegClass(SrcReg);
    bool IsWave32 = ST.isWave32();
    unsigned MovOpcForExec = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
    unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
    if (Stratergy == WAVE_REDUCE_STRATEGY::ITERATIVE ||
        !ST.hasDPP()) { // If target doesn't support DPP operations, default to
                        // iterative stratergy
 
      // To reduce the VGPR using iterative approach, we need to iterate
      // over all the active lanes. Lowering consists of ComputeLoop,
      // which iterate over only active lanes. We use copy of EXEC register
      // as induction variable and every active lane modifies it using bitset0
      // so that we will get the next active lane for next iteration.
 
      // Create Control flow for loop
      // Split MI's Machine Basic block into For loop
      auto [ComputeLoop, ComputeEnd] = splitBlockForLoop(MI, BB, true);
 
      Register LoopIterator = MRI.createVirtualRegister(WaveMaskRegClass);
      Register IdentityValReg = MRI.createVirtualRegister(DstRegClass);
      Register AccumulatorReg = MRI.createVirtualRegister(DstRegClass);
      Register ActiveBitsReg = MRI.createVirtualRegister(WaveMaskRegClass);
      Register NewActiveBitsReg = MRI.createVirtualRegister(WaveMaskRegClass);
      Register FF1Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
      Register LaneValueReg = MRI.createVirtualRegister(DstRegClass);
 
      // Create initial values of induction variable from Exec, Accumulator and
      // insert branch instr to newly created ComputeBlock
      BuildMI(BB, I, DL, TII->get(MovOpcForExec), LoopIterator).addReg(ExecReg);
      uint64_t IdentityValue =
          MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
              ? 0x0 // +0.0 for double sub reduction
              : getIdentityValueForWaveReduction(Opc);
      BuildMI(BB, I, DL,
              TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
                                  : AMDGPU::S_MOV_B64_IMM_PSEUDO),
              IdentityValReg)
          .addImm(IdentityValue);
      // clang-format off
      BuildMI(BB, I, DL, TII->get(AMDGPU::S_BRANCH))
          .addMBB(ComputeLoop);
      // clang-format on
 
      // Start constructing ComputeLoop
      I = ComputeLoop->begin();
      auto Accumulator =
          BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::PHI), AccumulatorReg)
              .addReg(IdentityValReg)
              .addMBB(&BB);
      auto ActiveBits =
          BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::PHI), ActiveBitsReg)
              .addReg(LoopIterator)
              .addMBB(&BB);
 
      I = ComputeLoop->end();
      MachineInstr *NewAccumulator;
      // Perform the computations
      unsigned SFFOpc =
          IsWave32 ? AMDGPU::S_FF1_I32_B32 : AMDGPU::S_FF1_I32_B64;
      BuildMI(*ComputeLoop, I, DL, TII->get(SFFOpc), FF1Reg)
          .addReg(ActiveBitsReg);
      if (is32BitOpc) {
        Register OpDstReg = DstReg;
        bool hasSrc0Modifier = AMDGPU::getNamedOperandIdx(
                                   Opc, AMDGPU::OpName::src0_modifiers) != -1;
        bool hasSrc1Modifier = AMDGPU::getNamedOperandIdx(
                                   Opc, AMDGPU::OpName::src1_modifiers) != -1;
        bool hasClamp =
            AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1;
        bool hasOpSel =
            AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel) != -1;
        bool hasOMod =
            AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1;
        BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
                LaneValueReg)
            .addReg(SrcReg)
            .addReg(FF1Reg);
        if (ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/true)) {
          // Get the Lane Value in VGPR to avoid the Constant Bus Restriction
          Register LaneValVgpr = MRI.createVirtualRegister(SrcRegClass);
          Register VgprResultReg = MRI.createVirtualRegister(SrcRegClass);
          BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::COPY), LaneValVgpr)
              .addReg(LaneValueReg);
          OpDstReg = VgprResultReg;
          LaneValueReg = LaneValVgpr;
        }
        auto OpInstr = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), OpDstReg);
        if (hasSrc0Modifier)
          OpInstr.addImm(SISrcMods::NONE); // src0 modifier
        OpInstr.addReg(AccumulatorReg);    // src0
        if (hasSrc1Modifier)
          OpInstr.addImm(SISrcMods::NONE); // src1 modifier
        OpInstr.addReg(LaneValueReg);      // src1
        if (hasClamp)
          OpInstr.addImm(0); // clamp
        if (hasOpSel)
          OpInstr.addImm(0); // opsel
        if (hasOMod)
          OpInstr.addImm(0); // omod
        if (ST.getInstrInfo()->isVALU(Opc, /*AllowLDSDMA=*/true)) {
          BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
                  DstReg)
              .addReg(OpDstReg);
        }
      } else {
        Register LaneValueLoReg =
            MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
        Register LaneValueHiReg =
            MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
        Register LaneValReg =
            MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
        auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
                                           MRI.getRegClass(SrcReg), ST, MRI);
        // lane value input should be in an sgpr
        BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
                LaneValueLoReg)
            .addReg(Op1L)
            .addReg(FF1Reg);
        BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::V_READLANE_B32),
                LaneValueHiReg)
            .addReg(Op1H)
            .addReg(FF1Reg);
        auto LaneValue = BuildRegSequence(*ComputeLoop, I, LaneValReg,
                                          LaneValueLoReg, LaneValueHiReg);
        switch (Opc) {
        case AMDGPU::S_OR_B64:
        case AMDGPU::S_AND_B64:
        case AMDGPU::S_XOR_B64: {
          NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg)
                               .addReg(Accumulator->getOperand(0).getReg())
                               .addReg(LaneValue->getOperand(0).getReg())
                               .setOperandDead(3); // Dead scc
          break;
        }
        case AMDGPU::V_CMP_GT_I64_e64:
        case AMDGPU::V_CMP_GT_U64_e64:
        case AMDGPU::V_CMP_LT_I64_e64:
        case AMDGPU::V_CMP_LT_U64_e64: {
          Register LaneMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
          Register ComparisonResultReg =
              MRI.createVirtualRegister(WaveMaskRegClass);
          int SrcIdx =
              AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
          const TargetRegisterClass *VregClass =
              TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
          Register AccumulatorVReg = MRI.createVirtualRegister(VregClass);
          auto [SrcReg0Sub0, SrcReg0Sub1] = ExtractSubRegs(
              MI, Accumulator->getOperand(0), VregClass, ST, MRI);
          BuildRegSequence(*ComputeLoop, I, AccumulatorVReg, SrcReg0Sub0,
                           SrcReg0Sub1);
          BuildMI(*ComputeLoop, I, DL, TII->get(Opc), LaneMaskReg)
              .addReg(LaneValue->getOperand(0).getReg())
              .addReg(AccumulatorVReg);
 
          unsigned AndOpc = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
          BuildMI(*ComputeLoop, I, DL, TII->get(AndOpc), ComparisonResultReg)
              .addReg(LaneMaskReg)
              .addReg(ActiveBitsReg);
 
          NewAccumulator = BuildMI(*ComputeLoop, I, DL,
                                   TII->get(AMDGPU::S_CSELECT_B64), DstReg)
                               .addReg(LaneValue->getOperand(0).getReg())
                               .addReg(Accumulator->getOperand(0).getReg());
          break;
        }
        case AMDGPU::V_MIN_F64_e64:
        case AMDGPU::V_MIN_NUM_F64_e64:
        case AMDGPU::V_MAX_F64_e64:
        case AMDGPU::V_MAX_NUM_F64_e64:
        case AMDGPU::V_ADD_F64_e64:
        case AMDGPU::V_ADD_F64_pseudo_e64: {
          int SrcIdx =
              AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src);
          const TargetRegisterClass *VregRC =
              TRI->getAllocatableClass(TII->getRegClass(MI.getDesc(), SrcIdx));
          Register AccumulatorVReg = MRI.createVirtualRegister(VregRC);
          Register DstVreg = MRI.createVirtualRegister(VregRC);
          Register LaneValLo =
              MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
          Register LaneValHi =
              MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
          BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::COPY), AccumulatorVReg)
              .addReg(Accumulator->getOperand(0).getReg());
          unsigned Modifier =
              MI.getOpcode() == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64
                  ? SISrcMods::NEG
                  : SISrcMods::NONE;
          auto DstVregInst =
              BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstVreg)
                  .addImm(Modifier) // src0 modifiers
                  .addReg(LaneValue->getOperand(0).getReg())
                  .addImm(SISrcMods::NONE) // src1 modifiers
                  .addReg(AccumulatorVReg)
                  .addImm(SISrcMods::NONE)  // clamp
                  .addImm(SISrcMods::NONE); // omod
          auto ReadLaneLo =
              BuildMI(*ComputeLoop, I, DL,
                      TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValLo);
          auto ReadLaneHi =
              BuildMI(*ComputeLoop, I, DL,
                      TII->get(AMDGPU::V_READFIRSTLANE_B32), LaneValHi);
          MachineBasicBlock::iterator Iters = *ReadLaneLo;
          auto [Op1L, Op1H] = ExtractSubRegs(*Iters, DstVregInst->getOperand(0),
                                             VregRC, ST, MRI);
          ReadLaneLo.addReg(Op1L);
          ReadLaneHi.addReg(Op1H);
          NewAccumulator =
              BuildRegSequence(*ComputeLoop, I, DstReg, LaneValLo, LaneValHi);
          break;
        }
        case AMDGPU::S_ADD_U64_PSEUDO:
        case AMDGPU::S_SUB_U64_PSEUDO: {
          NewAccumulator = BuildMI(*ComputeLoop, I, DL, TII->get(Opc), DstReg)
                               .addReg(Accumulator->getOperand(0).getReg())
                               .addReg(LaneValue->getOperand(0).getReg());
          ComputeLoop =
              expand64BitScalarArithmetic(*NewAccumulator, ComputeLoop);
          break;
        }
        }
      }
      // Manipulate the iterator to get the next active lane
      unsigned BITSETOpc =
          IsWave32 ? AMDGPU::S_BITSET0_B32 : AMDGPU::S_BITSET0_B64;
      BuildMI(*ComputeLoop, I, DL, TII->get(BITSETOpc), NewActiveBitsReg)
          .addReg(FF1Reg)
          .addReg(ActiveBitsReg);
 
      // Add phi nodes
      Accumulator.addReg(DstReg).addMBB(ComputeLoop);
      ActiveBits.addReg(NewActiveBitsReg).addMBB(ComputeLoop);
 
      // Creating branching
      MachineInstrBuilder SetSCCInstr;
      if (!ST.hasScalarCompareEq64()) {
        // For targets <= gfx7, use an S_OR_B32/B64 instruction to set SCC.
        Register LaneMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
        unsigned CMPOpc = IsWave32 ? AMDGPU::S_OR_B32 : AMDGPU::S_OR_B64;
        SetSCCInstr =
            BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc), LaneMaskReg);
      } else {
        unsigned CMPOpc =
            IsWave32 ? AMDGPU::S_CMP_LG_U32 : AMDGPU::S_CMP_LG_U64;
        SetSCCInstr = BuildMI(*ComputeLoop, I, DL, TII->get(CMPOpc));
      }
      SetSCCInstr.addReg(NewActiveBitsReg);
      if (ST.hasScalarCompareEq64())
        SetSCCInstr.addImm(0);
      else
        SetSCCInstr.addReg(NewActiveBitsReg);
      BuildMI(*ComputeLoop, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
          .addMBB(ComputeLoop);
 
      RetBB = ComputeEnd;
    } else {
      assert(ST.hasDPP() && "Sub Target does not support DPP Operations");
      MachineBasicBlock *CurrBB = &BB;
      Register SrcWithIdentity = MRI.createVirtualRegister(SrcRegClass);
      Register IdentityVGPR = MRI.createVirtualRegister(SrcRegClass);
      Register IdentitySGPR = MRI.createVirtualRegister(DstRegClass);
      Register DPPRowShr1 = MRI.createVirtualRegister(SrcRegClass);
      Register DPPRowShr2 = MRI.createVirtualRegister(SrcRegClass);
      Register DPPRowShr4 = MRI.createVirtualRegister(SrcRegClass);
      Register DPPRowShr8 = MRI.createVirtualRegister(SrcRegClass);
      Register RowBcast15 = MRI.createVirtualRegister(SrcRegClass);
      Register ReducedValSGPR = MRI.createVirtualRegister(DstRegClass);
      Register NegatedReducedVal = MRI.createVirtualRegister(DstRegClass);
      Register RowBcast31 = MRI.createVirtualRegister(SrcRegClass);
      Register UndefExec = MRI.createVirtualRegister(WaveMaskRegClass);
      Register FinalDPPResult;
      MachineInstr *SrcWithIdentityInstr;
      MachineInstr *LastBcastInstr;
      BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), UndefExec);
 
      uint64_t IdentityValue = getIdentityValueForWaveReduction(Opc);
      BuildMI(*CurrBB, MI, DL,
              TII->get(is32BitOpc ? AMDGPU::S_MOV_B32
                                  : AMDGPU::S_MOV_B64_IMM_PSEUDO),
              IdentitySGPR)
          .addImm(IdentityValue);
      auto IdentityCopyInstr =
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::COPY), IdentityVGPR)
              .addReg(IdentitySGPR);
      auto DPPClampOpcPair = getDPPOpcForWaveReduction(Opc, ST);
      unsigned DPPOpc = std::get<0>(DPPClampOpcPair);
      unsigned ClampOpc = std::get<1>(DPPClampOpcPair);
      auto BuildSetInactiveInstr = [&](Register Dst, Register Src0,
                                       Register Src1) {
        return BuildMI(BB, MI, DL, TII->get(AMDGPU::V_SET_INACTIVE_B32),
                       Dst)
            .addImm(0)          // src0 modifiers
            .addReg(Src0)       // src0
            .addImm(0)          // src1 modifiers
            .addReg(Src1)       // identity value for inactive lanes
            .addReg(UndefExec); // bool i1
      };
      auto BuildDPPMachineInstr = [&](Register Dst, Register Src,
                                      unsigned DPPCtrl) {
        auto DPPInstr =
            BuildMI(*CurrBB, MI, DL, TII->get(DPPOpc), Dst).addReg(Src); // old
        if (isFPOp && !NeedsMovDPP)
          DPPInstr.addImm(SISrcMods::NONE); // src0 modifier
        DPPInstr.addReg(Src);               // src0
        if (isFPOp && !NeedsMovDPP)
          DPPInstr.addImm(SISrcMods::NONE); // src1 modifier
        if (!NeedsMovDPP)
          DPPInstr.addReg(Src); // src1
        if (AMDGPU::getNamedOperandIdx(DPPOpc, AMDGPU::OpName::clamp) >= 0)
          DPPInstr.addImm(0); // clamp
        DPPInstr
            .addImm(DPPCtrl) // dpp-ctrl
            .addImm(0xf)     // row-mask
            .addImm(0xf)     // bank-mask
            .addImm(0);      // bound-control
      };
      auto BuildClampInstr = [&](Register Dst, Register Src0, Register Src1,
                                 bool isAddSub = false,
                                 bool needsCarryIn = false,
                                 Register CarryIn = Register()) {
        unsigned InstrOpc = ClampOpc;
        Register CarryOutReg = MRI.createVirtualRegister(WaveMaskRegClass);
        if (needsCarryIn)
          InstrOpc = AMDGPU::V_ADDC_U32_e64;
        auto ClampInstr = BuildMI(*CurrBB, MI, DL, TII->get(InstrOpc), Dst);
        if (isFPOp)
          ClampInstr.addImm(SISrcMods::NONE); // src0 mod
        if (isAddSub) {
          if (needsCarryIn)
            ClampInstr.addReg(CarryOutReg,
                              RegState::Define |
                                  RegState::Dead); // killed carry-out reg
          else
            ClampInstr.addReg(CarryOutReg, RegState::Define); // carry-out reg
        }
        ClampInstr.addReg(Src0);              // src0
        if (isFPOp)
          ClampInstr.addImm(SISrcMods::NONE); // src1 mod
        ClampInstr.addReg(Src1);              // src1
        if (needsCarryIn)
          ClampInstr.addReg(CarryIn, RegState::Kill); // carry-in reg
        if (AMDGPU::getNamedOperandIdx(InstrOpc, AMDGPU::OpName::clamp) >= 0)
          ClampInstr.addImm(0); // clamp
        if (isFPOp)
          ClampInstr.addImm(0); // omod
        LastBcastInstr = ClampInstr;
        return CarryOutReg;
      };
      auto BuildPostDPPInstr = [&](Register Src0, Register Src1) {
        bool isAddSubOpc =
            Opc == AMDGPU::S_ADD_U64_PSEUDO || Opc == AMDGPU::S_SUB_U64_PSEUDO;
        bool isBitWiseOpc = Opc == AMDGPU::S_AND_B64 ||
                            Opc == AMDGPU::S_OR_B64 || Opc == AMDGPU::S_XOR_B64;
        Register ReturnReg = MRI.createVirtualRegister(SrcRegClass);
        if (isAddSubOpc || isBitWiseOpc) {
          Register ResLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          Register ResHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          MachineOperand Src0Operand =
              MachineOperand::CreateReg(Src0, /*isDef=*/false);
          MachineOperand Src1Operand =
              MachineOperand::CreateReg(Src1, /*isDef=*/false);
          auto [Src0Lo, Src0Hi] =
              ExtractSubRegs(MI, Src0Operand, SrcRegClass, ST, MRI);
          auto [Src1Lo, Src1Hi] =
              ExtractSubRegs(MI, Src1Operand, SrcRegClass, ST, MRI);
          Register CarryReg = BuildClampInstr(
              ResLo, Src0Lo, Src1Lo, isAddSubOpc, /*needsCarryIn*/ false);
          BuildClampInstr(ResHi, Src0Hi, Src1Hi, isAddSubOpc,
                          /*needsCarryIn*/ isAddSubOpc, CarryReg);
          BuildRegSequence(*CurrBB, MI, ReturnReg, ResLo, ResHi);
        } else {
          if (isFPOp) {
            BuildMI(*CurrBB, MI, DL, TII->get(Opc), ReturnReg)
                .addImm(SISrcMods::NONE) // src0 modifiers
                .addReg(Src0)
                .addImm(SISrcMods::NONE) // src1 modifiers
                .addReg(Src1)
                .addImm(SISrcMods::NONE)  // clamp
                .addImm(SISrcMods::NONE); // omod
          } else {
            Register CmpMaskReg = MRI.createVirtualRegister(WaveMaskRegClass);
            BuildMI(*CurrBB, MI, DL, TII->get(Opc), CmpMaskReg)
                .addReg(Src0)  // src0
                .addReg(Src1); // src1
            LastBcastInstr =
                BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B64_PSEUDO),
                        ReturnReg)
                    .addReg(Src1)        // src0
                    .addReg(Src0)        // src1
                    .addReg(CmpMaskReg); // src2
            expand64BitV_CNDMASK(*LastBcastInstr, CurrBB);
          }
        }
        return ReturnReg;
      };
 
      // Set inactive lanes to the identity value.
      if (is32BitOpc) {
        SrcWithIdentityInstr =
            BuildSetInactiveInstr(SrcWithIdentity, SrcReg, IdentityVGPR);
      } else {
        Register SrcWithIdentitylo =
            MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
        Register SrcWithIdentityhi =
            MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
        auto [Reg0Sub0, Reg0Sub1] = ExtractSubRegs(
            MI, IdentityCopyInstr->getOperand(0), SrcRegClass, ST, MRI);
        auto [SrcReg0Sub0, SrcReg0Sub1] =
            ExtractSubRegs(MI, MI.getOperand(1), SrcRegClass, ST, MRI);
        MachineInstr *SetInactiveLoInstr =
            BuildSetInactiveInstr(SrcWithIdentitylo, SrcReg0Sub0, Reg0Sub0);
        MachineInstr *SetInactiveHiInstr =
            BuildSetInactiveInstr(SrcWithIdentityhi, SrcReg0Sub1, Reg0Sub1);
        SrcWithIdentityInstr =
            BuildRegSequence(*CurrBB, MI, SrcWithIdentity,
                             SetInactiveLoInstr->getOperand(0).getReg(),
                             SetInactiveHiInstr->getOperand(0).getReg());
      }
      // DPP reduction
      Register SrcWithIdentityReg =
          SrcWithIdentityInstr->getOperand(0).getReg();
      BuildDPPMachineInstr(DPPRowShr1, SrcWithIdentityReg,
                           AMDGPU::DPP::ROW_SHR_FIRST);
      if (NeedsMovDPP)
        DPPRowShr1 = BuildPostDPPInstr(SrcWithIdentityReg, DPPRowShr1);
 
      BuildDPPMachineInstr(DPPRowShr2, DPPRowShr1,
                           (AMDGPU::DPP::ROW_SHR_FIRST + 1));
      if (NeedsMovDPP)
        DPPRowShr2 = BuildPostDPPInstr(DPPRowShr1, DPPRowShr2);
 
      BuildDPPMachineInstr(DPPRowShr4, DPPRowShr2,
                           (AMDGPU::DPP::ROW_SHR_FIRST + 3));
      if (NeedsMovDPP)
        DPPRowShr4 = BuildPostDPPInstr(DPPRowShr2, DPPRowShr4);
 
      BuildDPPMachineInstr(DPPRowShr8, DPPRowShr4,
                           (AMDGPU::DPP::ROW_SHR_FIRST + 7));
      if (NeedsMovDPP)
        DPPRowShr8 = BuildPostDPPInstr(DPPRowShr4, DPPRowShr8);
 
      if (ST.hasDPPBroadcasts()) {
        BuildDPPMachineInstr(RowBcast15, DPPRowShr8, AMDGPU::DPP::BCAST15);
        if (NeedsMovDPP)
          RowBcast15 = BuildPostDPPInstr(DPPRowShr8, RowBcast15);
      } else {
        // magic constant: 0x1E0
        // To Set BIT_MODE : bit 15 = 0
        // XOR mask : bit [14:10] = 0
        // OR mask : bit [9:5] = 15
        // AND mask : bit [4:0] = 0
        if (is32BitOpc) {
          Register SwizzledValue =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_SWIZZLE_B32),
                  SwizzledValue)
              .addReg(DPPRowShr8) // addr
              .addImm(0x1E0)      // swizzle offset (i16)
              .addImm(0x0);       // gds (i1)
          BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue);
        } else {
          Register SwizzledValuelo =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          Register SwizzledValuehi =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          Register SwizzledValue64 = MRI.createVirtualRegister(SrcRegClass);
          MachineOperand DPPRowShr8Op =
              MachineOperand::CreateReg(DPPRowShr8, /*isDef=*/false);
          auto [Op1L, Op1H] =
              ExtractSubRegs(MI, DPPRowShr8Op, SrcRegClass, ST, MRI);
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_SWIZZLE_B32),
                  SwizzledValuelo)
              .addReg(Op1L)  // addr
              .addImm(0x1E0) // swizzle offset (i16)
              .addImm(0x0);  // gds (i1)
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_SWIZZLE_B32),
                  SwizzledValuehi)
              .addReg(Op1H)  // addr
              .addImm(0x1E0) // swizzle offset (i16)
              .addImm(0x0);  // gds (i1)
          BuildRegSequence(*CurrBB, MI, SwizzledValue64, SwizzledValuelo,
                           SwizzledValuehi);
          if (NeedsMovDPP)
            RowBcast15 = BuildPostDPPInstr(DPPRowShr8, SwizzledValue64);
          else
            BuildClampInstr(RowBcast15, DPPRowShr8, SwizzledValue64);
        }
      }
      FinalDPPResult = RowBcast15;
      if (!IsWave32) {
        if (ST.hasDPPBroadcasts()) {
          BuildDPPMachineInstr(RowBcast31, RowBcast15, AMDGPU::DPP::BCAST31);
          if (NeedsMovDPP)
            RowBcast31 = BuildPostDPPInstr(RowBcast15, RowBcast31);
        } else {
          Register ShiftedThreadID =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          Register PermuteByteOffset =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          Register PermutedValue = MRI.createVirtualRegister(SrcRegClass);
          Register Lane32Offset =
              MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
          Register WordSizeConst =
              MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
          Register ThreadIDRegLo =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          Register ThreadIDReg =
              MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
          // Get the thread ID.
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
                  ThreadIDRegLo)
              .addImm(-1)
              .addImm(0);
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
                  ThreadIDReg)
              .addImm(-1)
              .addReg(ThreadIDRegLo);
          // shift each lane over by 32 positions, so value in 31st lane is
          // present in 63rd lane.
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), Lane32Offset)
              .addImm(0x20);
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64),
                  ShiftedThreadID)
              .addReg(ThreadIDReg)
              .addReg(Lane32Offset)
              .addImm(0); // clamp
          // multiply by reg size.
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), WordSizeConst)
              .addImm(0x4);
          BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_MUL_LO_U32_e64),
                  PermuteByteOffset)
              .addReg(WordSizeConst)
              .addReg(ShiftedThreadID);
          // Permute the lanes
          if (is32BitOpc) {
            BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_PERMUTE_B32),
                    PermutedValue)
                .addReg(PermuteByteOffset) // addr
                .addReg(RowBcast15)        // data
                .addImm(0);                // offset
          } else {
            Register PermutedValuelo =
                MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
            Register PermutedValuehi =
                MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
            MachineOperand RowBcast15Op =
                MachineOperand::CreateReg(RowBcast15, /*isDef=*/false);
            auto [RowBcast15Lo, RowBcast15Hi] =
                ExtractSubRegs(MI, RowBcast15Op, SrcRegClass, ST, MRI);
            BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_PERMUTE_B32),
                    PermutedValuelo)
                .addReg(PermuteByteOffset) // addr
                .addReg(RowBcast15Lo)      // swizzle offset (i16)
                .addImm(0x0);              // gds (i1)
            BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::DS_PERMUTE_B32),
                    PermutedValuehi)
                .addReg(PermuteByteOffset) // addr
                .addReg(RowBcast15Hi)      // swizzle offset (i16)
                .addImm(0x0);              // gds (i1)
            BuildRegSequence(*CurrBB, MI, PermutedValue, PermutedValuelo,
                             PermutedValuehi);
          }
          if (NeedsMovDPP)
            RowBcast31 = BuildPostDPPInstr(RowBcast15, PermutedValue);
          else
            BuildClampInstr(RowBcast31, RowBcast15, PermutedValue);
        }
        FinalDPPResult = RowBcast31;
      }
      if (MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32 ||
          MIOpc == AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64) {
        Register NegatedValVGPR = MRI.createVirtualRegister(SrcRegClass);
        // Opc for f32 reduction is V_SUB_F32.
        // For f64, there is no equivalent V_SUB_F64 opcode, so use
        // V_ADD_F64/V_ADD_F64_pseudo, and negate the second operand.
        BuildMI(*CurrBB, MI, DL, TII->get(Opc),
                NegatedValVGPR)
            .addImm(SISrcMods::NONE)                               // src0 mods
            .addReg(IdentityVGPR)                                  // src0
            .addImm(is32BitOpc ? SISrcMods::NONE : SISrcMods::NEG) // src1 mods
            .addReg(IsWave32 ? RowBcast15 : RowBcast31)            // src1
            .addImm(SISrcMods::NONE)                               // clamp
            .addImm(SISrcMods::NONE);                              // omod
        FinalDPPResult = NegatedValVGPR;
      }
      // The final reduced value is in the last lane.
      if (is32BitOpc) {
        BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
                ReducedValSGPR)
            .addReg(FinalDPPResult)
            .addImm(ST.getWavefrontSize() - 1);
      } else {
        Register LaneValueLoReg =
            MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
        Register LaneValueHiReg =
            MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
        const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
        MachineOperand FinalDPPResultOperand =
            MachineOperand::CreateReg(FinalDPPResult, /*isDef=*/false);
        auto [Op1L, Op1H] =
            ExtractSubRegs(MI, FinalDPPResultOperand, SrcRC, ST, MRI);
        // lane value input should be in an sgpr
        BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
                LaneValueLoReg)
            .addReg(Op1L)
            .addImm(ST.getWavefrontSize() - 1);
        BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
                LaneValueHiReg)
            .addReg(Op1H)
            .addImm(ST.getWavefrontSize() - 1);
        BuildRegSequence(*CurrBB, MI, ReducedValSGPR, LaneValueLoReg,
                         LaneValueHiReg);
      }
      if (Opc == AMDGPU::S_SUB_I32) {
        BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedReducedVal)
            .addImm(0)
            .addReg(ReducedValSGPR);
      } else if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
        auto NegatedValInstr =
            BuildMI(*CurrBB, MI, DL, TII->get(Opc), NegatedReducedVal)
                .addImm(0)
                .addReg(ReducedValSGPR);
        CurrBB = expand64BitScalarArithmetic(*NegatedValInstr, CurrBB);
      }
      // Mark the final result as a whole-wave-mode calculation.
      BuildMI(*CurrBB, MI, DL, TII->get(AMDGPU::STRICT_WWM), DstReg)
          .addReg(Opc == AMDGPU::S_SUB_I32 || Opc == AMDGPU::S_SUB_U64_PSEUDO
                      ? NegatedReducedVal
                      : ReducedValSGPR);
      RetBB = CurrBB;
    }
  }
  MI.eraseFromParent();
  return RetBB;
}
 
MachineBasicBlock *
SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                                              MachineBasicBlock *BB) const {
  MachineFunction *MF = BB->getParent();
  SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
  const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
  MachineRegisterInfo &MRI = MF->getRegInfo();
  const DebugLoc &DL = MI.getDebugLoc();
 
  switch (MI.getOpcode()) {
  case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_U32);
  case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_LT_U64_e64);
  case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_I32);
  case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_LT_I64_e64);
  case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_MIN_F32_e64);
  case AMDGPU::WAVE_REDUCE_FMIN_PSEUDO_F64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(),
                           ST.getGeneration() >= AMDGPUSubtarget::GFX12
                               ? AMDGPU::V_MIN_NUM_F64_e64
                               : AMDGPU::V_MIN_F64_e64);
  case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_U32);
  case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_GT_U64_e64);
  case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_I32);
  case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_CMP_GT_I64_e64);
  case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_MAX_F32_e64);
  case AMDGPU::WAVE_REDUCE_FMAX_PSEUDO_F64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(),
                           ST.getGeneration() >= AMDGPUSubtarget::GFX12
                               ? AMDGPU::V_MAX_NUM_F64_e64
                               : AMDGPU::V_MAX_F64_e64);
  case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_ADD_I32);
  case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_U64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_ADD_U64_PSEUDO);
  case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_ADD_F32_e64);
  case AMDGPU::WAVE_REDUCE_FADD_PSEUDO_F64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(),
                           ST.getGeneration() >= AMDGPUSubtarget::GFX12
                               ? AMDGPU::V_ADD_F64_pseudo_e64
                               : AMDGPU::V_ADD_F64_e64);
  case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_SUB_I32);
  case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_U64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_SUB_U64_PSEUDO);
  case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::V_SUB_F32_e64);
  case AMDGPU::WAVE_REDUCE_FSUB_PSEUDO_F64:
    // There is no S/V_SUB_F64 opcode. Double type subtraction is expanded as
    // fadd + neg, by setting the NEG bit in the instruction.
    return lowerWaveReduce(MI, *BB, *getSubtarget(),
                           ST.getGeneration() >= AMDGPUSubtarget::GFX12
                               ? AMDGPU::V_ADD_F64_pseudo_e64
                               : AMDGPU::V_ADD_F64_e64);
  case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_AND_B32);
  case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_AND_B64);
  case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_OR_B32);
  case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_OR_B64);
  case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_XOR_B32);
  case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B64:
    return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_XOR_B64);
  case AMDGPU::S_UADDO_PSEUDO:
  case AMDGPU::S_USUBO_PSEUDO: {
    MachineOperand &Dest0 = MI.getOperand(0);
    MachineOperand &Dest1 = MI.getOperand(1);
    MachineOperand &Src0 = MI.getOperand(2);
    MachineOperand &Src1 = MI.getOperand(3);
 
    unsigned Opc = (MI.getOpcode() == AMDGPU::S_UADDO_PSEUDO)
                       ? AMDGPU::S_ADD_U32
                       : AMDGPU::S_SUB_U32;
    // clang-format off
    BuildMI(*BB, MI, DL, TII->get(Opc), Dest0.getReg())
        .add(Src0)
        .add(Src1);
    // clang-format on
 
    unsigned SelOpc =
        Subtarget->isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
    BuildMI(*BB, MI, DL, TII->get(SelOpc), Dest1.getReg()).addImm(-1).addImm(0);
 
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::S_ADD_U64_PSEUDO:
  case AMDGPU::S_SUB_U64_PSEUDO: {
    return expand64BitScalarArithmetic(MI, BB);
  }
  case AMDGPU::V_ADD_U64_PSEUDO:
  case AMDGPU::V_SUB_U64_PSEUDO: {
    bool IsAdd = (MI.getOpcode() == AMDGPU::V_ADD_U64_PSEUDO);
 
    MachineOperand &Dest = MI.getOperand(0);
    MachineOperand &Src0 = MI.getOperand(1);
    MachineOperand &Src1 = MI.getOperand(2);
 
    if (ST.hasAddSubU64Insts()) {
      auto I = BuildMI(*BB, MI, DL,
                       TII->get(IsAdd ? AMDGPU::V_ADD_U64_e64
                                      : AMDGPU::V_SUB_U64_e64),
                       Dest.getReg())
                   .add(Src0)
                   .add(Src1)
                   .addImm(0); // clamp
      TII->legalizeOperands(*I);
      MI.eraseFromParent();
      return BB;
    }
 
    if (IsAdd && ST.hasLshlAddU64Inst()) {
      auto Add = BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_LSHL_ADD_U64_e64),
                         Dest.getReg())
                     .add(Src0)
                     .addImm(0)
                     .add(Src1);
      TII->legalizeOperands(*Add);
      MI.eraseFromParent();
      return BB;
    }
 
    const auto *CarryRC = TRI->getWaveMaskRegClass();
 
    Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
    Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
    Register CarryReg = MRI.createVirtualRegister(CarryRC);
    Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
 
    const TargetRegisterClass *Src0RC = Src0.isReg()
                                            ? MRI.getRegClass(Src0.getReg())
                                            : &AMDGPU::VReg_64RegClass;
    const TargetRegisterClass *Src1RC = Src1.isReg()
                                            ? MRI.getRegClass(Src1.getReg())
                                            : &AMDGPU::VReg_64RegClass;
 
    const TargetRegisterClass *Src0SubRC =
        TRI->getSubRegisterClass(Src0RC, AMDGPU::sub0);
    const TargetRegisterClass *Src1SubRC =
        TRI->getSubRegisterClass(Src1RC, AMDGPU::sub1);
 
    MachineOperand SrcReg0Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, Src0RC, AMDGPU::sub0, Src0SubRC);
    MachineOperand SrcReg1Sub0 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, Src1RC, AMDGPU::sub0, Src1SubRC);
 
    MachineOperand SrcReg0Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src0, Src0RC, AMDGPU::sub1, Src0SubRC);
    MachineOperand SrcReg1Sub1 = TII->buildExtractSubRegOrImm(
        MI, MRI, Src1, Src1RC, AMDGPU::sub1, Src1SubRC);
 
    unsigned LoOpc =
        IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64;
    MachineInstr *LoHalf = BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0)
                               .addReg(CarryReg, RegState::Define)
                               .add(SrcReg0Sub0)
                               .add(SrcReg1Sub0)
                               .addImm(0); // clamp bit
 
    unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
    MachineInstr *HiHalf =
        BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1)
            .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
            .add(SrcReg0Sub1)
            .add(SrcReg1Sub1)
            .addReg(CarryReg, RegState::Kill)
            .addImm(0); // clamp bit
 
    BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
        .addReg(DestSub0)
        .addImm(AMDGPU::sub0)
        .addReg(DestSub1)
        .addImm(AMDGPU::sub1);
    TII->legalizeOperands(*LoHalf);
    TII->legalizeOperands(*HiHalf);
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::S_ADD_CO_PSEUDO:
  case AMDGPU::S_SUB_CO_PSEUDO: {
    // This pseudo has a chance to be selected
    // only from uniform add/subcarry node. All the VGPR operands
    // therefore assumed to be splat vectors.
    MachineBasicBlock::iterator MII = MI;
    MachineOperand &Dest = MI.getOperand(0);
    MachineOperand &CarryDest = MI.getOperand(1);
    MachineOperand &Src0 = MI.getOperand(2);
    MachineOperand &Src1 = MI.getOperand(3);
    MachineOperand &Src2 = MI.getOperand(4);
    if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) {
      Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
      BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0)
          .addReg(Src0.getReg());
      Src0.setReg(RegOp0);
    }
    if (Src1.isReg() && TRI->isVectorRegister(MRI, Src1.getReg())) {
      Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
      BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1)
          .addReg(Src1.getReg());
      Src1.setReg(RegOp1);
    }
    Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
    if (TRI->isVectorRegister(MRI, Src2.getReg())) {
      BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2)
          .addReg(Src2.getReg());
      Src2.setReg(RegOp2);
    }
 
    if (ST.isWave64()) {
      if (ST.hasScalarCompareEq64()) {
        BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64))
            .addReg(Src2.getReg())
            .addImm(0);
      } else {
        const TargetRegisterClass *Src2RC = MRI.getRegClass(Src2.getReg());
        const TargetRegisterClass *SubRC =
            TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
        MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
            MII, MRI, Src2, Src2RC, AMDGPU::sub0, SubRC);
        MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
            MII, MRI, Src2, Src2RC, AMDGPU::sub1, SubRC);
        Register Src2_32 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
 
        BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_OR_B32), Src2_32)
            .add(Src2Sub0)
            .add(Src2Sub1);
 
        BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
            .addReg(Src2_32, RegState::Kill)
            .addImm(0);
      }
    } else {
      BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
          .addReg(Src2.getReg())
          .addImm(0);
    }
 
    unsigned Opc = MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO
                       ? AMDGPU::S_ADDC_U32
                       : AMDGPU::S_SUBB_U32;
 
    BuildMI(*BB, MII, DL, TII->get(Opc), Dest.getReg()).add(Src0).add(Src1);
 
    unsigned SelOpc =
        ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
 
    BuildMI(*BB, MII, DL, TII->get(SelOpc), CarryDest.getReg())
        .addImm(-1)
        .addImm(0);
 
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::SI_INIT_M0: {
    MachineOperand &M0Init = MI.getOperand(0);
    BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
            TII->get(M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32),
            AMDGPU::M0)
        .add(M0Init);
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
    // Set SCC to true, in case the barrier instruction gets converted to a NOP.
    BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
            TII->get(AMDGPU::S_CMP_EQ_U32))
        .addImm(0)
        .addImm(0);
    return BB;
  }
  case AMDGPU::GET_GROUPSTATICSIZE: {
    assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
           getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_MOV_B32))
        .add(MI.getOperand(0))
        .addImm(MFI->getLDSSize());
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::GET_SHADERCYCLESHILO: {
    assert(MF->getSubtarget<GCNSubtarget>().hasShaderCyclesHiLoRegisters());
    // The algorithm is:
    //
    // hi1 = getreg(SHADER_CYCLES_HI)
    // lo1 = getreg(SHADER_CYCLES_LO)
    // hi2 = getreg(SHADER_CYCLES_HI)
    //
    // If hi1 == hi2 then there was no overflow and the result is hi2:lo1.
    // Otherwise there was overflow and the result is hi2:0. In both cases the
    // result should represent the actual time at some point during the sequence
    // of three getregs.
    using namespace AMDGPU::Hwreg;
    Register RegHi1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegHi1)
        .addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
    Register RegLo1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegLo1)
        .addImm(HwregEncoding::encode(ID_SHADER_CYCLES, 0, 32));
    Register RegHi2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_GETREG_B32), RegHi2)
        .addImm(HwregEncoding::encode(ID_SHADER_CYCLES_HI, 0, 32));
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CMP_EQ_U32))
        .addReg(RegHi1)
        .addReg(RegHi2);
    Register RegLo = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CSELECT_B32), RegLo)
        .addReg(RegLo1)
        .addImm(0);
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE))
        .add(MI.getOperand(0))
        .addReg(RegLo)
        .addImm(AMDGPU::sub0)
        .addReg(RegHi2)
        .addImm(AMDGPU::sub1);
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::SI_INDIRECT_SRC_V1:
  case AMDGPU::SI_INDIRECT_SRC_V2:
  case AMDGPU::SI_INDIRECT_SRC_V3:
  case AMDGPU::SI_INDIRECT_SRC_V4:
  case AMDGPU::SI_INDIRECT_SRC_V5:
  case AMDGPU::SI_INDIRECT_SRC_V6:
  case AMDGPU::SI_INDIRECT_SRC_V7:
  case AMDGPU::SI_INDIRECT_SRC_V8:
  case AMDGPU::SI_INDIRECT_SRC_V9:
  case AMDGPU::SI_INDIRECT_SRC_V10:
  case AMDGPU::SI_INDIRECT_SRC_V11:
  case AMDGPU::SI_INDIRECT_SRC_V12:
  case AMDGPU::SI_INDIRECT_SRC_V16:
  case AMDGPU::SI_INDIRECT_SRC_V32:
    return emitIndirectSrc(MI, *BB, *getSubtarget());
  case AMDGPU::SI_INDIRECT_DST_V1:
  case AMDGPU::SI_INDIRECT_DST_V2:
  case AMDGPU::SI_INDIRECT_DST_V3:
  case AMDGPU::SI_INDIRECT_DST_V4:
  case AMDGPU::SI_INDIRECT_DST_V5:
  case AMDGPU::SI_INDIRECT_DST_V6:
  case AMDGPU::SI_INDIRECT_DST_V7:
  case AMDGPU::SI_INDIRECT_DST_V8:
  case AMDGPU::SI_INDIRECT_DST_V9:
  case AMDGPU::SI_INDIRECT_DST_V10:
  case AMDGPU::SI_INDIRECT_DST_V11:
  case AMDGPU::SI_INDIRECT_DST_V12:
  case AMDGPU::SI_INDIRECT_DST_V16:
  case AMDGPU::SI_INDIRECT_DST_V32:
    return emitIndirectDst(MI, *BB, *getSubtarget());
  case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
  case AMDGPU::SI_KILL_I1_PSEUDO:
    return splitKillBlock(MI, BB);
  case AMDGPU::V_CNDMASK_B64_PSEUDO: {
    expand64BitV_CNDMASK(MI, BB);
    return BB;
  }
  case AMDGPU::SI_BR_UNDEF: {
    MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
                           .add(MI.getOperand(0));
    Br->getOperand(1).setIsUndef(); // read undef SCC
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::ADJCALLSTACKUP:
  case AMDGPU::ADJCALLSTACKDOWN: {
    const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
    MachineInstrBuilder MIB(*MF, &MI);
    MIB.addReg(Info->getStackPtrOffsetReg(), RegState::ImplicitDefine)
        .addReg(Info->getStackPtrOffsetReg(), RegState::Implicit);
    return BB;
  }
  case AMDGPU::SI_CALL_ISEL: {
    unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF);
 
    MachineInstrBuilder MIB;
    MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg);
 
    for (const MachineOperand &MO : MI.operands())
      MIB.add(MO);
 
    MIB.cloneMemRefs(MI);
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::V_ADD_CO_U32_e32:
  case AMDGPU::V_SUB_CO_U32_e32:
  case AMDGPU::V_SUBREV_CO_U32_e32: {
    // TODO: Define distinct V_*_I32_Pseudo instructions instead.
    unsigned Opc = MI.getOpcode();
 
    bool NeedClampOperand = false;
    if (TII->pseudoToMCOpcode(Opc) == -1) {
      Opc = AMDGPU::getVOPe64(Opc);
      NeedClampOperand = true;
    }
 
    auto I = BuildMI(*BB, MI, DL, TII->get(Opc), MI.getOperand(0).getReg());
    if (TII->isVOP3(*I)) {
      I.addReg(TRI->getVCC(), RegState::Define);
    }
    I.add(MI.getOperand(1)).add(MI.getOperand(2));
    if (NeedClampOperand)
      I.addImm(0); // clamp bit for e64 encoding
 
    TII->legalizeOperands(*I);
 
    MI.eraseFromParent();
    return BB;
  }
  case AMDGPU::V_ADDC_U32_e32:
  case AMDGPU::V_SUBB_U32_e32:
  case AMDGPU::V_SUBBREV_U32_e32:
    // These instructions have an implicit use of vcc which counts towards the
    // constant bus limit.
    TII->legalizeOperands(MI);
    return BB;
  case AMDGPU::DS_GWS_INIT:
  case AMDGPU::DS_GWS_SEMA_BR:
  case AMDGPU::DS_GWS_BARRIER:
  case AMDGPU::DS_GWS_SEMA_V:
  case AMDGPU::DS_GWS_SEMA_P:
  case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
    // A s_waitcnt 0 is required to be the instruction immediately following.
    if (getSubtarget()->hasGWSAutoReplay()) {
      bundleInstWithWaitcnt(MI);
      return BB;
    }
 
    return emitGWSMemViolTestLoop(MI, BB);
  case AMDGPU::S_SETREG_B32: {
    // Try to optimize cases that only set the denormal mode or rounding mode.
    //
    // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
    // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
    // instead.
    //
    // FIXME: This could be predicates on the immediate, but tablegen doesn't
    // allow you to have a no side effect instruction in the output of a
    // sideeffecting pattern.
    auto [ID, Offset, Width] =
        AMDGPU::Hwreg::HwregEncoding::decode(MI.getOperand(1).getImm());
    if (ID != AMDGPU::Hwreg::ID_MODE)
      return BB;
 
    const unsigned WidthMask = maskTrailingOnes<unsigned>(Width);
    const unsigned SetMask = WidthMask << Offset;
 
    if (getSubtarget()->hasDenormModeInst()) {
      unsigned SetDenormOp = 0;
      unsigned SetRoundOp = 0;
 
      // The dedicated instructions can only set the whole denorm or round mode
      // at once, not a subset of bits in either.
      if (SetMask ==
          (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
        // If this fully sets both the round and denorm mode, emit the two
        // dedicated instructions for these.
        SetRoundOp = AMDGPU::S_ROUND_MODE;
        SetDenormOp = AMDGPU::S_DENORM_MODE;
      } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
        SetRoundOp = AMDGPU::S_ROUND_MODE;
      } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
        SetDenormOp = AMDGPU::S_DENORM_MODE;
      }
 
      if (SetRoundOp || SetDenormOp) {
        MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg());
        if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) {
          unsigned ImmVal = Def->getOperand(1).getImm();
          if (SetRoundOp) {
            BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetRoundOp))
                .addImm(ImmVal & 0xf);
 
            // If we also have the denorm mode, get just the denorm mode bits.
            ImmVal >>= 4;
          }
 
          if (SetDenormOp) {
            BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetDenormOp))
                .addImm(ImmVal & 0xf);
          }
 
          MI.eraseFromParent();
          return BB;
        }
      }
    }
 
    // If only FP bits are touched, used the no side effects pseudo.
    if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
                    AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
      MI.setDesc(TII->get(AMDGPU::S_SETREG_B32_mode));
 
    return BB;
  }
  case AMDGPU::S_INVERSE_BALLOT_U32:
  case AMDGPU::S_INVERSE_BALLOT_U64:
    // These opcodes only exist to let SIFixSGPRCopies insert a readfirstlane if
    // necessary. After that they are equivalent to a COPY.
    MI.setDesc(TII->get(AMDGPU::COPY));
    return BB;
  case AMDGPU::ENDPGM_TRAP: {
    if (BB->succ_empty() && std::next(MI.getIterator()) == BB->end()) {
      MI.setDesc(TII->get(AMDGPU::S_ENDPGM));
      MI.addOperand(MachineOperand::CreateImm(0));
      return BB;
    }
 
    // We need a block split to make the real endpgm a terminator. We also don't
    // want to break phis in successor blocks, so we can't just delete to the
    // end of the block.
 
    MachineBasicBlock *SplitBB = BB->splitAt(MI, false /*UpdateLiveIns*/);
    MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
    MF->push_back(TrapBB);
    // clang-format off
    BuildMI(*TrapBB, TrapBB->end(), DL, TII->get(AMDGPU::S_ENDPGM))
        .addImm(0);
    BuildMI(*BB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
        .addMBB(TrapBB);
    // clang-format on
 
    BB->addSuccessor(TrapBB);
    MI.eraseFromParent();
    return SplitBB;
  }
  case AMDGPU::SIMULATED_TRAP: {
    assert(Subtarget->hasPrivEnabledTrap2NopBug());
    MachineBasicBlock *SplitBB =
        TII->insertSimulatedTrap(MRI, *BB, MI, MI.getDebugLoc());
    MI.eraseFromParent();
    return SplitBB;
  }
  case AMDGPU::SI_TCRETURN_GFX_WholeWave:
  case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN: {
    assert(MFI->isWholeWaveFunction());
 
    // During ISel, it's difficult to propagate the original EXEC mask to use as
    // an input to SI_WHOLE_WAVE_FUNC_RETURN. Set it up here instead.
    MachineInstr *Setup = TII->getWholeWaveFunctionSetup(*BB->getParent());
    assert(Setup && "Couldn't find SI_SETUP_WHOLE_WAVE_FUNC");
    Register OriginalExec = Setup->getOperand(0).getReg();
    MF->getRegInfo().clearKillFlags(OriginalExec);
    MI.getOperand(0).setReg(OriginalExec);
    return BB;
  }
  case AMDGPU::V_DOT2_F32_F16:
  case AMDGPU::V_DOT2_F32_BF16: {
    // Hint RA to assign dst and src2 the same physical register.
    // For targets without VOP2, but with VOPD, variant of the instruction this
    // is one of the conditions to attempt converting VOP3P to VOPD.
    MRI.setSimpleHint(MI.getOperand(0).getReg(), MI.getOperand(6).getReg());
    return BB;
  }
  default:
    if (TII->isImage(MI) || TII->isMUBUF(MI)) {
      if (!MI.mayStore())
        AddMemOpInit(MI);
      return BB;
    }
    return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
  }
}
 
bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
  // This currently forces unfolding various combinations of fsub into fma with
  // free fneg'd operands. As long as we have fast FMA (controlled by
  // isFMAFasterThanFMulAndFAdd), we should perform these.
 
  // When fma is quarter rate, for f64 where add / sub are at best half rate,
  // most of these combines appear to be cycle neutral but save on instruction
  // count / code size.
  return true;
}
 
bool SITargetLowering::enableAggressiveFMAFusion(LLT Ty) const { return true; }
 
EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
                                         EVT VT) const {
  if (!VT.isVector()) {
    return MVT::i1;
  }
  return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
}
 
MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
  // TODO: Should i16 be used always if legal? For now it would force VALU
  // shifts.
  return (VT == MVT::i16) ? MVT::i16 : MVT::i32;
}
 
LLT SITargetLowering::getPreferredShiftAmountTy(LLT Ty) const {
  return (Ty.getScalarSizeInBits() <= 16 && Subtarget->has16BitInsts())
             ? Ty.changeElementSize(16)
             : Ty.changeElementSize(32);
}
 
// Answering this is somewhat tricky and depends on the specific device which
// have different rates for fma or all f64 operations.
//
// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
// regardless of which device (although the number of cycles differs between
// devices), so it is always profitable for f64.
//
// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
// only on full rate devices. Normally, we should prefer selecting v_mad_f32
// which we can always do even without fused FP ops since it returns the same
// result as the separate operations and since it is always full
// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
// however does not support denormals, so we do report fma as faster if we have
// a fast fma device and require denormals.
//
bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT,
                                                  DenormalFPEnv FPEnv) const {
  VT = VT.getScalarType();
  if (!VT.isSimple())
    return false;
 
  switch (VT.getSimpleVT().SimpleTy) {
  case MVT::f32: {
    // If mad is not available this depends only on if f32 fma is full rate.
    if (!Subtarget->hasMadMacF32Insts())
      return Subtarget->hasFastFMAF32();
 
    // Otherwise f32 mad is always full rate and returns the same result as
    // the separate operations so should be preferred over fma.
    // However does not support denormals.
    if (FPEnv.F32Mode != DenormalMode::getPreserveSign())
      return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts();
 
    // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
    return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts();
  }
  case MVT::f64:
    return true;
  case MVT::f16:
  case MVT::bf16:
    return Subtarget->has16BitInsts() &&
           FPEnv.DefaultMode != DenormalMode::getPreserveSign();
  default:
    break;
  }
 
  return false;
}
 
bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
                                                  EVT VT) const {
  return isFMAFasterThanFMulAndFAdd(VT, getDenormalFPEnv(MF));
}
 
bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
                                                  Type *Ty) const {
  return isFMAFasterThanFMulAndFAdd(
      getValueType(F.getDataLayout(), Ty, /*AllowUnknown=*/true),
      F.getDenormalFPEnv());
}
 
bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
                                                  LLT Ty) const {
  switch (Ty.getScalarSizeInBits()) {
  case 16:
    return isFMAFasterThanFMulAndFAdd(MF, MVT::f16);
  case 32:
    return isFMAFasterThanFMulAndFAdd(MF, MVT::f32);
  case 64:
    return isFMAFasterThanFMulAndFAdd(MF, MVT::f64);
  default:
    break;
  }
 
  return false;
}
 
bool SITargetLowering::isFMADLegal(EVT VT, DenormalFPEnv FPEnv) const {
  // TODO: Check future ftz flag
  // v_mad_f32/v_mac_f32 do not support denormals.
  if (VT == MVT::f32)
    return Subtarget->hasMadMacF32Insts() &&
           FPEnv.F32Mode == DenormalMode::getPreserveSign();
  if (VT == MVT::f16)
    return Subtarget->hasMadF16() &&
           FPEnv.DefaultMode == DenormalMode::getPreserveSign();
 
  return false;
}
 
bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const {
  if (!Ty.isScalar())
    return false;
 
  DenormalFPEnv FPEnv = getDenormalFPEnv(*MI.getMF());
  if (Ty.getScalarSizeInBits() == 16)
    return isFMADLegal(MVT::f16, FPEnv);
  if (Ty.getScalarSizeInBits() == 32)
    return isFMADLegal(MVT::f32, FPEnv);
 
  return false;
}
 
bool SITargetLowering::isFMADLegal(const SelectionDAG &DAG,
                                   const SDNode *N) const {
  return isFMADLegal(N->getValueType(0),
                     getDenormalFPEnv(DAG.getMachineFunction()));
}
 
//===----------------------------------------------------------------------===//
// Custom DAG Lowering Operations
//===----------------------------------------------------------------------===//
 
// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
// wider vector type is legal.
SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
                                             SelectionDAG &DAG) const {
  unsigned Opc = Op.getOpcode();
  EVT VT = Op.getValueType();
  assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
 
  auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
  auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
 
  SDLoc SL(Op);
 
  // Forward any trailing scalar operands unchanged to both halves.
  SmallVector<SDValue, 2> LoOps = {Lo};
  SmallVector<SDValue, 2> HiOps = {Hi};
  auto TrailingOps = drop_begin(Op->ops());
  LoOps.append(TrailingOps.begin(), TrailingOps.end());
  HiOps.append(TrailingOps.begin(), TrailingOps.end());
 
  SDValue OpLo = DAG.getNode(Opc, SL, LoVT, LoOps, Op->getFlags());
  SDValue OpHi = DAG.getNode(Opc, SL, HiVT, HiOps, Op->getFlags());
 
  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
}
 
// Enable lowering of ROTR for vxi32 types. This is a workaround for a
// regression whereby extra unnecessary instructions were added to codegen
// for rotr operations, casued by legalising v2i32 or. This resulted in extra
// instructions to extract the result from the vector.
SDValue SITargetLowering::lowerROTR(SDValue Op, SelectionDAG &DAG) const {
  [[maybe_unused]] EVT VT = Op.getValueType();
 
  assert((VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v8i32 ||
          VT == MVT::v16i32) &&
         "Unexpected ValueType.");
 
  return DAG.UnrollVectorOp(Op.getNode());
}
 
// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
// wider vector type is legal.
SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
                                              SelectionDAG &DAG) const {
  unsigned Opc = Op.getOpcode();
  EVT VT = Op.getValueType();
  assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
 
  auto [Lo0, Hi0] = DAG.SplitVectorOperand(Op.getNode(), 0);
  auto [Lo1, Hi1] = DAG.SplitVectorOperand(Op.getNode(), 1);
 
  SDLoc SL(Op);
 
  SDValue OpLo =
      DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1, Op->getFlags());
  SDValue OpHi =
      DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1, Op->getFlags());
 
  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
}
 
SDValue SITargetLowering::splitTernaryVectorOp(SDValue Op,
                                               SelectionDAG &DAG) const {
  unsigned Opc = Op.getOpcode();
  EVT VT = Op.getValueType();
  assert(VT.isVector() && VT.getVectorElementCount().isKnownEven());
 
  SDValue Op0 = Op.getOperand(0);
  auto [Lo0, Hi0] = Op0.getValueType().isVector()
                        ? DAG.SplitVectorOperand(Op.getNode(), 0)
                        : std::pair(Op0, Op0);
 
  auto [Lo1, Hi1] = DAG.SplitVectorOperand(Op.getNode(), 1);
  auto [Lo2, Hi2] = DAG.SplitVectorOperand(Op.getNode(), 2);
 
  SDLoc SL(Op);
  auto ResVT = DAG.GetSplitDestVTs(VT);
 
  SDValue OpLo =
      DAG.getNode(Opc, SL, ResVT.first, Lo0, Lo1, Lo2, Op->getFlags());
  SDValue OpHi =
      DAG.getNode(Opc, SL, ResVT.second, Hi0, Hi1, Hi2, Op->getFlags());
 
  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
}
 
SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
  switch (Op.getOpcode()) {
  default:
    return AMDGPUTargetLowering::LowerOperation(Op, DAG);
  case ISD::BRCOND:
    return LowerBRCOND(Op, DAG);
  case ISD::RETURNADDR:
    return LowerRETURNADDR(Op, DAG);
  case ISD::SPONENTRY:
    return LowerSPONENTRY(Op, DAG);
  case ISD::LOAD: {
    SDValue Result = LowerLOAD(Op, DAG);
    assert((!Result.getNode() || Result.getNode()->getNumValues() == 2) &&
           "Load should return a value and a chain");
    return Result;
  }
  case ISD::FSQRT: {
    EVT VT = Op.getValueType();
    if (VT == MVT::f32)
      return lowerFSQRTF32(Op, DAG);
    if (VT == MVT::f64)
      return lowerFSQRTF64(Op, DAG);
    return SDValue();
  }
  case ISD::FSIN:
  case ISD::FCOS:
    return LowerTrig(Op, DAG);
  case ISD::SELECT:
    return LowerSELECT(Op, DAG);
  case ISD::FDIV:
    return LowerFDIV(Op, DAG);
  case ISD::FFREXP:
    return LowerFFREXP(Op, DAG);
  case ISD::ATOMIC_CMP_SWAP:
    return LowerATOMIC_CMP_SWAP(Op, DAG);
  case ISD::STORE:
    return LowerSTORE(Op, DAG);
  case ISD::GlobalAddress: {
    MachineFunction &MF = DAG.getMachineFunction();
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
    return LowerGlobalAddress(MFI, Op, DAG);
  }
  case ISD::BlockAddress:
    return LowerBlockAddress(Op, DAG);
  case ISD::ExternalSymbol:
    return LowerExternalSymbol(Op, DAG);
  case ISD::INTRINSIC_WO_CHAIN:
    return LowerINTRINSIC_WO_CHAIN(Op, DAG);
  case ISD::CONVERT_FROM_ARBITRARY_FP:
    return LowerCONVERT_FROM_ARBITRARY_FP(Op, DAG);
  case ISD::CONVERT_TO_ARBITRARY_FP:
    return LowerCONVERT_TO_ARBITRARY_FP(Op, DAG);
  case ISD::INTRINSIC_W_CHAIN:
    return LowerINTRINSIC_W_CHAIN(Op, DAG);
  case ISD::INTRINSIC_VOID:
    return LowerINTRINSIC_VOID(Op, DAG);
  case ISD::ADDRSPACECAST:
    return lowerADDRSPACECAST(Op, DAG);
  case ISD::INSERT_SUBVECTOR:
    return lowerINSERT_SUBVECTOR(Op, DAG);
  case ISD::INSERT_VECTOR_ELT:
    return lowerINSERT_VECTOR_ELT(Op, DAG);
  case ISD::EXTRACT_VECTOR_ELT:
    return lowerEXTRACT_VECTOR_ELT(Op, DAG);
  case ISD::VECTOR_SHUFFLE:
    return lowerVECTOR_SHUFFLE(Op, DAG);
  case ISD::SCALAR_TO_VECTOR:
    return lowerSCALAR_TO_VECTOR(Op, DAG);
  case ISD::BUILD_VECTOR:
    return lowerBUILD_VECTOR(Op, DAG);
  case ISD::FP_ROUND:
  case ISD::STRICT_FP_ROUND:
    return lowerFP_ROUND(Op, DAG);
  case ISD::TRAP:
    return lowerTRAP(Op, DAG);
  case ISD::DEBUGTRAP:
    return lowerDEBUGTRAP(Op, DAG);
  case ISD::ABS:
  case ISD::FABS:
  case ISD::FNEG:
  case ISD::FCANONICALIZE:
  case ISD::BSWAP:
    return splitUnaryVectorOp(Op, DAG);
  case ISD::FP_TO_SINT_SAT:
  case ISD::FP_TO_UINT_SAT:
    if (Op.getValueType().isVector() && Op.getValueType() != MVT::v2i16 &&
        Op.getOperand(0).getValueType().getScalarType() == MVT::f32)
      return splitUnaryVectorOp(Op, DAG);
    return LowerFP_TO_INT_SAT(Op, DAG);
  case ISD::FSUB:
    if (Op.getValueType() == MVT::bf16) {
      // Custom expansion:
      //   fsub bf16 %a, %b -> fadd v2bf16(widen %a), fneg v2bf16(widen %b)
      // Then extract back to bf16.
      //
      // We create fneg on v2bf16 (not bf16) so the instruction selector can
      // fold the negation into the packed add's neg_lo/neg_hi modifiers,
      // generating a single v_pk_add_bf16 instruction. If we negate bf16 first,
      // it becomes a separate v_xor instruction before widening.
      SDLoc DL(Op);
      SDValue Op0 = Op.getOperand(0);
      SDValue Op1 = Op.getOperand(1);
 
      // Widen both operands to v2bf16
      SDValue Vec0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Op0);
      SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2bf16, Op1);
 
      // Create FNEG v2bf16 for the second operand
      SDValue NegVec1 = DAG.getNode(ISD::FNEG, DL, MVT::v2bf16, Vec1);
 
      // Perform FADD v2bf16
      SDValue Result = DAG.getNode(ISD::FADD, DL, MVT::v2bf16, Vec0, NegVec1);
 
      // Extract element 0 back to bf16
      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Result,
                         DAG.getConstant(0, DL, MVT::i32));
    }
    return SDValue();
  case ISD::FMINNUM:
  case ISD::FMAXNUM:
    return lowerFMINNUM_FMAXNUM(Op, DAG);
  case ISD::FMINIMUMNUM:
  case ISD::FMAXIMUMNUM:
    return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG);
  case ISD::FLDEXP:
  case ISD::STRICT_FLDEXP:
    return lowerFLDEXP(Op, DAG);
  case ISD::FMA:
    return splitTernaryVectorOp(Op, DAG);
  case ISD::FP_TO_SINT:
  case ISD::FP_TO_UINT:
    if (Subtarget->hasVCvtPkIU16F32() && Op.getValueType() == MVT::i16 &&
        Op.getOperand(0).getValueType() == MVT::f32) {
      // Make f32->i16 legal so we can select V_CVT_PK_[IU]16_F32.
      return Op;
    }
    return LowerFP_TO_INT(Op, DAG);
  case ISD::SHL:
  case ISD::SRA:
  case ISD::SRL:
  case ISD::ADD:
  case ISD::SUB:
  case ISD::SMIN:
  case ISD::SMAX:
  case ISD::UMIN:
  case ISD::UMAX:
  case ISD::FMINNUM_IEEE:
  case ISD::FMAXNUM_IEEE:
  case ISD::FMINIMUM:
  case ISD::FMAXIMUM:
  case ISD::UADDSAT:
  case ISD::USUBSAT:
  case ISD::SADDSAT:
  case ISD::SSUBSAT:
  case ISD::FADD:
  case ISD::FMUL:
    return splitBinaryVectorOp(Op, DAG);
  case ISD::FCOPYSIGN:
    return lowerFCOPYSIGN(Op, DAG);
  case ISD::MUL:
    return lowerMUL(Op, DAG);
  case ISD::SMULO:
  case ISD::UMULO:
    return lowerXMULO(Op, DAG);
  case ISD::SMUL_LOHI:
  case ISD::UMUL_LOHI:
    return lowerXMUL_LOHI(Op, DAG);
  case ISD::DYNAMIC_STACKALLOC:
    return LowerDYNAMIC_STACKALLOC(Op, DAG);
  case ISD::STACKSAVE:
    return LowerSTACKSAVE(Op, DAG);
  case ISD::GET_ROUNDING:
    return lowerGET_ROUNDING(Op, DAG);
  case ISD::SET_ROUNDING:
    return lowerSET_ROUNDING(Op, DAG);
  case ISD::PREFETCH:
    return lowerPREFETCH(Op, DAG);
  case ISD::FP_EXTEND:
  case ISD::STRICT_FP_EXTEND:
    return lowerFP_EXTEND(Op, DAG);
  case ISD::GET_FPENV:
    return lowerGET_FPENV(Op, DAG);
  case ISD::SET_FPENV:
    return lowerSET_FPENV(Op, DAG);
  case ISD::ROTR:
    return lowerROTR(Op, DAG);
  case ISD::INLINEASM:
    return LowerINLINEASM(Op, DAG);
  }
  return SDValue();
}
 
// Used for D16: Casts the result of an instruction into the right vector,
// packs values if loads return unpacked values.
static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
                                       const SDLoc &DL, SelectionDAG &DAG,
                                       bool Unpacked) {
  if (!LoadVT.isVector())
    return Result;
 
  // Cast back to the original packed type or to a larger type that is a
  // multiple of 32 bit for D16. Widening the return type is a required for
  // legalization.
  EVT FittingLoadVT = LoadVT;
  if ((LoadVT.getVectorNumElements() % 2) == 1) {
    FittingLoadVT =
        EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(),
                         LoadVT.getVectorNumElements() + 1);
  }
 
  if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
    // Truncate to v2i16/v4i16.
    EVT IntLoadVT = FittingLoadVT.changeTypeToInteger();
 
    // Workaround legalizer not scalarizing truncate after vector op
    // legalization but not creating intermediate vector trunc.
    SmallVector<SDValue, 4> Elts;
    DAG.ExtractVectorElements(Result, Elts);
    for (SDValue &Elt : Elts)
      Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt);
 
    // Pad illegal v1i16/v3fi6 to v4i16
    if ((LoadVT.getVectorNumElements() % 2) == 1)
      Elts.push_back(DAG.getPOISON(MVT::i16));
 
    Result = DAG.getBuildVector(IntLoadVT, DL, Elts);
 
    // Bitcast to original type (v2f16/v4f16).
    return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result);
  }
 
  // Cast back to the original packed type.
  return DAG.getNode(ISD::BITCAST, DL, FittingLoadVT, Result);
}
 
SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode, MemSDNode *M,
                                              SelectionDAG &DAG,
                                              ArrayRef<SDValue> Ops,
                                              bool IsIntrinsic) const {
  SDLoc DL(M);
 
  bool Unpacked = Subtarget->hasUnpackedD16VMem();
  EVT LoadVT = M->getValueType(0);
 
  EVT EquivLoadVT = LoadVT;
  if (LoadVT.isVector()) {
    if (Unpacked) {
      EquivLoadVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
                                     LoadVT.getVectorNumElements());
    } else if ((LoadVT.getVectorNumElements() % 2) == 1) {
      // Widen v3f16 to legal type
      EquivLoadVT =
          EVT::getVectorVT(*DAG.getContext(), LoadVT.getVectorElementType(),
                           LoadVT.getVectorNumElements() + 1);
    }
  }
 
  // Change from v4f16/v2f16 to EquivLoadVT.
  SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other);
 
  SDValue Load = DAG.getMemIntrinsicNode(
      IsIntrinsic ? (unsigned)ISD::INTRINSIC_W_CHAIN : Opcode, DL, VTList, Ops,
      M->getMemoryVT(), M->getMemOperand());
 
  SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked);
 
  return DAG.getMergeValues({Adjusted, Load.getValue(1)}, DL);
}
 
SDValue SITargetLowering::lowerIntrinsicLoad(MemSDNode *M, bool IsFormat,
                                             SelectionDAG &DAG,
                                             ArrayRef<SDValue> Ops) const {
  SDLoc DL(M);
  EVT LoadVT = M->getValueType(0);
  EVT EltType = LoadVT.getScalarType();
  EVT IntVT = LoadVT.changeTypeToInteger();
 
  bool IsD16 = IsFormat && (EltType.getSizeInBits() == 16);
 
  if (IsFormat && !IsD16 && EltType.getSizeInBits() < 32) {
    DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
        DAG.getMachineFunction().getFunction(),
        "unsupported sub-dword format buffer load", DL.getDebugLoc()));
    return DAG.getMergeValues({DAG.getPOISON(LoadVT), M->getOperand(0)}, DL);
  }
 
  assert(M->getNumValues() == 2 || M->getNumValues() == 3);
  bool IsTFE = M->getNumValues() == 3;
 
  unsigned Opc = IsFormat ? (IsTFE ? AMDGPUISD::BUFFER_LOAD_FORMAT_TFE
                                   : AMDGPUISD::BUFFER_LOAD_FORMAT)
                 : IsTFE  ? AMDGPUISD::BUFFER_LOAD_TFE
                          : AMDGPUISD::BUFFER_LOAD;
 
  if (IsD16) {
    return adjustLoadValueType(AMDGPUISD::BUFFER_LOAD_FORMAT_D16, M, DAG, Ops);
  }
 
  // Handle BUFFER_LOAD_BYTE/UBYTE/SHORT/USHORT overloaded intrinsics
  if (!IsD16 && !LoadVT.isVector() && EltType.getSizeInBits() < 32)
    return handleByteShortBufferLoads(DAG, LoadVT, DL, Ops, M->getMemOperand(),
                                      IsTFE);
 
  if (isTypeLegal(LoadVT)) {
    return getMemIntrinsicNode(Opc, DL, M->getVTList(), Ops, IntVT,
                               M->getMemOperand(), DAG);
  }
 
  EVT CastVT = getEquivalentMemType(*DAG.getContext(), LoadVT);
  SDVTList VTList = DAG.getVTList(CastVT, MVT::Other);
  SDValue MemNode = getMemIntrinsicNode(Opc, DL, VTList, Ops, CastVT,
                                        M->getMemOperand(), DAG);
  return DAG.getMergeValues(
      {DAG.getNode(ISD::BITCAST, DL, LoadVT, MemNode), MemNode.getValue(1)},
      DL);
}
 
static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
                                  SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
  unsigned CondCode = N->getConstantOperandVal(3);
  if (!ICmpInst::isIntPredicate(static_cast<ICmpInst::Predicate>(CondCode)))
    return DAG.getPOISON(VT);
 
  ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode);
 
  SDValue LHS = N->getOperand(1);
  SDValue RHS = N->getOperand(2);
 
  SDLoc DL(N);
 
  EVT CmpVT = LHS.getValueType();
  if (CmpVT == MVT::i16 && !TLI.isTypeLegal(MVT::i16)) {
    unsigned PromoteOp =
        ICmpInst::isSigned(IcInput) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS);
    RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS);
  }
 
  ISD::CondCode CCOpcode = getICmpCondCode(IcInput);
 
  unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
  EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize);
 
  SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS,
                              DAG.getCondCode(CCOpcode));
  if (VT.bitsEq(CCVT))
    return SetCC;
  return DAG.getZExtOrTrunc(SetCC, DL, VT);
}
 
static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI, SDNode *N,
                                  SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
 
  unsigned CondCode = N->getConstantOperandVal(3);
  if (!FCmpInst::isFPPredicate(static_cast<FCmpInst::Predicate>(CondCode)))
    return DAG.getPOISON(VT);
 
  SDValue Src0 = N->getOperand(1);
  SDValue Src1 = N->getOperand(2);
  EVT CmpVT = Src0.getValueType();
  SDLoc SL(N);
 
  if (CmpVT == MVT::f16 && !TLI.isTypeLegal(CmpVT)) {
    Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
    Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
  }
 
  FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode);
  ISD::CondCode CCOpcode = getFCmpCondCode(IcInput);
  unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
  EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize);
  SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0, Src1,
                              DAG.getCondCode(CCOpcode));
  if (VT.bitsEq(CCVT))
    return SetCC;
  return DAG.getZExtOrTrunc(SetCC, SL, VT);
}
 
static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
                                    SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
  SDValue Src = N->getOperand(1);
  SDLoc SL(N);
 
  if (Src.getOpcode() == ISD::SETCC) {
    SDValue Op0 = Src.getOperand(0);
    SDValue Op1 = Src.getOperand(1);
    // Need to expand bfloat to float for comparison (setcc).
    if (Op0.getValueType() == MVT::bf16) {
      Op0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op0);
      Op1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op1);
    }
    // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
    return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Op0, Op1, Src.getOperand(2));
  }
  if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) {
    // (ballot 0) -> 0
    if (Arg->isZero())
      return DAG.getConstant(0, SL, VT);
 
    // (ballot 1) -> EXEC/EXEC_LO
    if (Arg->isOne()) {
      Register Exec;
      if (VT.getScalarSizeInBits() == 32)
        Exec = AMDGPU::EXEC_LO;
      else if (VT.getScalarSizeInBits() == 64)
        Exec = AMDGPU::EXEC;
      else
        return SDValue();
 
      return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
    }
  }
 
  // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
  // ISD::SETNE)
  return DAG.getNode(
      AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
      DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
}
 
static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
                                         EVT VT);
 
static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
                           SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
  unsigned ValSize = VT.getSizeInBits();
  unsigned IID = N->getConstantOperandVal(0);
  bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
                      IID == Intrinsic::amdgcn_permlanex16;
  bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
                       IID == Intrinsic::amdgcn_set_inactive_chain_arg;
  bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
                           IID == Intrinsic::amdgcn_permlane_up ||
                           IID == Intrinsic::amdgcn_permlane_down ||
                           IID == Intrinsic::amdgcn_permlane_xor;
  SDLoc SL(N);
  MVT IntVT = MVT::getIntegerVT(ValSize);
  const GCNSubtarget *ST = TLI.getSubtarget();
 
  unsigned SplitSize = 32;
  if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
      ST->hasDPALU_DPP() &&
      AMDGPU::isLegalDPALU_DPPControl(*ST, N->getConstantOperandVal(3)))
    SplitSize = 64;
 
  auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
                                          SDValue Src2, MVT ValT) -> SDValue {
    SmallVector<SDValue, 8> Operands;
    switch (IID) {
    case Intrinsic::amdgcn_permlane16:
    case Intrinsic::amdgcn_permlanex16:
    case Intrinsic::amdgcn_update_dpp:
      Operands.push_back(N->getOperand(6));
      Operands.push_back(N->getOperand(5));
      Operands.push_back(N->getOperand(4));
      [[fallthrough]];
    case Intrinsic::amdgcn_writelane:
    case Intrinsic::amdgcn_permlane_bcast:
    case Intrinsic::amdgcn_permlane_up:
    case Intrinsic::amdgcn_permlane_down:
    case Intrinsic::amdgcn_permlane_xor:
      Operands.push_back(Src2);
      [[fallthrough]];
    case Intrinsic::amdgcn_readlane:
    case Intrinsic::amdgcn_set_inactive:
    case Intrinsic::amdgcn_set_inactive_chain_arg:
    case Intrinsic::amdgcn_mov_dpp8:
      Operands.push_back(Src1);
      [[fallthrough]];
    case Intrinsic::amdgcn_readfirstlane:
    case Intrinsic::amdgcn_permlane64:
      Operands.push_back(Src0);
      break;
    default:
      llvm_unreachable("unhandled lane op");
    }
 
    Operands.push_back(DAG.getTargetConstant(IID, SL, MVT::i32));
    std::reverse(Operands.begin(), Operands.end());
 
    if (SDNode *GL = N->getGluedNode()) {
      assert(GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
      GL = GL->getOperand(0).getNode();
      Operands.push_back(DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
                                     SDValue(GL, 0)));
    }
 
    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, ValT, Operands);
  };
 
  SDValue Src0 = N->getOperand(1);
  SDValue Src1, Src2;
  if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
      IID == Intrinsic::amdgcn_mov_dpp8 ||
      IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
      IsPermlaneShuffle) {
    Src1 = N->getOperand(2);
    if (IID == Intrinsic::amdgcn_writelane ||
        IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
        IsPermlaneShuffle)
      Src2 = N->getOperand(3);
  }
 
  if (ValSize == SplitSize) {
    // Already legal
    return SDValue();
  }
 
  if (ValSize < 32) {
    bool IsFloat = VT.isFloatingPoint();
    Src0 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src0) : Src0,
                                SL, MVT::i32);
 
    if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
      Src1 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src1) : Src1,
                                  SL, MVT::i32);
    }
 
    if (IID == Intrinsic::amdgcn_writelane) {
      Src2 = DAG.getAnyExtOrTrunc(IsFloat ? DAG.getBitcast(IntVT, Src2) : Src2,
                                  SL, MVT::i32);
    }
 
    SDValue LaneOp = createLaneOp(Src0, Src1, Src2, MVT::i32);
    SDValue Trunc = DAG.getAnyExtOrTrunc(LaneOp, SL, IntVT);
    return IsFloat ? DAG.getBitcast(VT, Trunc) : Trunc;
  }
 
  if (ValSize % SplitSize != 0)
    return SDValue();
 
  auto unrollLaneOp = [&DAG, &SL](SDNode *N) -> SDValue {
    EVT VT = N->getValueType(0);
    unsigned NE = VT.getVectorNumElements();
    EVT EltVT = VT.getVectorElementType();
    SmallVector<SDValue, 8> Scalars;
    unsigned NumOperands = N->getNumOperands();
    SmallVector<SDValue, 4> Operands(NumOperands);
    SDNode *GL = N->getGluedNode();
 
    // only handle convergencectrl_glue
    assert(!GL || GL->getOpcode() == ISD::CONVERGENCECTRL_GLUE);
 
    for (unsigned i = 0; i != NE; ++i) {
      for (unsigned j = 0, e = GL ? NumOperands - 1 : NumOperands; j != e;
           ++j) {
        SDValue Operand = N->getOperand(j);
        EVT OperandVT = Operand.getValueType();
        if (OperandVT.isVector()) {
          // A vector operand; extract a single element.
          EVT OperandEltVT = OperandVT.getVectorElementType();
          Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, OperandEltVT,
                                    Operand, DAG.getVectorIdxConstant(i, SL));
        } else {
          // A scalar operand; just use it as is.
          Operands[j] = Operand;
        }
      }
 
      if (GL)
        Operands[NumOperands - 1] =
            DAG.getNode(ISD::CONVERGENCECTRL_GLUE, SL, MVT::Glue,
                        SDValue(GL->getOperand(0).getNode(), 0));
 
      Scalars.push_back(DAG.getNode(N->getOpcode(), SL, EltVT, Operands));
    }
 
    EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NE);
    return DAG.getBuildVector(VecVT, SL, Scalars);
  };
 
  if (VT.isVector()) {
    switch (MVT::SimpleValueType EltTy =
                VT.getVectorElementType().getSimpleVT().SimpleTy) {
    case MVT::i32:
    case MVT::f32:
      if (SplitSize == 32) {
        SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VT.getSimpleVT());
        return unrollLaneOp(LaneOp.getNode());
      }
      [[fallthrough]];
    case MVT::i16:
    case MVT::f16:
    case MVT::bf16: {
      unsigned SubVecNumElt =
          SplitSize / VT.getVectorElementType().getSizeInBits();
      MVT SubVecVT = MVT::getVectorVT(EltTy, SubVecNumElt);
      SmallVector<SDValue, 4> Pieces;
      SDValue Src0SubVec, Src1SubVec, Src2SubVec;
      for (unsigned i = 0, EltIdx = 0; i < ValSize / SplitSize; i++) {
        Src0SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src0,
                                 DAG.getConstant(EltIdx, SL, MVT::i32));
 
        if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
            IsPermLane16) {
          Src1SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src1,
                                   DAG.getConstant(EltIdx, SL, MVT::i32));
 
          Pieces.push_back(
              createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
        } else if (IID == Intrinsic::amdgcn_writelane) {
          Src2SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src2,
                                   DAG.getConstant(EltIdx, SL, MVT::i32));
          Pieces.push_back(
              createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
        } else {
          Pieces.push_back(createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
        }
 
        EltIdx += SubVecNumElt;
      }
      return DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, Pieces);
    }
    default:
      // Handle all other cases by bitcasting to i32 vectors
      break;
    }
  }
 
  MVT VecVT =
      MVT::getVectorVT(MVT::getIntegerVT(SplitSize), ValSize / SplitSize);
  Src0 = DAG.getBitcast(VecVT, Src0);
 
  if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
    Src1 = DAG.getBitcast(VecVT, Src1);
 
  if (IID == Intrinsic::amdgcn_writelane)
    Src2 = DAG.getBitcast(VecVT, Src2);
 
  SDValue LaneOp = createLaneOp(Src0, Src1, Src2, VecVT);
  SDValue UnrolledLaneOp = unrollLaneOp(LaneOp.getNode());
  return DAG.getBitcast(VT, UnrolledLaneOp);
}
 
static SDValue lowerWaveShuffle(const SITargetLowering &TLI, SDNode *N,
                                SelectionDAG &DAG) {
  EVT VT = N->getValueType(0);
 
  if (VT.getSizeInBits() != 32)
    return SDValue();
 
  SDLoc SL(N);
 
  SDValue Value = N->getOperand(1);
  SDValue Index = N->getOperand(2);
 
  // ds_bpermute requires index to be multiplied by 4
  SDValue ShiftAmount = DAG.getShiftAmountConstant(2, MVT::i32, SL);
  SDValue ShiftedIndex =
      DAG.getNode(ISD::SHL, SL, Index.getValueType(), Index, ShiftAmount);
 
  // Intrinsics will require i32 to operate on
  SDValue ValueI32 = DAG.getBitcast(MVT::i32, Value);
 
  auto MakeIntrinsic = [&DAG, &SL](unsigned IID, MVT RetVT,
                                   SmallVector<SDValue> IntrinArgs) -> SDValue {
    SmallVector<SDValue> Operands(1);
    Operands[0] = DAG.getTargetConstant(IID, SL, MVT::i32);
    Operands.append(IntrinArgs);
    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, RetVT, Operands);
  };
 
  // If we can bpermute across the whole wave, then just do that
  if (TLI.getSubtarget()->supportsWaveWideBPermute()) {
    SDValue BPermute = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
                                     {ShiftedIndex, ValueI32});
    return DAG.getBitcast(VT, BPermute);
  }
 
  assert(TLI.getSubtarget()->isWave64());
 
  // Otherwise, we need to make use of whole wave mode
  SDValue PoisonVal = DAG.getPOISON(ValueI32->getValueType(0));
 
  // Set inactive lanes to poison
  SDValue WWMValue = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
                                   {ValueI32, PoisonVal});
  SDValue WWMIndex = MakeIntrinsic(Intrinsic::amdgcn_set_inactive, MVT::i32,
                                   {ShiftedIndex, PoisonVal});
 
  SDValue Swapped =
      MakeIntrinsic(Intrinsic::amdgcn_permlane64, MVT::i32, {WWMValue});
 
  // Get permutation of each half, then we'll select which one to use
  SDValue BPermSameHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute, MVT::i32,
                                        {WWMIndex, WWMValue});
  SDValue BPermOtherHalf = MakeIntrinsic(Intrinsic::amdgcn_ds_bpermute,
                                         MVT::i32, {WWMIndex, Swapped});
  SDValue BPermOtherHalfWWM =
      MakeIntrinsic(Intrinsic::amdgcn_wwm, MVT::i32, {BPermOtherHalf});
 
  // Select which side to take the permute from
  SDValue ThreadIDMask = DAG.getAllOnesConstant(SL, MVT::i32);
  // We can get away with only using mbcnt_lo here since we're only
  // trying to detect which side of 32 each lane is on, and mbcnt_lo
  // returns 32 for lanes 32-63.
  SDValue ThreadID =
      MakeIntrinsic(Intrinsic::amdgcn_mbcnt_lo, MVT::i32,
                    {ThreadIDMask, DAG.getTargetConstant(0, SL, MVT::i32)});
 
  SDValue SameOrOtherHalf =
      DAG.getNode(ISD::AND, SL, MVT::i32,
                  DAG.getNode(ISD::XOR, SL, MVT::i32, ThreadID, Index),
                  DAG.getTargetConstant(32, SL, MVT::i32));
  SDValue UseSameHalf =
      DAG.getSetCC(SL, MVT::i1, SameOrOtherHalf,
                   DAG.getConstant(0, SL, MVT::i32), ISD::SETEQ);
  SDValue Result = DAG.getSelect(SL, MVT::i32, UseSameHalf, BPermSameHalf,
                                 BPermOtherHalfWWM);
  return DAG.getBitcast(VT, Result);
}
 
void SITargetLowering::ReplaceNodeResults(SDNode *N,
                                          SmallVectorImpl<SDValue> &Results,
                                          SelectionDAG &DAG) const {
  switch (N->getOpcode()) {
  case ISD::INSERT_VECTOR_ELT: {
    if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
      Results.push_back(Res);
    return;
  }
  case ISD::EXTRACT_VECTOR_ELT: {
    if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
      Results.push_back(Res);
    return;
  }
  case ISD::CONVERT_TO_ARBITRARY_FP: {
    if (SDValue Res = LowerCONVERT_TO_ARBITRARY_FP(SDValue(N, 0), DAG))
      Results.push_back(Res);
    return;
  }
  case ISD::INTRINSIC_WO_CHAIN: {
    unsigned IID = N->getConstantOperandVal(0);
    switch (IID) {
    case Intrinsic::amdgcn_wave_reduce_min:
    case Intrinsic::amdgcn_wave_reduce_umin:
    case Intrinsic::amdgcn_wave_reduce_max:
    case Intrinsic::amdgcn_wave_reduce_umax:
    case Intrinsic::amdgcn_wave_reduce_add:
    case Intrinsic::amdgcn_wave_reduce_sub:
    case Intrinsic::amdgcn_wave_reduce_and:
    case Intrinsic::amdgcn_wave_reduce_or:
    case Intrinsic::amdgcn_wave_reduce_xor: {
      EVT VT = N->getValueType(0);
      if (isTypeLegal(VT))
        return;
      SDLoc SL(N);
      bool NeedsSignExt = IID == Intrinsic::amdgcn_wave_reduce_min ||
                          IID == Intrinsic::amdgcn_wave_reduce_max ||
                          IID == Intrinsic::amdgcn_wave_reduce_add ||
                          IID == Intrinsic::amdgcn_wave_reduce_sub;
      unsigned ExtOpc = NeedsSignExt ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
      SDValue ExtSrc = DAG.getNode(ExtOpc, SL, MVT::i32, N->getOperand(1));
      SDValue Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SL, MVT::i32,
                                   N->getOperand(0), ExtSrc, N->getOperand(2));
      Results.push_back(DAG.getNode(ISD::TRUNCATE, SL, VT, Result));
      return;
    }
    case Intrinsic::amdgcn_make_buffer_rsrc:
      Results.push_back(lowerPointerAsRsrcIntrin(N, DAG));
      return;
    case Intrinsic::amdgcn_cvt_pkrtz: {
      SDValue Src0 = N->getOperand(1);
      SDValue Src1 = N->getOperand(2);
      SDLoc SL(N);
      SDValue Cvt =
          DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32, Src0, Src1);
      Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt));
      return;
    }
    case Intrinsic::amdgcn_cvt_pknorm_i16:
    case Intrinsic::amdgcn_cvt_pknorm_u16:
    case Intrinsic::amdgcn_cvt_pk_i16:
    case Intrinsic::amdgcn_cvt_pk_u16: {
      SDValue Src0 = N->getOperand(1);
      SDValue Src1 = N->getOperand(2);
      SDLoc SL(N);
      unsigned Opcode;
 
      if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
        Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
      else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
        Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
      else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
        Opcode = AMDGPUISD::CVT_PK_I16_I32;
      else
        Opcode = AMDGPUISD::CVT_PK_U16_U32;
 
      EVT VT = N->getValueType(0);
      if (isTypeLegal(VT))
        Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1));
      else {
        SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
        Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt));
      }
      return;
    }
    case Intrinsic::amdgcn_s_buffer_load: {
      // Lower llvm.amdgcn.s.buffer.load.(i8, u8) intrinsics. First, we generate
      // s_buffer_load_u8 for signed and unsigned load instructions. Next, DAG
      // combiner tries to merge the s_buffer_load_u8 with a sext instruction
      // (performSignExtendInRegCombine()) and it replaces s_buffer_load_u8 with
      // s_buffer_load_i8.
      if (!Subtarget->hasScalarSubwordLoads())
        return;
      SDValue Op = SDValue(N, 0);
      SDValue Rsrc = Op.getOperand(1);
      SDValue Offset = Op.getOperand(2);
      SDValue CachePolicy = Op.getOperand(3);
      EVT VT = Op.getValueType();
      assert(VT == MVT::i8 && "Expected 8-bit s_buffer_load intrinsics.\n");
      SDLoc DL(Op);
      MachineFunction &MF = DAG.getMachineFunction();
      const DataLayout &DataLayout = DAG.getDataLayout();
      Align Alignment =
          DataLayout.getABITypeAlign(VT.getTypeForEVT(*DAG.getContext()));
      MachineMemOperand *MMO = MF.getMachineMemOperand(
          MachinePointerInfo(),
          MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
              MachineMemOperand::MOInvariant,
          VT.getStoreSize(), Alignment);
      SDValue LoadVal;
      if (!Offset->isDivergent()) {
        SDValue Ops[] = {DAG.getEntryNode(), // Chain
                         Rsrc,               // source register
                         Offset, CachePolicy};
        SDValue BufferLoad = DAG.getMemIntrinsicNode(
            AMDGPUISD::SBUFFER_LOAD_UBYTE, DL,
            DAG.getVTList(MVT::i32, MVT::Other), Ops, VT, MMO);
        LoadVal = DAG.getNode(ISD::TRUNCATE, DL, VT, BufferLoad);
      } else {
        SDValue Ops[] = {
            DAG.getEntryNode(),                    // Chain
            Rsrc,                                  // rsrc
            DAG.getConstant(0, DL, MVT::i32),      // vindex
            {},                                    // voffset
            {},                                    // soffset
            {},                                    // offset
            CachePolicy,                           // cachepolicy
            DAG.getTargetConstant(0, DL, MVT::i1), // idxen
        };
        setBufferOffsets(Offset, DAG, &Ops[3], Align(4));
        LoadVal = handleByteShortBufferLoads(DAG, VT, DL, Ops, MMO);
      }
      Results.push_back(LoadVal);
      return;
    }
    case Intrinsic::amdgcn_dead: {
      for (unsigned I = 0, E = N->getNumValues(); I < E; ++I)
        Results.push_back(DAG.getPOISON(N->getValueType(I)));
      return;
    }
    }
    break;
  }
  case ISD::INTRINSIC_W_CHAIN: {
    if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) {
      if (Res.getOpcode() == ISD::MERGE_VALUES) {
        // FIXME: Hacky
        for (unsigned I = 0; I < Res.getNumOperands(); I++) {
          Results.push_back(Res.getOperand(I));
        }
      } else {
        Results.push_back(Res);
        Results.push_back(Res.getValue(1));
      }
      return;
    }
 
    break;
  }
  case ISD::SELECT: {
    SDLoc SL(N);
    EVT VT = N->getValueType(0);
    EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
    SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
    SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
 
    EVT SelectVT = NewVT;
    if (NewVT.bitsLT(MVT::i32)) {
      LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS);
      RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS);
      SelectVT = MVT::i32;
    }
 
    SDValue NewSelect =
        DAG.getNode(ISD::SELECT, SL, SelectVT, N->getOperand(0), LHS, RHS);
 
    if (NewVT != SelectVT)
      NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect);
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect));
    return;
  }
  case ISD::FNEG: {
    if (N->getValueType(0) != MVT::v2f16)
      break;
 
    SDLoc SL(N);
    SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
 
    SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32, BC,
                             DAG.getConstant(0x80008000, SL, MVT::i32));
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
    return;
  }
  case ISD::FABS: {
    if (N->getValueType(0) != MVT::v2f16)
      break;
 
    SDLoc SL(N);
    SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
 
    SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32, BC,
                             DAG.getConstant(0x7fff7fff, SL, MVT::i32));
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
    return;
  }
  case ISD::FSQRT: {
    if (N->getValueType(0) != MVT::f16)
      break;
    Results.push_back(lowerFSQRTF16(SDValue(N, 0), DAG));
    break;
  }
  default:
    AMDGPUTargetLowering::ReplaceNodeResults(N, Results, DAG);
    break;
  }
}
 
/// Helper function for LowerBRCOND
static SDNode *findUser(SDValue Value, unsigned Opcode) {
 
  for (SDUse &U : Value->uses()) {
    if (U.get() != Value)
      continue;
 
    if (U.getUser()->getOpcode() == Opcode)
      return U.getUser();
  }
  return nullptr;
}
 
unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
  if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
    switch (Intr->getConstantOperandVal(1)) {
    case Intrinsic::amdgcn_if:
      return AMDGPUISD::IF;
    case Intrinsic::amdgcn_else:
      return AMDGPUISD::ELSE;
    case Intrinsic::amdgcn_loop:
      return AMDGPUISD::LOOP;
    case Intrinsic::amdgcn_end_cf:
      llvm_unreachable("should not occur");
    default:
      return 0;
    }
  }
 
  // break, if_break, else_break are all only used as inputs to loop, not
  // directly as branch conditions.
  return 0;
}
 
bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
  const Triple &TT = getTargetMachine().getTargetTriple();
  return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
          GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
         AMDGPU::shouldEmitConstantsToTextSection(TT);
}
 
bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
  if (Subtarget->isAmdPalOS() || Subtarget->isMesa3DOS())
    return false;
 
  // FIXME: Either avoid relying on address space here or change the default
  // address space for functions to avoid the explicit check.
  return (GV->getValueType()->isFunctionTy() ||
          !isNonGlobalAddrSpace(GV->getAddressSpace())) &&
         !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(GV);
}
 
bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
  return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
}
 
bool SITargetLowering::shouldUseLDSConstAddress(const GlobalValue *GV) const {
  if (!GV->hasExternalLinkage())
    return true;
 
  // With object linking, external LDS declarations need relocations so the
  // linker can assign their offsets.
  if (AMDGPUTargetMachine::EnableObjectLinking) {
    if (const auto *GVar = dyn_cast<GlobalVariable>(GV)) {
      if (GVar->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
        assert(GVar->isDeclaration() && "AS3 GVs should be declaration here "
                                        "when object linking is enabled");
        return false;
      }
    }
  }
 
  const auto OS = getTargetMachine().getTargetTriple().getOS();
  return OS == Triple::AMDHSA || OS == Triple::AMDPAL;
}
 
/// This transforms the control flow intrinsics to get the branch destination as
/// last parameter, also switches branch target with BR if the need arise
SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, SelectionDAG &DAG) const {
  SDLoc DL(BRCOND);
 
  SDNode *Intr = BRCOND.getOperand(1).getNode();
  SDValue Target = BRCOND.getOperand(2);
  SDNode *BR = nullptr;
  SDNode *SetCC = nullptr;
 
  switch (Intr->getOpcode()) {
  case ISD::SETCC: {
    // As long as we negate the condition everything is fine
    SetCC = Intr;
    Intr = SetCC->getOperand(0).getNode();
    break;
  }
  case ISD::XOR: {
    // Similar to SETCC, if we have (xor c, -1), we will be fine.
    SDValue LHS = Intr->getOperand(0);
    SDValue RHS = Intr->getOperand(1);
    if (auto *C = dyn_cast<ConstantSDNode>(RHS); C && C->getZExtValue()) {
      Intr = LHS.getNode();
      break;
    }
    [[fallthrough]];
  }
  default: {
    // Get the target from BR if we don't negate the condition
    BR = findUser(BRCOND, ISD::BR);
    assert(BR && "brcond missing unconditional branch user");
    Target = BR->getOperand(1);
  }
  }
 
  unsigned CFNode = isCFIntrinsic(Intr);
  if (CFNode == 0) {
    // This is a uniform branch so we don't need to legalize.
    return BRCOND;
  }
 
  bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
                   Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
 
  assert(!SetCC ||
         (SetCC->getConstantOperandVal(1) == 1 &&
          cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
              ISD::SETNE));
 
  // operands of the new intrinsic call
  SmallVector<SDValue, 4> Ops;
  if (HaveChain)
    Ops.push_back(BRCOND.getOperand(0));
 
  Ops.append(Intr->op_begin() + (HaveChain ? 2 : 1), Intr->op_end());
  Ops.push_back(Target);
 
  ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
 
  // build the new intrinsic call
  SDNode *Result = DAG.getNode(CFNode, DL, DAG.getVTList(Res), Ops).getNode();
 
  if (!HaveChain) {
    SDValue Ops[] = {SDValue(Result, 0), BRCOND.getOperand(0)};