SPIRVISelLowering.cpp

Go to the documentation of this file.

//===- SPIRVISelLowering.cpp - SPIR-V DAG Lowering Impl ---------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the SPIRVTargetLowering class.
//
//===----------------------------------------------------------------------===//
 
#include "SPIRVISelLowering.h"
#include "SPIRV.h"
#include "SPIRVInstrInfo.h"
#include "SPIRVRegisterBankInfo.h"
#include "SPIRVRegisterInfo.h"
#include "SPIRVSubtarget.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
 
#define DEBUG_TYPE "spirv-lower"
 
using namespace llvm;
 
SPIRVTargetLowering::SPIRVTargetLowering(const TargetMachine &TM,
                                         const SPIRVSubtarget &ST)
    : TargetLowering(TM, ST), STI(ST) {
  // Even with SPV_ALTERA_arbitrary_precision_integers enabled, atomic sizes are
  // limited by atomicrmw xchg operation, which only supports operand up to 64
  // bits wide, as defined in SPIR-V legalizer. Currently, spirv-val doesn't
  // consider 128-bit OpTypeInt as valid either.
  setMaxAtomicSizeInBitsSupported(64);
  setMinCmpXchgSizeInBits(8);
}
 
// Returns true of the types logically match, as defined in
// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpCopyLogical.
static bool typesLogicallyMatch(const SPIRVTypeInst Ty1,
                                const SPIRVTypeInst Ty2,
                                SPIRVGlobalRegistry &GR) {
  if (Ty1->getOpcode() != Ty2->getOpcode())
    return false;
 
  if (Ty1->getNumOperands() != Ty2->getNumOperands())
    return false;
 
  if (Ty1->getOpcode() == SPIRV::OpTypeArray) {
    // Array must have the same size.
    if (Ty1->getOperand(2).getReg() != Ty2->getOperand(2).getReg())
      return false;
 
    SPIRVTypeInst ElemType1 =
        GR.getSPIRVTypeForVReg(Ty1->getOperand(1).getReg());
    SPIRVTypeInst ElemType2 =
        GR.getSPIRVTypeForVReg(Ty2->getOperand(1).getReg());
    return ElemType1 == ElemType2 ||
           typesLogicallyMatch(ElemType1, ElemType2, GR);
  }
 
  if (Ty1->getOpcode() == SPIRV::OpTypeStruct) {
    for (unsigned I = 1; I < Ty1->getNumOperands(); I++) {
      SPIRVTypeInst ElemType1 =
          GR.getSPIRVTypeForVReg(Ty1->getOperand(I).getReg());
      SPIRVTypeInst ElemType2 =
          GR.getSPIRVTypeForVReg(Ty2->getOperand(I).getReg());
      if (ElemType1 != ElemType2 &&
          !typesLogicallyMatch(ElemType1, ElemType2, GR))
        return false;
    }
    return true;
  }
  return false;
}
 
unsigned SPIRVTargetLowering::getNumRegistersForCallingConv(
    LLVMContext &Context, CallingConv::ID CC, EVT VT) const {
  // This code avoids CallLowering fail inside getVectorTypeBreakdown
  // on v3i1 arguments. Maybe we need to return 1 for all types.
  // TODO: remove it once this case is supported by the default implementation.
  if (VT.isVector() && VT.getVectorNumElements() == 3 &&
      (VT.getVectorElementType() == MVT::i1 ||
       VT.getVectorElementType() == MVT::i8))
    return 1;
  if (!VT.isVector() && VT.isInteger() && VT.getSizeInBits() <= 64)
    return 1;
  return getNumRegisters(Context, VT);
}
 
MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                       CallingConv::ID CC,
                                                       EVT VT) const {
  // This code avoids CallLowering fail inside getVectorTypeBreakdown
  // on v3i1 arguments. Maybe we need to return i32 for all types.
  // TODO: remove it once this case is supported by the default implementation.
  if (VT.isVector()) {
    if (VT.getVectorNumElements() == 3) {
      if (VT.getVectorElementType() == MVT::i1)
        return MVT::v4i1;
      else if (VT.getVectorElementType() == MVT::i8)
        return MVT::v4i8;
    } else if (!isPowerOf2_32(VT.getVectorNumElements()) &&
               STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)) {
      // Non POT element counts are not yet supported by GISEL.
      return MVT::getVectorVT(
          VT.getVectorElementType().getSimpleVT(),
          ElementCount::getFixed(VT.getVectorNumElements()));
    }
  }
  return getRegisterType(Context, VT);
}
 
void SPIRVTargetLowering::getTgtMemIntrinsic(
    SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
    MachineFunction &MF, unsigned Intrinsic) const {
  IntrinsicInfo Info;
 
  unsigned AlignIdx = 0;
  unsigned OrderingIdx = 0;
  unsigned FlagsIdx;
 
  switch (Intrinsic) {
  case Intrinsic::spv_load:
    FlagsIdx = 1;
    AlignIdx = 2;
    break;
  case Intrinsic::spv_store:
    FlagsIdx = 2;
    AlignIdx = 3;
    break;
  case Intrinsic::spv_atomic_load:
    FlagsIdx = 1;
    OrderingIdx = 2;
    break;
  case Intrinsic::spv_atomic_store:
    FlagsIdx = 2;
    OrderingIdx = 3;
    break;
  default:
    return;
  }
 
  Info.flags = static_cast<MachineMemOperand::Flags>(
      cast<ConstantInt>(I.getOperand(FlagsIdx))->getZExtValue());
  Info.memVT = MVT::i64;
  // TODO: take into account opaque pointers (don't use getElementType).
  // MVT::getVT(PtrTy->getElementType());
 
  if (AlignIdx) {
    auto *AlignOp = cast<ConstantInt>(I.getOperand(AlignIdx));
    Info.align = Align(AlignOp->getZExtValue());
  }
 
  if (OrderingIdx) {
    Info.order = static_cast<AtomicOrdering>(
        cast<ConstantInt>(I.getOperand(OrderingIdx))->getZExtValue());
  }
  Infos.push_back(Info);
}
 
TargetLowering::ConstraintType
SPIRVTargetLowering::getConstraintType(StringRef Constraint) const {
  // SPIR-V represents inline assembly via OpAsmINTEL where constraints are
  // passed through as literals defined by client API. Return C_RegisterClass
  // for non-memory constraints since SPIR-V does not distinguish between
  // register, immediate, or memory operands at this level. We do have to return
  // C_Memory for memory constraints as otherwise IRTranslator gets confused
  // trying to allocate registers for them.
  if (Constraint == "m")
    return C_Memory;
  return C_RegisterClass;
}
 
std::pair<unsigned, const TargetRegisterClass *>
SPIRVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
                                                  StringRef Constraint,
                                                  MVT VT) const {
  const TargetRegisterClass *RC = nullptr;
  if (Constraint.starts_with("{"))
    return std::make_pair(0u, RC);
 
  if (VT.isFloatingPoint())
    RC = VT.isVector() ? &SPIRV::vfIDRegClass : &SPIRV::fIDRegClass;
  else if (VT.isInteger())
    RC = VT.isVector() ? &SPIRV::viIDRegClass : &SPIRV::iIDRegClass;
  else
    RC = &SPIRV::iIDRegClass;
 
  return std::make_pair(0u, RC);
}
 
inline Register getTypeReg(MachineRegisterInfo *MRI, Register OpReg) {
  const MachineInstr *Inst = MRI->getVRegDef(OpReg);
  return Inst && Inst->getOpcode() == SPIRV::OpFunctionParameter
             ? Inst->getOperand(1).getReg()
             : OpReg;
}
 
static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
                            SPIRVGlobalRegistry &GR, MachineInstr &I,
                            Register OpReg, unsigned OpIdx,
                            SPIRVTypeInst NewPtrType) {
  MachineIRBuilder MIB(I);
  Register NewReg = createVirtualRegister(NewPtrType, &GR, MRI, MIB.getMF());
  MIB.buildInstr(SPIRV::OpBitcast)
      .addDef(NewReg)
      .addUse(GR.getSPIRVTypeID(NewPtrType))
      .addUse(OpReg)
      .constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(),
                        *STI.getRegBankInfo());
  I.getOperand(OpIdx).setReg(NewReg);
}
 
static SPIRVTypeInst createNewPtrType(SPIRVGlobalRegistry &GR, MachineInstr &I,
                                      SPIRVTypeInst OpType, bool ReuseType,
                                      SPIRVTypeInst ResType,
                                      const Type *ResTy) {
  SPIRV::StorageClass::StorageClass SC =
      static_cast<SPIRV::StorageClass::StorageClass>(
          OpType->getOperand(1).getImm());
  MachineIRBuilder MIB(I);
  SPIRVTypeInst NewBaseType =
      ReuseType ? ResType
                : GR.getOrCreateSPIRVType(
                      ResTy, MIB, SPIRV::AccessQualifier::ReadWrite, false);
  return GR.getOrCreateSPIRVPointerType(NewBaseType, MIB, SC);
}
 
// Insert a bitcast before the instruction to keep SPIR-V code valid
// when there is a type mismatch between results and operand types.
static void validatePtrTypes(const SPIRVSubtarget &STI,
                             MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR,
                             MachineInstr &I, unsigned OpIdx,
                             SPIRVTypeInst ResType,
                             const Type *ResTy = nullptr) {
  // Get operand type
  MachineFunction *MF = I.getParent()->getParent();
  Register OpReg = I.getOperand(OpIdx).getReg();
  Register OpTypeReg = getTypeReg(MRI, OpReg);
  const MachineInstr *OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);
  if (!ResType || !OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
    return;
  // Get operand's pointee type
  Register ElemTypeReg = OpType->getOperand(2).getReg();
  SPIRVTypeInst ElemType = GR.getSPIRVTypeForVReg(ElemTypeReg, MF);
  if (!ElemType)
    return;
  // Check if we need a bitcast to make a statement valid
  bool IsSameMF = MF == ResType->getParent()->getParent();
  bool IsEqualTypes = IsSameMF ? ElemType == ResType
                               : GR.getTypeForSPIRVType(ElemType) == ResTy;
  if (IsEqualTypes)
    return;
  // There is a type mismatch between results and operand types
  // and we insert a bitcast before the instruction to keep SPIR-V code valid
  SPIRVTypeInst NewPtrType =
      createNewPtrType(GR, I, OpType, IsSameMF, ResType, ResTy);
  if (!GR.isBitcastCompatible(NewPtrType, OpType))
    report_fatal_error(
        "insert validation bitcast: incompatible result and operand types");
  doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
}
 
// Insert a bitcast before OpGroupWaitEvents if the last argument is a pointer
// that doesn't point to OpTypeEvent.
static void validateGroupWaitEventsPtr(const SPIRVSubtarget &STI,
                                       MachineRegisterInfo *MRI,
                                       SPIRVGlobalRegistry &GR,
                                       MachineInstr &I) {
  constexpr unsigned OpIdx = 2;
  MachineFunction *MF = I.getParent()->getParent();
  Register OpReg = I.getOperand(OpIdx).getReg();
  Register OpTypeReg = getTypeReg(MRI, OpReg);
  SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);
  if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
    return;
  SPIRVTypeInst ElemType =
      GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg());
  if (!ElemType || ElemType->getOpcode() == SPIRV::OpTypeEvent)
    return;
  // Insert a bitcast before the instruction to keep SPIR-V code valid.
  LLVMContext &Context = MF->getFunction().getContext();
  SPIRVTypeInst NewPtrType =
      createNewPtrType(GR, I, OpType, false, nullptr,
                       TargetExtType::get(Context, "spirv.Event"));
  doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
}
 
static void validateLifetimeStart(const SPIRVSubtarget &STI,
                                  MachineRegisterInfo *MRI,
                                  SPIRVGlobalRegistry &GR, MachineInstr &I) {
  Register PtrReg = I.getOperand(0).getReg();
  MachineFunction *MF = I.getParent()->getParent();
  Register PtrTypeReg = getTypeReg(MRI, PtrReg);
  SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(PtrTypeReg, MF);
  SPIRVTypeInst PonteeElemType = PtrType ? GR.getPointeeType(PtrType) : nullptr;
  if (!PonteeElemType || PonteeElemType->getOpcode() == SPIRV::OpTypeVoid ||
      (PonteeElemType->getOpcode() == SPIRV::OpTypeInt &&
       PonteeElemType->getOperand(1).getImm() == 8))
    return;
  // To keep the code valid a bitcast must be inserted
  SPIRV::StorageClass::StorageClass SC =
      static_cast<SPIRV::StorageClass::StorageClass>(
          PtrType->getOperand(1).getImm());
  MachineIRBuilder MIB(I);
  LLVMContext &Context = MF->getFunction().getContext();
  SPIRVTypeInst NewPtrType =
      GR.getOrCreateSPIRVPointerType(IntegerType::getInt8Ty(Context), MIB, SC);
  doInsertBitcast(STI, MRI, GR, I, PtrReg, 0, NewPtrType);
}
 
static void validatePtrUnwrapStructField(const SPIRVSubtarget &STI,
                                         MachineRegisterInfo *MRI,
                                         SPIRVGlobalRegistry &GR,
                                         MachineInstr &I, unsigned OpIdx) {
  MachineFunction *MF = I.getParent()->getParent();
  Register OpReg = I.getOperand(OpIdx).getReg();
  Register OpTypeReg = getTypeReg(MRI, OpReg);
  SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);
  if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
    return;
  SPIRVTypeInst ElemType =
      GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg());
  if (!ElemType || ElemType->getOpcode() != SPIRV::OpTypeStruct ||
      ElemType->getNumOperands() != 2)
    return;
  // It's a structure-wrapper around another type with a single member field.
  SPIRVTypeInst MemberType =
      GR.getSPIRVTypeForVReg(ElemType->getOperand(1).getReg());
  if (!MemberType)
    return;
  unsigned MemberTypeOp = MemberType->getOpcode();
  if (!isVectorType(MemberType) && MemberTypeOp != SPIRV::OpTypeInt &&
      MemberTypeOp != SPIRV::OpTypeFloat && MemberTypeOp != SPIRV::OpTypeBool)
    return;
  // It's a structure-wrapper around a valid type. Insert a bitcast before the
  // instruction to keep SPIR-V code valid.
  SPIRV::StorageClass::StorageClass SC =
      static_cast<SPIRV::StorageClass::StorageClass>(
          OpType->getOperand(1).getImm());
  MachineIRBuilder MIB(I);
  SPIRVTypeInst NewPtrType =
      GR.getOrCreateSPIRVPointerType(MemberType, MIB, SC);
  doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);
}
 
// Insert a bitcast before the function call instruction to keep SPIR-V code
// valid when there is a type mismatch between actual and expected types of an
// argument:
// %formal = OpFunctionParameter %formal_type
// ...
// %res = OpFunctionCall %ty %fun %actual ...
// implies that %actual is of %formal_type, and in case of opaque pointers.
// We may need to insert a bitcast to ensure this.
void validateFunCallMachineDef(const SPIRVSubtarget &STI,
                               MachineRegisterInfo *DefMRI,
                               MachineRegisterInfo *CallMRI,
                               SPIRVGlobalRegistry &GR, MachineInstr &FunCall,
                               MachineInstr *FunDef) {
  if (FunDef->getOpcode() != SPIRV::OpFunction)
    return;
  unsigned OpIdx = 3;
  for (FunDef = FunDef->getNextNode();
       FunDef && FunDef->getOpcode() == SPIRV::OpFunctionParameter &&
       OpIdx < FunCall.getNumOperands();
       FunDef = FunDef->getNextNode(), OpIdx++) {
    SPIRVTypeInst DefPtrType =
        DefMRI->getVRegDef(FunDef->getOperand(1).getReg());
    SPIRVTypeInst DefElemType =
        DefPtrType && DefPtrType->getOpcode() == SPIRV::OpTypePointer
            ? GR.getSPIRVTypeForVReg(DefPtrType->getOperand(2).getReg(),
                                     DefPtrType->getParent()->getParent())
            : nullptr;
    if (DefElemType) {
      const Type *DefElemTy = GR.getTypeForSPIRVType(DefElemType);
      // validatePtrTypes() works in the context if the call site
      // When we process historical records about forward calls
      // we need to switch context to the (forward) call site and
      // then restore it back to the current machine function.
      MachineFunction *CurMF =
          GR.setCurrentFunc(*FunCall.getParent()->getParent());
      validatePtrTypes(STI, CallMRI, GR, FunCall, OpIdx, DefElemType,
                       DefElemTy);
      GR.setCurrentFunc(*CurMF);
    }
  }
}
 
// Ensure there is no mismatch between actual and expected arg types: calls
// with a processed definition. Return Function pointer if it's a forward
// call (ahead of definition), and nullptr otherwise.
const Function *validateFunCall(const SPIRVSubtarget &STI,
                                MachineRegisterInfo *CallMRI,
                                SPIRVGlobalRegistry &GR,
                                MachineInstr &FunCall) {
  const GlobalValue *GV = FunCall.getOperand(2).getGlobal();
  const Function *F = dyn_cast<Function>(GV);
  MachineInstr *FunDef =
      const_cast<MachineInstr *>(GR.getFunctionDefinition(F));
  if (!FunDef)
    return F;
  MachineRegisterInfo *DefMRI = &FunDef->getParent()->getParent()->getRegInfo();
  validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall, FunDef);
  return nullptr;
}
 
// Ensure there is no mismatch between actual and expected arg types: calls
// ahead of a processed definition.
void validateForwardCalls(const SPIRVSubtarget &STI,
                          MachineRegisterInfo *DefMRI, SPIRVGlobalRegistry &GR,
                          MachineInstr &FunDef) {
  const Function *F = GR.getFunctionByDefinition(&FunDef);
  if (SmallPtrSet<MachineInstr *, 8> *FwdCalls = GR.getForwardCalls(F))
    for (MachineInstr *FunCall : *FwdCalls) {
      MachineRegisterInfo *CallMRI =
          &FunCall->getParent()->getParent()->getRegInfo();
      validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, *FunCall, &FunDef);
    }
}
 
// Validation of an access chain.
void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
                         SPIRVGlobalRegistry &GR, MachineInstr &I) {
  SPIRVTypeInst BaseTypeInst = GR.getSPIRVTypeForVReg(I.getOperand(0).getReg());
  if (BaseTypeInst && BaseTypeInst->getOpcode() == SPIRV::OpTypePointer) {
    SPIRVTypeInst BaseElemType =
        GR.getSPIRVTypeForVReg(BaseTypeInst->getOperand(2).getReg());
    validatePtrTypes(STI, MRI, GR, I, 2, BaseElemType);
  }
}
 
static void validateVec1Ops(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,
                            SPIRVGlobalRegistry &GR, MachineInstr &MI) {
  // IRTranslator does not believe that rank-1 vectors exist, unlike upstream
  // LLVM which happily creates <1 x T> vectors. This leads to operations over
  // <1 x T> vectors getting translated as their scalar counterparts, which is
  // wrong if we used SPV_EXT_long_vector to preserve the actual vector-ness.
  switch (MI.getOpcode()) {
  case SPIRV::OpBitwiseAndS:
  case SPIRV::OpBitwiseOrS:
  case SPIRV::OpBitwiseXorS:
  case SPIRV::OpFAddS:
  case SPIRV::OpFDivS:
  case SPIRV::OpFMulS:
  case SPIRV::OpFNegate:
  case SPIRV::OpFRemS:
  case SPIRV::OpFSubS:
  case SPIRV::OpIAddCarryS:
  case SPIRV::OpIAddS:
  case SPIRV::OpIMulS:
  case SPIRV::OpISubBorrowS:
  case SPIRV::OpISubS:
  case SPIRV::OpSDivS:
  case SPIRV::OpSRemS:
  case SPIRV::OpShiftLeftLogicalS:
  case SPIRV::OpShiftRightArithmeticS:
  case SPIRV::OpShiftRightLogicalS:
  case SPIRV::OpStrictFAddS:
  case SPIRV::OpStrictFDivS:
  case SPIRV::OpStrictFMulS:
  case SPIRV::OpStrictFRemS:
  case SPIRV::OpStrictFSubS:
  case SPIRV::OpUDivS:
  case SPIRV::OpUModS: {
    SPIRVTypeInst ResTy = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());
 
    if (!isVectorType(ResTy))
      return;
 
    // Restore original Vec1 type.
    Register NewResultReg = createVirtualRegister(ResTy, &GR, MRI, *MI.getMF());
    MRI->replaceRegWith(MI.getOperand(0).getReg(), NewResultReg);
    // Vector opcodes are always next after scalar (if this ceases to hold we
    // will have to adapt).
    MI.setDesc(STI.getInstrInfo()->get(MI.getOpcode() + 1));
    // IRTranslator would've inserted COPYs from the vector into a scalar, which
    // are spurious and have to be walked through.
    for (unsigned I = 2; I != MI.getNumOperands(); ++I) {
      MachineOperand &Op = MI.getOperand(I);
      if (!Op.isReg())
        continue;
 
      SPIRVTypeInst OpTy = GR.getSPIRVTypeForVReg(Op.getReg());
      if (OpTy == ResTy)
        continue;
 
      MachineInstr *OpDef = getDef(Op, MRI);
      assert(OpDef &&
             GR.getSPIRVTypeForVReg(OpDef->getOperand(0).getReg()) == ResTy &&
             "Expected to find Result Type (Vec1)!");
      MI.substituteRegister(Op.getReg(), OpDef->getOperand(0).getReg(), 0,
                            *STI.getRegisterInfo());
    }
    break;
  }
  case TargetOpcode::COPY: {
    Register ResVReg = MI.getOperand(0).getReg();
    SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());
    SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(ResVReg);
 
    if (!SrcTy || !DstTy || isVectorType(DstTy) || !isVectorType(SrcTy))
      return;
 
    Register ExtractReg = createVirtualRegister(DstTy, &GR, MRI, *MI.getMF());
    BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
            STI.getInstrInfo()->get(SPIRV::OpCompositeExtract))
        .addDef(ExtractReg)
        .addUse(GR.getSPIRVTypeID(DstTy))
        .addUse(MI.getOperand(1).getReg())
        .addImm(0);
    for (auto &&U : MRI->use_instructions(ResVReg))
      U.substituteRegister(ResVReg, ExtractReg, 0, *STI.getRegisterInfo());
    break;
  }
  default:
    break;
  }
}
 
// TODO: the logic of inserting additional bitcast's is to be moved
// to pre-IRTranslation passes eventually
void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {
  // finalizeLowering() is called twice (see GlobalISel/InstructionSelect.cpp)
  // We'd like to avoid the needless second processing pass.
  if (MF.getRegInfo().reservedRegsFrozen())
    return;
 
  MachineRegisterInfo *MRI = &MF.getRegInfo();
  SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
  GR.setCurrentFunc(MF);
  for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {
    MachineBasicBlock *MBB = &*I;
    for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end();
         MBBI != MBBE;) {
      MachineInstr &MI = *MBBI++;
      validateVec1Ops(STI, MRI, GR, MI);
      switch (MI.getOpcode()) {
      case SPIRV::OpAtomicLoad:
      case SPIRV::OpAtomicExchange:
      case SPIRV::OpAtomicCompareExchange:
      case SPIRV::OpAtomicCompareExchangeWeak:
      case SPIRV::OpAtomicIIncrement:
      case SPIRV::OpAtomicIDecrement:
      case SPIRV::OpAtomicIAdd:
      case SPIRV::OpAtomicISub:
      case SPIRV::OpAtomicSMin:
      case SPIRV::OpAtomicUMin:
      case SPIRV::OpAtomicSMax:
      case SPIRV::OpAtomicUMax:
      case SPIRV::OpAtomicAnd:
      case SPIRV::OpAtomicOr:
      case SPIRV::OpAtomicXor:
        // for the above listed instructions
        // OpAtomicXXX <ResType>, ptr %Op, ...
        // implies that %Op is a pointer to <ResType>
      case SPIRV::OpLoad:
        // OpLoad <ResType>, ptr %Op implies that %Op is a pointer to <ResType>
        if (enforcePtrTypeCompatibility(MI, 2, 0))
          break;
 
        validatePtrTypes(STI, MRI, GR, MI, 2,
                         GR.getSPIRVTypeForVReg(MI.getOperand(0).getReg()));
        break;
      case SPIRV::OpAtomicStore:
        // OpAtomicStore ptr %Op, <Scope>, <Mem>, <Obj>
        // implies that %Op points to the <Obj>'s type
        validatePtrTypes(STI, MRI, GR, MI, 0,
                         GR.getSPIRVTypeForVReg(MI.getOperand(3).getReg()));
        break;
      case SPIRV::OpStore:
        // OpStore ptr %Op, <Obj> implies that %Op points to the <Obj>'s type
        validatePtrTypes(STI, MRI, GR, MI, 0,
                         GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg()));
        break;
      case SPIRV::OpPtrCastToGeneric:
      case SPIRV::OpGenericCastToPtr:
      case SPIRV::OpGenericCastToPtrExplicit:
        validateAccessChain(STI, MRI, GR, MI);
        break;
      case SPIRV::OpPtrAccessChain:
      case SPIRV::OpInBoundsPtrAccessChain:
        if (MI.getNumOperands() == 4)
          validateAccessChain(STI, MRI, GR, MI);
        break;
 
      case SPIRV::OpFunctionCall:
        // ensure there is no mismatch between actual and expected arg types:
        // calls with a processed definition
        if (MI.getNumOperands() > 3)
          if (const Function *F = validateFunCall(STI, MRI, GR, MI))
            GR.addForwardCall(F, &MI);
        break;
      case SPIRV::OpFunction:
        // ensure there is no mismatch between actual and expected arg types:
        // calls ahead of a processed definition
        validateForwardCalls(STI, MRI, GR, MI);
        break;
 
      // ensure that LLVM IR add/sub instructions result in logical SPIR-V
      // instructions when applied to bool type
      case SPIRV::OpIAddS:
      case SPIRV::OpIAddV:
      case SPIRV::OpISubS:
      case SPIRV::OpISubV:
        if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
                                      SPIRV::OpTypeBool))
          MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
        break;
      // multiplication of bool operands is equivalent to a logical AND
      case SPIRV::OpIMulS:
      case SPIRV::OpIMulV:
        if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
                                      SPIRV::OpTypeBool))
          MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd));
        break;
 
      // ensure that LLVM IR bitwise instructions result in logical SPIR-V
      // instructions when applied to bool type
      case SPIRV::OpBitwiseOrS:
      case SPIRV::OpBitwiseOrV:
        if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
                                      SPIRV::OpTypeBool))
          MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalOr));
        break;
      case SPIRV::OpBitwiseAndS:
      case SPIRV::OpBitwiseAndV:
        if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
                                      SPIRV::OpTypeBool))
          MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd));
        break;
      case SPIRV::OpBitwiseXorS:
      case SPIRV::OpBitwiseXorV:
        if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),
                                      SPIRV::OpTypeBool))
          MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));
        break;
      case SPIRV::OpLifetimeStart:
      case SPIRV::OpLifetimeStop:
        if (MI.getOperand(1).getImm() > 0)
          validateLifetimeStart(STI, MRI, GR, MI);
        break;
      case SPIRV::OpGroupAsyncCopy:
        validatePtrUnwrapStructField(STI, MRI, GR, MI, 3);
        validatePtrUnwrapStructField(STI, MRI, GR, MI, 4);
        break;
      case SPIRV::OpGroupWaitEvents:
        // OpGroupWaitEvents ..., ..., <pointer to OpTypeEvent>
        validateGroupWaitEventsPtr(STI, MRI, GR, MI);
        break;
      case SPIRV::OpConstantI: {
        SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());
        if (Type->getOpcode() != SPIRV::OpTypeInt && MI.getOperand(2).isImm() &&
            MI.getOperand(2).getImm() == 0) {
          // Validate the null constant of a target extension type
          MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull));
          for (unsigned i = MI.getNumOperands() - 1; i > 1; --i)
            MI.removeOperand(i);
        }
      } break;
      case SPIRV::OpExtInst: {
        // prefetch
        if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isImm() ||
            MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std)
          continue;
        switch (MI.getOperand(3).getImm()) {
        case SPIRV::OpenCLExtInst::frexp:
        case SPIRV::OpenCLExtInst::lgamma_r:
        case SPIRV::OpenCLExtInst::remquo: {
          // The last operand must be of a pointer to i32 or vector of i32
          // values.
          MachineIRBuilder MIB(MI);
          SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(32, MIB);
          SPIRVTypeInst RetType = MRI->getVRegDef(MI.getOperand(1).getReg());
          assert(RetType && "Expected return type");
          validatePtrTypes(
              STI, MRI, GR, MI, MI.getNumOperands() - 1,
              (!isVectorType(RetType))
                  ? Int32Type
                  : GR.getOrCreateSPIRVVectorType(
                        Int32Type, GR.getScalarOrVectorComponentCount(RetType),
                        MIB, false));
        } break;
        case SPIRV::OpenCLExtInst::fract:
        case SPIRV::OpenCLExtInst::modf:
        case SPIRV::OpenCLExtInst::sincos:
          // The last operand must be of a pointer to the base type represented
          // by the previous operand.
          assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
                 "Expected v-reg");
          validatePtrTypes(
              STI, MRI, GR, MI, MI.getNumOperands() - 1,
              GR.getSPIRVTypeForVReg(
                  MI.getOperand(MI.getNumOperands() - 2).getReg()));
          break;
        case SPIRV::OpenCLExtInst::prefetch:
          // Expected `ptr` type is a pointer to float, integer or vector, but
          // the pontee value can be wrapped into a struct.
          assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&
                 "Expected v-reg");
          validatePtrUnwrapStructField(STI, MRI, GR, MI,
                                       MI.getNumOperands() - 2);
          break;
        }
      } break;
      }
    }
  }
  TargetLowering::finalizeLowering(MF);
}
 
// Modifies either operand PtrOpIdx or OpIdx so that the pointee type of
// PtrOpIdx matches the type for operand OpIdx. Returns true if they already
// match or if the instruction was modified to make them match.
bool SPIRVTargetLowering::enforcePtrTypeCompatibility(
    MachineInstr &I, unsigned int PtrOpIdx, unsigned int OpIdx) const {
  SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
  SPIRVTypeInst PtrType = GR.getResultType(I.getOperand(PtrOpIdx).getReg());
 
  if (PtrType && PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR)
    return true;
 
  SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType);
  SPIRVTypeInst OpType = GR.getResultType(I.getOperand(OpIdx).getReg());
 
  if (PointeeType == OpType)
    return true;
 
  // getPointeeType yields nullptr for anything that is not an OpTypePointer.
  // The early return above does not cover an untyped pointer nested in another
  // type, such as a vector of pointers built for a scalarized vector GEP.
  // typesLogicallyMatch dereferences both of its arguments, so bail out before
  // calling it.
  if (PointeeType && OpType && typesLogicallyMatch(PointeeType, OpType, GR)) {
    // Apply OpCopyLogical to OpIdx.
    if (I.getOperand(OpIdx).isDef() &&
        insertLogicalCopyOnResult(I, PointeeType)) {
      return true;
    }
 
    llvm_unreachable("Unable to add OpCopyLogical yet.");
    return false;
  }
 
  return false;
}
 
bool SPIRVTargetLowering::insertLogicalCopyOnResult(
    MachineInstr &I, SPIRVTypeInst NewResultType) const {
  MachineRegisterInfo *MRI = &I.getMF()->getRegInfo();
  SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();
 
  Register NewResultReg =
      createVirtualRegister(NewResultType, &GR, MRI, *I.getMF());
  Register NewTypeReg = GR.getSPIRVTypeID(NewResultType);
 
  assert(llvm::size(I.defs()) == 1 && "Expected only one def");
  MachineOperand &OldResult = *I.defs().begin();
  Register OldResultReg = OldResult.getReg();
  MachineOperand &OldType = *I.uses().begin();
  Register OldTypeReg = OldType.getReg();
 
  OldResult.setReg(NewResultReg);
  OldType.setReg(NewTypeReg);
 
  MachineIRBuilder MIB(*I.getNextNode());
  MIB.buildInstr(SPIRV::OpCopyLogical)
      .addDef(OldResultReg)
      .addUse(OldTypeReg)
      .addUse(NewResultReg)
      .constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(),
                        *STI.getRegBankInfo());
  return true;
}
 
TargetLowering::AtomicExpansionKind
SPIRVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *RMW) const {
  switch (RMW->getOperation()) {
  case AtomicRMWInst::FAdd:
  case AtomicRMWInst::FSub:
  case AtomicRMWInst::FMin:
  case AtomicRMWInst::FMax:
    return AtomicExpansionKind::None;
  case AtomicRMWInst::UIncWrap:
  case AtomicRMWInst::UDecWrap:
  case AtomicRMWInst::Nand:
    return AtomicExpansionKind::CmpXChg;
  default:
    return TargetLowering::shouldExpandAtomicRMWInIR(RMW);
  }
}
 
TargetLowering::AtomicExpansionKind
SPIRVTargetLowering::shouldCastAtomicRMWIInIR(AtomicRMWInst *RMWI) const {
  // TODO: Pointer operand should be cast to integer in atomicrmw xchg, since
  // SPIR-V only supports atomic exchange for integer and floating-point types.
  return AtomicExpansionKind::None;
}
 
TargetLowering::AtomicExpansionKind
SPIRVTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
  // TODO: pointer load should return CastToInteger, but
  // convertAtomicLoadToIntegerType uses BitCast which asserts on pointer types.
  return AtomicExpansionKind::None;
}
 
TargetLowering::AtomicExpansionKind
SPIRVTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
  // TODO: pointer store should return CastToInteger, but
  // convertAtomicStoreToIntegerType uses BitCast which asserts on pointer
  // types.
  return AtomicExpansionKind::None;
}