IRBuilder.cpp

Go to the documentation of this file.

//===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the IRBuilder class, which is used as a convenient way
// to create LLVM instructions with a consistent and simplified interface.
//
//===----------------------------------------------------------------------===//
 
#include "llvm/IR/IRBuilder.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/NoFolder.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include <cassert>
#include <cstdint>
#include <optional>
#include <vector>
 
using namespace llvm;
 
/// CreateGlobalString - Make a new global variable with an initializer that
/// has array of i8 type filled in with the nul terminated string value
/// specified.  If Name is specified, it is the name of the global variable
/// created.
GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
                                                  const Twine &Name,
                                                  unsigned AddressSpace,
                                                  Module *M, bool AddNull) {
  Constant *StrConstant = ConstantDataArray::getString(Context, Str, AddNull);
  if (!M)
    M = BB->getParent()->getParent();
  auto *GV = new GlobalVariable(
      *M, StrConstant->getType(), true, GlobalValue::PrivateLinkage,
      StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace);
  GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
  GV->setAlignment(M->getDataLayout().getPrefTypeAlign(getInt8Ty()));
  return GV;
}
 
Type *IRBuilderBase::getCurrentFunctionReturnType() const {
  assert(BB && BB->getParent() && "No current function!");
  return BB->getParent()->getReturnType();
}
 
DebugLoc IRBuilderBase::getCurrentDebugLocation() const { return StoredDL; }
void IRBuilderBase::SetInstDebugLocation(Instruction *I) const {
  // We prefer to set our current debug location if any has been set, but if
  // our debug location is empty and I has a valid location, we shouldn't
  // overwrite it.
  I->setDebugLoc(StoredDL.orElse(I->getDebugLoc()));
}
 
Value *IRBuilderBase::CreateAggregateCast(Value *V, Type *DestTy) {
  Type *SrcTy = V->getType();
  if (SrcTy == DestTy)
    return V;
 
  if (SrcTy->isAggregateType()) {
    unsigned NumElements;
    if (SrcTy->isStructTy()) {
      assert(DestTy->isStructTy() && "Expected StructType");
      assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements() &&
             "Expected StructTypes with equal number of elements");
      NumElements = SrcTy->getStructNumElements();
    } else {
      assert(SrcTy->isArrayTy() && DestTy->isArrayTy() && "Expected ArrayType");
      assert(SrcTy->getArrayNumElements() == DestTy->getArrayNumElements() &&
             "Expected ArrayTypes with equal number of elements");
      NumElements = SrcTy->getArrayNumElements();
    }
 
    Value *Result = PoisonValue::get(DestTy);
    for (unsigned I = 0; I < NumElements; ++I) {
      Type *ElementTy = SrcTy->isStructTy() ? DestTy->getStructElementType(I)
                                            : DestTy->getArrayElementType();
      Value *Element =
          CreateAggregateCast(CreateExtractValue(V, ArrayRef(I)), ElementTy);
 
      Result = CreateInsertValue(Result, Element, ArrayRef(I));
    }
    return Result;
  }
 
  return CreateBitOrPointerCast(V, DestTy);
}
 
Value *IRBuilderBase::CreateBitPreservingCastChain(const DataLayout &DL,
                                                   Value *V, Type *NewTy) {
  Type *OldTy = V->getType();
 
  if (OldTy == NewTy)
    return V;
 
  assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&
         "Integer types must be the exact same to convert.");
 
  // A variant of bitcast that supports a mixture of fixed and scalable types
  // that are know to have the same size.
  auto CreateBitCastLike = [this](Value *In, Type *Ty) -> Value * {
    Type *InTy = In->getType();
    if (InTy == Ty)
      return In;
 
    if (isa<FixedVectorType>(InTy) && isa<ScalableVectorType>(Ty)) {
      // For vscale_range(2) expand <4 x i32> to <vscale x 4 x i16> -->
      //   <4 x i32> to <vscale x 2 x i32> to <vscale x 4 x i16>
      auto *VTy = VectorType::getWithSizeAndScalar(cast<VectorType>(Ty), InTy);
      return CreateBitCast(
          CreateInsertVector(VTy, PoisonValue::get(VTy), In, getInt64(0)), Ty);
    }
 
    if (isa<ScalableVectorType>(InTy) && isa<FixedVectorType>(Ty)) {
      // For vscale_range(2) expand <vscale x 4 x i16> to <4 x i32> -->
      //   <vscale x 4 x i16> to <vscale x 2 x i32> to <4 x i32>
      auto *VTy = VectorType::getWithSizeAndScalar(cast<VectorType>(InTy), Ty);
      return CreateExtractVector(Ty, CreateBitCast(In, VTy), getInt64(0));
    }
 
    return CreateBitCast(In, Ty);
  };
 
  // See if we need inttoptr for this type pair. May require additional bitcast.
  if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
    // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8*
    // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*>
    // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*>
    // Directly handle i64 to i8*
    return CreateIntToPtr(CreateBitCastLike(V, DL.getIntPtrType(NewTy)), NewTy);
  }
 
  // See if we need ptrtoint for this type pair. May require additional bitcast.
  if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) {
    // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128
    // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32>
    // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32>
    // Expand i8* to i64 --> i8* to i64 to i64
    return CreateBitCastLike(CreatePtrToInt(V, DL.getIntPtrType(OldTy)), NewTy);
  }
 
  if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) {
    unsigned OldAS = OldTy->getPointerAddressSpace();
    unsigned NewAS = NewTy->getPointerAddressSpace();
    // To convert pointers with different address spaces (they are already
    // checked convertible, i.e. they have the same pointer size), so far we
    // cannot use `bitcast` (which has restrict on the same address space) or
    // `addrspacecast` (which is not always no-op casting). Instead, use a pair
    // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit
    // size.
    if (OldAS != NewAS) {
      return CreateIntToPtr(
          CreateBitCastLike(CreatePtrToInt(V, DL.getIntPtrType(OldTy)),
                            DL.getIntPtrType(NewTy)),
          NewTy);
    }
  }
 
  return CreateBitCastLike(V, NewTy);
}
 
CallInst *
IRBuilderBase::createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
                                const Twine &Name, FMFSource FMFSource,
                                ArrayRef<OperandBundleDef> OpBundles) {
  CallInst *CI = CreateCall(Callee, Ops, OpBundles, Name);
  if (isa<FPMathOperator>(CI))
    CI->setFastMathFlags(FMFSource.get(FMF));
  return CI;
}
 
static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {
  Value *VScale = B.CreateVScale(Ty);
  if (Scale == 1)
    return VScale;
 
  return B.CreateNUWMul(VScale, ConstantInt::get(Ty, Scale));
}
 
Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {
  if (EC.isFixed() || EC.isZero())
    return ConstantInt::get(Ty, EC.getKnownMinValue());
 
  return CreateVScaleMultiple(*this, Ty, EC.getKnownMinValue());
}
 
Value *IRBuilderBase::CreateTypeSize(Type *Ty, TypeSize Size) {
  if (Size.isFixed() || Size.isZero())
    return ConstantInt::get(Ty, Size.getKnownMinValue());
 
  return CreateVScaleMultiple(*this, Ty, Size.getKnownMinValue());
}
 
Value *IRBuilderBase::CreateAllocationSize(Type *DestTy, AllocaInst *AI) {
  const DataLayout &DL = BB->getDataLayout();
  TypeSize ElemSize = DL.getTypeAllocSize(AI->getAllocatedType());
  Value *Size = CreateTypeSize(DestTy, ElemSize);
  if (AI->isArrayAllocation())
    Size = CreateMul(CreateZExtOrTrunc(AI->getArraySize(), DestTy), Size);
  return Size;
}
 
Value *IRBuilderBase::CreateStepVector(Type *DstType, const Twine &Name) {
  Type *STy = DstType->getScalarType();
  if (isa<ScalableVectorType>(DstType)) {
    Type *StepVecType = DstType;
    // TODO: We expect this special case (element type < 8 bits) to be
    // temporary - once the intrinsic properly supports < 8 bits this code
    // can be removed.
    if (STy->getScalarSizeInBits() < 8)
      StepVecType =
          VectorType::get(getInt8Ty(), cast<ScalableVectorType>(DstType));
    Value *Res = CreateIntrinsic(Intrinsic::stepvector, {StepVecType}, {},
                                 nullptr, Name);
    if (StepVecType != DstType)
      Res = CreateTrunc(Res, DstType);
    return Res;
  }
 
  unsigned NumEls = cast<FixedVectorType>(DstType)->getNumElements();
 
  // Create a vector of consecutive numbers from zero to VF.
  // It's okay if the values wrap around.
  SmallVector<Constant *, 8> Indices;
  for (unsigned i = 0; i < NumEls; ++i)
    Indices.push_back(
        ConstantInt::get(STy, i, /*IsSigned=*/false, /*ImplicitTrunc=*/true));
 
  // Add the consecutive indices to the vector value.
  return ConstantVector::get(Indices);
}
 
CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
                                      MaybeAlign Align, bool isVolatile,
                                      const AAMDNodes &AAInfo) {
  Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
  Type *Tys[] = {Ptr->getType(), Size->getType()};
 
  auto *CI = cast<MemSetInst>(
      CreateIntrinsicWithoutFolding(Intrinsic::memset, Tys, Ops));
 
  if (Align)
    CI->setDestAlignment(*Align);
  CI->setAAMetadata(AAInfo);
  return CI;
}
 
CallInst *IRBuilderBase::CreateMemSetInline(Value *Dst, MaybeAlign DstAlign,
                                            Value *Val, Value *Size,
                                            bool IsVolatile,
                                            const AAMDNodes &AAInfo) {
  Value *Ops[] = {Dst, Val, Size, getInt1(IsVolatile)};
  Type *Tys[] = {Dst->getType(), Size->getType()};
 
  auto *CI = cast<MemSetInst>(
      CreateIntrinsicWithoutFolding(Intrinsic::memset_inline, Tys, Ops));
 
  if (DstAlign)
    CI->setDestAlignment(*DstAlign);
  CI->setAAMetadata(AAInfo);
  return CI;
}
 
CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
    Value *Ptr, Value *Val, Value *Size, Align Alignment, uint32_t ElementSize,
    const AAMDNodes &AAInfo) {
 
  Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
  Type *Tys[] = {Ptr->getType(), Size->getType()};
 
  auto *CI = cast<AnyMemSetInst>(CreateIntrinsicWithoutFolding(
      Intrinsic::memset_element_unordered_atomic, Tys, Ops));
  CI->setDestAlignment(Alignment);
  CI->setAAMetadata(AAInfo);
  return CI;
}
 
CallInst *IRBuilderBase::CreateMemTransferInst(Intrinsic::ID IntrID, Value *Dst,
                                               MaybeAlign DstAlign, Value *Src,
                                               MaybeAlign SrcAlign, Value *Size,
                                               bool isVolatile,
                                               const AAMDNodes &AAInfo) {
  assert((IntrID == Intrinsic::memcpy || IntrID == Intrinsic::memcpy_inline ||
          IntrID == Intrinsic::memmove) &&
         "Unexpected intrinsic ID");
  Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 
  auto *MCI =
      cast<MemTransferInst>(CreateIntrinsicWithoutFolding(IntrID, Tys, Ops));
 
  if (DstAlign)
    MCI->setDestAlignment(*DstAlign);
  if (SrcAlign)
    MCI->setSourceAlignment(*SrcAlign);
  MCI->setAAMetadata(AAInfo);
  return MCI;
}
 
CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
    Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
    uint32_t ElementSize, const AAMDNodes &AAInfo) {
  assert(DstAlign >= ElementSize &&
         "Pointer alignment must be at least element size");
  assert(SrcAlign >= ElementSize &&
         "Pointer alignment must be at least element size");
  Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 
  auto *AMCI = cast<AnyMemCpyInst>(CreateIntrinsicWithoutFolding(
      Intrinsic::memcpy_element_unordered_atomic, Tys, Ops));
 
  // Set the alignment of the pointer args.
  AMCI->setDestAlignment(DstAlign);
  AMCI->setSourceAlignment(SrcAlign);
  AMCI->setAAMetadata(AAInfo);
  return AMCI;
}
 
/// isConstantOne - Return true only if val is constant int 1
static bool isConstantOne(const Value *Val) {
  assert(Val && "isConstantOne does not work with nullptr Val");
  const ConstantInt *CVal = dyn_cast<ConstantInt>(Val);
  return CVal && CVal->isOne();
}
 
CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
                                      Value *AllocSize, Value *ArraySize,
                                      ArrayRef<OperandBundleDef> OpB,
                                      Function *MallocF, const Twine &Name) {
  // malloc(type) becomes:
  //       i8* malloc(typeSize)
  // malloc(type, arraySize) becomes:
  //       i8* malloc(typeSize*arraySize)
  if (!ArraySize)
    ArraySize = ConstantInt::get(IntPtrTy, 1);
  else if (ArraySize->getType() != IntPtrTy)
    ArraySize = CreateIntCast(ArraySize, IntPtrTy, false);
 
  if (!isConstantOne(ArraySize)) {
    if (isConstantOne(AllocSize)) {
      AllocSize = ArraySize; // Operand * 1 = Operand
    } else {
      // Multiply type size by the array size...
      AllocSize = CreateMul(ArraySize, AllocSize, "mallocsize");
    }
  }
 
  assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
  // Create the call to Malloc.
  Module *M = BB->getParent()->getParent();
  Type *BPTy = PointerType::getUnqual(Context);
  FunctionCallee MallocFunc = MallocF;
  if (!MallocFunc)
    // prototype malloc as "void *malloc(size_t)"
    MallocFunc = M->getOrInsertFunction("malloc", BPTy, IntPtrTy);
  CallInst *MCall = CreateCall(MallocFunc, AllocSize, OpB, Name);
 
  MCall->setTailCall();
  if (Function *F = dyn_cast<Function>(MallocFunc.getCallee())) {
    MCall->setCallingConv(F->getCallingConv());
    F->setReturnDoesNotAlias();
  }
 
  assert(!MCall->getType()->isVoidTy() && "Malloc has void return type");
 
  return MCall;
}
 
CallInst *IRBuilderBase::CreateMalloc(Type *IntPtrTy, Type *AllocTy,
                                      Value *AllocSize, Value *ArraySize,
                                      Function *MallocF, const Twine &Name) {
 
  return CreateMalloc(IntPtrTy, AllocTy, AllocSize, ArraySize, {}, MallocF,
                      Name);
}
 
/// CreateFree - Generate the IR for a call to the builtin free function.
CallInst *IRBuilderBase::CreateFree(Value *Source,
                                    ArrayRef<OperandBundleDef> Bundles) {
  assert(Source->getType()->isPointerTy() &&
         "Can not free something of nonpointer type!");
 
  Module *M = BB->getParent()->getParent();
 
  Type *VoidTy = Type::getVoidTy(M->getContext());
  Type *VoidPtrTy = PointerType::getUnqual(M->getContext());
  // prototype free as "void free(void*)"
  FunctionCallee FreeFunc = M->getOrInsertFunction("free", VoidTy, VoidPtrTy);
  CallInst *Result = CreateCall(FreeFunc, Source, Bundles, "");
  Result->setTailCall();
  if (Function *F = dyn_cast<Function>(FreeFunc.getCallee()))
    Result->setCallingConv(F->getCallingConv());
 
  return Result;
}
 
CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
    Value *Dst, Align DstAlign, Value *Src, Align SrcAlign, Value *Size,
    uint32_t ElementSize, const AAMDNodes &AAInfo) {
  assert(DstAlign >= ElementSize &&
         "Pointer alignment must be at least element size");
  assert(SrcAlign >= ElementSize &&
         "Pointer alignment must be at least element size");
  Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
  Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 
  CallInst *CI = CreateIntrinsicWithoutFolding(
      Intrinsic::memmove_element_unordered_atomic, Tys, Ops);
 
  // Set the alignment of the pointer args.
  CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
  CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
  CI->setAAMetadata(AAInfo);
  return CI;
}
 
Value *IRBuilderBase::getReductionIntrinsic(Intrinsic::ID ID, Value *Src) {
  Value *Ops[] = {Src};
  Type *Tys[] = { Src->getType() };
  return CreateIntrinsic(ID, Tys, Ops);
}
 
Value *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
  Value *Ops[] = {Acc, Src};
  return CreateIntrinsic(Intrinsic::vector_reduce_fadd, {Src->getType()}, Ops);
}
 
Value *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
  Value *Ops[] = {Acc, Src};
  return CreateIntrinsic(Intrinsic::vector_reduce_fmul, {Src->getType()}, Ops);
}
 
Value *IRBuilderBase::CreateAddReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_add, Src);
}
 
Value *IRBuilderBase::CreateMulReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_mul, Src);
}
 
Value *IRBuilderBase::CreateAndReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_and, Src);
}
 
Value *IRBuilderBase::CreateOrReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_or, Src);
}
 
Value *IRBuilderBase::CreateXorReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_xor, Src);
}
 
Value *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
  auto ID =
      IsSigned ? Intrinsic::vector_reduce_smax : Intrinsic::vector_reduce_umax;
  return getReductionIntrinsic(ID, Src);
}
 
Value *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
  auto ID =
      IsSigned ? Intrinsic::vector_reduce_smin : Intrinsic::vector_reduce_umin;
  return getReductionIntrinsic(ID, Src);
}
 
Value *IRBuilderBase::CreateFPMaxReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_fmax, Src);
}
 
Value *IRBuilderBase::CreateFPMinReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_fmin, Src);
}
 
Value *IRBuilderBase::CreateFPMaximumReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximum, Src);
}
 
Value *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);
}
 
Value *IRBuilderBase::CreateFPMaximumNumReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_fmaximumnum, Src);
}
 
Value *IRBuilderBase::CreateFPMinimumNumReduce(Value *Src) {
  return getReductionIntrinsic(Intrinsic::vector_reduce_fminimumnum, Src);
}
 
CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
  assert(isa<PointerType>(Ptr->getType()) &&
         "lifetime.start only applies to pointers.");
  return CreateIntrinsicWithoutFolding(Intrinsic::lifetime_start,
                                       {Ptr->getType()}, {Ptr});
}
 
CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
  assert(isa<PointerType>(Ptr->getType()) &&
         "lifetime.end only applies to pointers.");
  return CreateIntrinsicWithoutFolding(Intrinsic::lifetime_end,
                                       {Ptr->getType()}, {Ptr});
}
 
CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
 
  assert(isa<PointerType>(Ptr->getType()) &&
         "invariant.start only applies to pointers.");
  if (!Size)
    Size = getInt64(-1);
  else
    assert(Size->getType() == getInt64Ty() &&
           "invariant.start requires the size to be an i64");
 
  Value *Ops[] = {Size, Ptr};
  // Fill in the single overloaded type: memory object type.
  Type *ObjectPtr[1] = {Ptr->getType()};
  return CreateIntrinsicWithoutFolding(Intrinsic::invariant_start, ObjectPtr,
                                       Ops);
}
 
static MaybeAlign getAlign(Value *Ptr) {
  if (auto *V = dyn_cast<GlobalVariable>(Ptr))
    return V->getAlign();
  if (auto *A = dyn_cast<GlobalAlias>(Ptr))
    return getAlign(A->getAliaseeObject());
  return {};
}
 
CallInst *IRBuilderBase::CreateThreadLocalAddress(Value *Ptr) {
  assert(isa<GlobalValue>(Ptr) && cast<GlobalValue>(Ptr)->isThreadLocal() &&
         "threadlocal_address only applies to thread local variables.");
  CallInst *CI = CreateIntrinsicWithoutFolding(
      llvm::Intrinsic::threadlocal_address, {Ptr->getType()}, {Ptr});
  if (MaybeAlign A = getAlign(Ptr)) {
    CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *A));
    CI->addRetAttr(Attribute::getWithAlignment(CI->getContext(), *A));
  }
  return CI;
}
 
CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
  assert(Cond->getType() == getInt1Ty() &&
         "an assumption condition must be of type i1");
  return CreateIntrinsicWithoutFolding(Intrinsic::assume, /*OverloadTypes=*/{},
                                       {Cond});
}
 
CallInst *
IRBuilderBase::CreateAssumption(ArrayRef<OperandBundleDef> OpBundles) {
  Value *Args[] = {ConstantInt::getTrue(getContext())};
  return CreateIntrinsicWithoutFolding(
      Intrinsic::assume, /*OverloadTypes=*/{}, Args,
      /*FMFSource=*/nullptr, /*Name=*/"", OpBundles);
}
 
Instruction *IRBuilderBase::CreateNoAliasScopeDeclaration(Value *Scope) {
  return CreateIntrinsicWithoutFolding(
      Intrinsic::experimental_noalias_scope_decl, {}, {Scope});
}
 
/// Create a call to a Masked Load intrinsic.
/// \p Ty        - vector type to load
/// \p Ptr       - base pointer for the load
/// \p Alignment - alignment of the source location
/// \p Mask      - vector of booleans which indicates what vector lanes should
///                be accessed in memory
/// \p PassThru  - pass-through value that is used to fill the masked-off lanes
///                of the result
/// \p Name      - name of the result variable
CallInst *IRBuilderBase::CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment,
                                          Value *Mask, Value *PassThru,
                                          const Twine &Name) {
  auto *PtrTy = cast<PointerType>(Ptr->getType());
  assert(Ty->isVectorTy() && "Type should be vector");
  assert(Mask && "Mask should not be all-ones (null)");
  if (!PassThru)
    PassThru = PoisonValue::get(Ty);
  Type *OverloadedTypes[] = { Ty, PtrTy };
  Value *Ops[] = {Ptr, Mask, PassThru};
  CallInst *CI =
      CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, OverloadedTypes, Name);
  CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
  return CI;
}
 
/// Create a call to a Masked Store intrinsic.
/// \p Val       - data to be stored,
/// \p Ptr       - base pointer for the store
/// \p Alignment - alignment of the destination location
/// \p Mask      - vector of booleans which indicates what vector lanes should
///                be accessed in memory
CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
                                           Align Alignment, Value *Mask) {
  auto *PtrTy = cast<PointerType>(Ptr->getType());
  Type *DataTy = Val->getType();
  assert(DataTy->isVectorTy() && "Val should be a vector");
  assert(Mask && "Mask should not be all-ones (null)");
  Type *OverloadedTypes[] = { DataTy, PtrTy };
  Value *Ops[] = {Val, Ptr, Mask};
  CallInst *CI =
      CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
  CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
  return CI;
}
 
/// Create a call to a Masked intrinsic, with given intrinsic Id,
/// an array of operands - Ops, and an array of overloaded types -
/// OverloadedTypes.
CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
                                               ArrayRef<Value *> Ops,
                                               ArrayRef<Type *> OverloadedTypes,
                                               const Twine &Name) {
  return CreateIntrinsicWithoutFolding(Id, OverloadedTypes, Ops, {}, Name);
}
 
/// Create a call to a Masked Gather intrinsic.
/// \p Ty       - vector type to gather
/// \p Ptrs     - vector of pointers for loading
/// \p Align    - alignment for one element
/// \p Mask     - vector of booleans which indicates what vector lanes should
///               be accessed in memory
/// \p PassThru - pass-through value that is used to fill the masked-off lanes
///               of the result
/// \p Name     - name of the result variable
CallInst *IRBuilderBase::CreateMaskedGather(Type *Ty, Value *Ptrs,
                                            Align Alignment, Value *Mask,
                                            Value *PassThru,
                                            const Twine &Name) {
  auto *VecTy = cast<VectorType>(Ty);
  ElementCount NumElts = VecTy->getElementCount();
  auto *PtrsTy = cast<VectorType>(Ptrs->getType());
  assert(NumElts == PtrsTy->getElementCount() && "Element count mismatch");
 
  if (!Mask)
    Mask = getAllOnesMask(NumElts);
 
  if (!PassThru)
    PassThru = PoisonValue::get(Ty);
 
  Type *OverloadedTypes[] = {Ty, PtrsTy};
  Value *Ops[] = {Ptrs, Mask, PassThru};
 
  // We specify only one type when we create this intrinsic. Types of other
  // arguments are derived from this type.
  CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops,
                                       OverloadedTypes, Name);
  CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), Alignment));
  return CI;
}
 
/// Create a call to a Masked Scatter intrinsic.
/// \p Data  - data to be stored,
/// \p Ptrs  - the vector of pointers, where the \p Data elements should be
///            stored
/// \p Align - alignment for one element
/// \p Mask  - vector of booleans which indicates what vector lanes should
///            be accessed in memory
CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
                                             Align Alignment, Value *Mask) {
  auto *PtrsTy = cast<VectorType>(Ptrs->getType());
  auto *DataTy = cast<VectorType>(Data->getType());
  ElementCount NumElts = PtrsTy->getElementCount();
 
  if (!Mask)
    Mask = getAllOnesMask(NumElts);
 
  Type *OverloadedTypes[] = {DataTy, PtrsTy};
  Value *Ops[] = {Data, Ptrs, Mask};
 
  // We specify only one type when we create this intrinsic. Types of other
  // arguments are derived from this type.
  CallInst *CI =
      CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
  CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), Alignment));
  return CI;
}
 
/// Create a call to Masked Expand Load intrinsic
/// \p Ty        - vector type to load
/// \p Ptr       - base pointer for the load
/// \p Align     - alignment of \p Ptr
/// \p Mask      - vector of booleans which indicates what vector lanes should
///                be accessed in memory
/// \p PassThru  - pass-through value that is used to fill the masked-off lanes
///                of the result
/// \p Name      - name of the result variable
CallInst *IRBuilderBase::CreateMaskedExpandLoad(Type *Ty, Value *Ptr,
                                                MaybeAlign Align, Value *Mask,
                                                Value *PassThru,
                                                const Twine &Name) {
  assert(Ty->isVectorTy() && "Type should be vector");
  assert(Mask && "Mask should not be all-ones (null)");
  if (!PassThru)
    PassThru = PoisonValue::get(Ty);
  Type *PtrTy = Ptr->getType();
  Type *OverloadedTypes[] = {Ty, PtrTy};
  Value *Ops[] = {Ptr, Mask, PassThru};
  CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_expandload, Ops,
                                       OverloadedTypes, Name);
  if (Align)
    CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), *Align));
  return CI;
}
 
/// Create a call to Masked Compress Store intrinsic
/// \p Val       - data to be stored,
/// \p Ptr       - base pointer for the store
/// \p Align     - alignment of \p Ptr
/// \p Mask      - vector of booleans which indicates what vector lanes should
///                be accessed in memory
CallInst *IRBuilderBase::CreateMaskedCompressStore(Value *Val, Value *Ptr,
                                                   MaybeAlign Align,
                                                   Value *Mask) {
  Type *DataTy = Val->getType();
  assert(DataTy->isVectorTy() && "Val should be a vector");
  assert(Mask && "Mask should not be all-ones (null)");
  Type *PtrTy = Ptr->getType();
  Type *OverloadedTypes[] = {DataTy, PtrTy};
  Value *Ops[] = {Val, Ptr, Mask};
  CallInst *CI = CreateMaskedIntrinsic(Intrinsic::masked_compressstore, Ops,
                                       OverloadedTypes);
  if (Align)
    CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), *Align));
  return CI;
}
 
template <typename T0>
static std::vector<Value *>
getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
                  Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs) {
  std::vector<Value *> Args;
  Args.push_back(B.getInt64(ID));
  Args.push_back(B.getInt32(NumPatchBytes));
  Args.push_back(ActualCallee);
  Args.push_back(B.getInt32(CallArgs.size()));
  Args.push_back(B.getInt32(Flags));
  llvm::append_range(Args, CallArgs);
  // GC Transition and Deopt args are now always handled via operand bundle.
  // They will be removed from the signature of gc.statepoint shortly.
  Args.push_back(B.getInt32(0));
  Args.push_back(B.getInt32(0));
  // GC args are now encoded in the gc-live operand bundle
  return Args;
}
 
template<typename T1, typename T2, typename T3>
static std::vector<OperandBundleDef>
getStatepointBundles(std::optional<ArrayRef<T1>> TransitionArgs,
                     std::optional<ArrayRef<T2>> DeoptArgs,
                     ArrayRef<T3> GCArgs) {
  std::vector<OperandBundleDef> Rval;
  if (DeoptArgs)
    Rval.emplace_back("deopt", SmallVector<Value *, 16>(*DeoptArgs));
  if (TransitionArgs)
    Rval.emplace_back("gc-transition",
                      SmallVector<Value *, 16>(*TransitionArgs));
  if (GCArgs.size())
    Rval.emplace_back("gc-live", SmallVector<Value *, 16>(GCArgs));
  return Rval;
}
 
template <typename T0, typename T1, typename T2, typename T3>
static CallInst *CreateGCStatepointCallCommon(
    IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
    FunctionCallee ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
    std::optional<ArrayRef<T1>> TransitionArgs,
    std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
    const Twine &Name) {
  Module *M = Builder->GetInsertBlock()->getParent()->getParent();
  // Fill in the one generic type'd argument (the function is also vararg)
  Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
      M, Intrinsic::experimental_gc_statepoint,
      {ActualCallee.getCallee()->getType()});
 
  std::vector<Value *> Args = getStatepointArgs(
      *Builder, ID, NumPatchBytes, ActualCallee.getCallee(), Flags, CallArgs);
 
  CallInst *CI = Builder->CreateCall(
      FnStatepoint, Args,
      getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
  CI->addParamAttr(2,
                   Attribute::get(Builder->getContext(), Attribute::ElementType,
                                  ActualCallee.getFunctionType()));
  return CI;
}
 
CallInst *IRBuilderBase::CreateGCStatepointCall(
    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
    ArrayRef<Value *> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
    ArrayRef<Value *> GCArgs, const Twine &Name) {
  return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
      this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
      CallArgs, std::nullopt /* No Transition Args */, DeoptArgs, GCArgs, Name);
}
 
CallInst *IRBuilderBase::CreateGCStatepointCall(
    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
    uint32_t Flags, ArrayRef<Value *> CallArgs,
    std::optional<ArrayRef<Use>> TransitionArgs,
    std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
    const Twine &Name) {
  return CreateGCStatepointCallCommon<Value *, Use, Use, Value *>(
      this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
      DeoptArgs, GCArgs, Name);
}
 
CallInst *IRBuilderBase::CreateGCStatepointCall(
    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualCallee,
    ArrayRef<Use> CallArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
    ArrayRef<Value *> GCArgs, const Twine &Name) {
  return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
      this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
      CallArgs, std::nullopt, DeoptArgs, GCArgs, Name);
}
 
template <typename T0, typename T1, typename T2, typename T3>
static InvokeInst *CreateGCStatepointInvokeCommon(
    IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
    FunctionCallee ActualInvokee, BasicBlock *NormalDest,
    BasicBlock *UnwindDest, uint32_t Flags, ArrayRef<T0> InvokeArgs,
    std::optional<ArrayRef<T1>> TransitionArgs,
    std::optional<ArrayRef<T2>> DeoptArgs, ArrayRef<T3> GCArgs,
    const Twine &Name) {
  Module *M = Builder->GetInsertBlock()->getParent()->getParent();
  // Fill in the one generic type'd argument (the function is also vararg)
  Function *FnStatepoint = Intrinsic::getOrInsertDeclaration(
      M, Intrinsic::experimental_gc_statepoint,
      {ActualInvokee.getCallee()->getType()});
 
  std::vector<Value *> Args =
      getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee.getCallee(),
                        Flags, InvokeArgs);
 
  InvokeInst *II = Builder->CreateInvoke(
      FnStatepoint, NormalDest, UnwindDest, Args,
      getStatepointBundles(TransitionArgs, DeoptArgs, GCArgs), Name);
  II->addParamAttr(2,
                   Attribute::get(Builder->getContext(), Attribute::ElementType,
                                  ActualInvokee.getFunctionType()));
  return II;
}
 
InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
    BasicBlock *NormalDest, BasicBlock *UnwindDest,
    ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Value *>> DeoptArgs,
    ArrayRef<Value *> GCArgs, const Twine &Name) {
  return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
      uint32_t(StatepointFlags::None), InvokeArgs,
      std::nullopt /* No Transition Args*/, DeoptArgs, GCArgs, Name);
}
 
InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
    BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
    ArrayRef<Value *> InvokeArgs, std::optional<ArrayRef<Use>> TransitionArgs,
    std::optional<ArrayRef<Use>> DeoptArgs, ArrayRef<Value *> GCArgs,
    const Twine &Name) {
  return CreateGCStatepointInvokeCommon<Value *, Use, Use, Value *>(
      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
      InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
}
 
InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
    uint64_t ID, uint32_t NumPatchBytes, FunctionCallee ActualInvokee,
    BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
    std::optional<ArrayRef<Value *>> DeoptArgs, ArrayRef<Value *> GCArgs,
    const Twine &Name) {
  return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
      this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
      uint32_t(StatepointFlags::None), InvokeArgs, std::nullopt, DeoptArgs,
      GCArgs, Name);
}
 
CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
                                        Type *ResultType, const Twine &Name) {
  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
  Type *Types[] = {ResultType};
 
  Value *Args[] = {Statepoint};
  return CreateIntrinsicWithoutFolding(ID, Types, Args, {}, Name);
}
 
CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
                                          int BaseOffset, int DerivedOffset,
                                          Type *ResultType, const Twine &Name) {
  Type *Types[] = {ResultType};
 
  Value *Args[] = {Statepoint, getInt32(BaseOffset), getInt32(DerivedOffset)};
  return CreateIntrinsicWithoutFolding(Intrinsic::experimental_gc_relocate,
                                       Types, Args, {}, Name);
}
 
CallInst *IRBuilderBase::CreateGCGetPointerBase(Value *DerivedPtr,
                                                const Twine &Name) {
  Type *PtrTy = DerivedPtr->getType();
  return CreateIntrinsicWithoutFolding(
      Intrinsic::experimental_gc_get_pointer_base, PtrTy, DerivedPtr, {}, Name);
}
 
CallInst *IRBuilderBase::CreateGCGetPointerOffset(Value *DerivedPtr,
                                                  const Twine &Name) {
  Type *PtrTy = DerivedPtr->getType();
  return CreateIntrinsicWithoutFolding(
      Intrinsic::experimental_gc_get_pointer_offset, {PtrTy}, {DerivedPtr}, {},
      Name);
}
 
Value *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op,
                                           FMFSource FMFSource,
                                           const Twine &Name) {
  Module *M = BB->getModule();
  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, Op->getType());
  if (Value *V =
          Folder.FoldIntrinsic(ID, Op, Fn->getReturnType(), FMFSource.get(FMF),
                               GetInsertBlock()->getParent()))
    return V;
  return createCallHelper(Fn, Op, Name, FMFSource);
}
 
Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
                                            Value *RHS, FMFSource FMFSource,
                                            const Twine &Name) {
  Module *M = BB->getModule();
  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, {LHS->getType()});
  if (Value *V = Folder.FoldIntrinsic(ID, {LHS, RHS}, Fn->getReturnType(),
                                      FMFSource.get(FMF),
                                      GetInsertBlock()->getParent()))
    return V;
  return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
}
 
CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(
    Intrinsic::ID ID, ArrayRef<Type *> OverloadTypes, ArrayRef<Value *> Args,
    FMFSource FMFSource, const Twine &Name,
    ArrayRef<OperandBundleDef> OpBundles) {
  Module *M = BB->getModule();
  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, OverloadTypes);
  return createCallHelper(Fn, Args, Name, FMFSource, OpBundles);
}
 
CallInst *IRBuilderBase::CreateIntrinsicWithoutFolding(Type *RetTy,
                                                       Intrinsic::ID ID,
                                                       ArrayRef<Value *> Args,
                                                       FMFSource FMFSource,
                                                       const Twine &Name) {
  Module *M = BB->getModule();
  SmallVector<Type *> ArgTys = llvm::map_to_vector(Args, &Value::getType);
  Function *Fn = Intrinsic::getOrInsertDeclaration(M, ID, RetTy, ArgTys);
  return createCallHelper(Fn, Args, Name, FMFSource);
}
 
Value *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
                                      ArrayRef<Type *> OverloadTypes,
                                      ArrayRef<Value *> Args,
                                      FMFSource FMFSource, const Twine &Name,
                                      ArrayRef<OperandBundleDef> OpBundles,
                                      function_ref<void(CallInst *)> SetFn) {
  Type *RetTy = Intrinsic::getType(Context, ID, OverloadTypes)->getReturnType();
  if (Value *V = Folder.FoldIntrinsic(ID, Args, RetTy, FMFSource.get(FMF),
                                      GetInsertBlock()->getParent()))
    return V;
  CallInst *CI = CreateIntrinsicWithoutFolding(ID, OverloadTypes, Args,
                                               FMFSource, Name, OpBundles);
  SetFn(CI);
  return CI;
}
 
Value *IRBuilderBase::CreateIntrinsic(Type *RetTy, Intrinsic::ID ID,
                                      ArrayRef<Value *> Args,
                                      FMFSource FMFSource, const Twine &Name,
                                      function_ref<void(CallInst *)> SetFn) {
  if (Value *V = Folder.FoldIntrinsic(ID, Args, RetTy, FMFSource.get(FMF),
                                      GetInsertBlock()->getParent()))
    return V;
  CallInst *CI =
      CreateIntrinsicWithoutFolding(RetTy, ID, Args, FMFSource, Name);
  SetFn(CI);
  return CI;
}
 
CallInst *IRBuilderBase::CreateConstrainedFPBinOp(
    Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
    const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
    std::optional<fp::ExceptionBehavior> Except) {
  Value *RoundingV = getConstrainedFPRounding(Rounding);
  Value *ExceptV = getConstrainedFPExcept(Except);
 
  FastMathFlags UseFMF = FMFSource.get(FMF);
  CallInst *C = CreateIntrinsicWithoutFolding(
      ID, {L->getType()}, {L, R, RoundingV, ExceptV}, nullptr, Name, {});
  setConstrainedFPCallAttr(C);
  setFPAttrs(C, FPMathTag, UseFMF);
  return C;
}
 
CallInst *IRBuilderBase::CreateConstrainedFPIntrinsic(
    Intrinsic::ID ID, ArrayRef<Type *> Types, ArrayRef<Value *> Args,
    FMFSource FMFSource, const Twine &Name, MDNode *FPMathTag,
    std::optional<RoundingMode> Rounding,
    std::optional<fp::ExceptionBehavior> Except) {
  Value *RoundingV = getConstrainedFPRounding(Rounding);
  Value *ExceptV = getConstrainedFPExcept(Except);
 
  FastMathFlags UseFMF = FMFSource.get(FMF);
 
  llvm::SmallVector<Value *, 5> ExtArgs(Args);
  ExtArgs.push_back(RoundingV);
  ExtArgs.push_back(ExceptV);
  CallInst *C =
      CreateIntrinsicWithoutFolding(ID, Types, ExtArgs, nullptr, Name, {});
  setConstrainedFPCallAttr(C);
  setFPAttrs(C, FPMathTag, UseFMF);
  return C;
}
 
CallInst *IRBuilderBase::CreateConstrainedFPUnroundedBinOp(
    Intrinsic::ID ID, Value *L, Value *R, FMFSource FMFSource,
    const Twine &Name, MDNode *FPMathTag,
    std::optional<fp::ExceptionBehavior> Except) {
  Value *ExceptV = getConstrainedFPExcept(Except);
 
  FastMathFlags UseFMF = FMFSource.get(FMF);
  CallInst *C = CreateIntrinsicWithoutFolding(
      ID, {L->getType()}, {L, R, ExceptV}, nullptr, Name, {});
  setConstrainedFPCallAttr(C);
  setFPAttrs(C, FPMathTag, UseFMF);
  return C;
}
 
Value *IRBuilderBase::CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops,
                                   const Twine &Name, MDNode *FPMathTag) {
  if (Instruction::isBinaryOp(Opc)) {
    assert(Ops.size() == 2 && "Invalid number of operands!");
    return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc),
                       Ops[0], Ops[1], Name, FPMathTag);
  }
  if (Instruction::isUnaryOp(Opc)) {
    assert(Ops.size() == 1 && "Invalid number of operands!");
    return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc),
                      Ops[0], Name, FPMathTag);
  }
  llvm_unreachable("Unexpected opcode!");
}
 
CallInst *IRBuilderBase::CreateConstrainedFPCast(
    Intrinsic::ID ID, Value *V, Type *DestTy, FMFSource FMFSource,
    const Twine &Name, MDNode *FPMathTag, std::optional<RoundingMode> Rounding,
    std::optional<fp::ExceptionBehavior> Except) {
  Value *ExceptV = getConstrainedFPExcept(Except);
 
  FastMathFlags UseFMF = FMFSource.get(FMF);
 
  CallInst *C;
  if (Intrinsic::hasConstrainedFPRoundingModeOperand(ID)) {
    Value *RoundingV = getConstrainedFPRounding(Rounding);
    C = CreateIntrinsicWithoutFolding(
        ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, nullptr, Name, {});
  } else
    C = CreateIntrinsicWithoutFolding(ID, {DestTy, V->getType()}, {V, ExceptV},
                                      nullptr, Name, {});
  setConstrainedFPCallAttr(C);
 
  if (isa<FPMathOperator>(C))
    setFPAttrs(C, FPMathTag, UseFMF);
  return C;
}
 
Value *IRBuilderBase::CreateFCmpHelper(CmpInst::Predicate P, Value *LHS,
                                       Value *RHS, const Twine &Name,
                                       MDNode *FPMathTag, FMFSource FMFSource,
                                       bool IsSignaling) {
  if (IsFPConstrained) {
    auto ID = IsSignaling ? Intrinsic::experimental_constrained_fcmps
                          : Intrinsic::experimental_constrained_fcmp;
    return CreateConstrainedFPCmp(ID, P, LHS, RHS, Name);
  }
 
  if (auto *V = Folder.FoldCmp(P, LHS, RHS))
    return V;
  return Insert(
      setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMFSource.get(FMF)),
      Name);
}
 
CallInst *IRBuilderBase::CreateConstrainedFPCmp(
    Intrinsic::ID ID, CmpInst::Predicate P, Value *L, Value *R,
    const Twine &Name, std::optional<fp::ExceptionBehavior> Except) {
  Value *PredicateV = getConstrainedFPPredicate(P);
  Value *ExceptV = getConstrainedFPExcept(Except);
 
  CallInst *C = CreateIntrinsicWithoutFolding(
      ID, {L->getType()}, {L, R, PredicateV, ExceptV}, nullptr, Name, {});
  setConstrainedFPCallAttr(C);
  return C;
}
 
CallInst *IRBuilderBase::CreateConstrainedFPCall(
    Function *Callee, ArrayRef<Value *> Args, const Twine &Name,
    std::optional<RoundingMode> Rounding,
    std::optional<fp::ExceptionBehavior> Except) {
  llvm::SmallVector<Value *, 6> UseArgs(Args);
 
  if (Intrinsic::hasConstrainedFPRoundingModeOperand(Callee->getIntrinsicID()))
    UseArgs.push_back(getConstrainedFPRounding(Rounding));
  UseArgs.push_back(getConstrainedFPExcept(Except));
 
  CallInst *C = CreateCall(Callee, UseArgs, Name);
  setConstrainedFPCallAttr(C);
  return C;
}
 
Value *IRBuilderBase::CreateSelectWithUnknownProfile(Value *C, Value *True,
                                                     Value *False,
                                                     StringRef PassName,
                                                     const Twine &Name) {
  Value *Ret = CreateSelectFMF(C, True, False, {}, Name);
  if (auto *SI = dyn_cast<SelectInst>(Ret)) {
    setExplicitlyUnknownBranchWeightsIfProfiled(*SI, PassName);
  }
  return Ret;
}
 
Value *IRBuilderBase::CreateSelectFMFWithUnknownProfile(Value *C, Value *True,
                                                        Value *False,
                                                        FMFSource FMFSource,
                                                        StringRef PassName,
                                                        const Twine &Name) {
  Value *Ret = CreateSelectFMF(C, True, False, FMFSource, Name);
  if (auto *SI = dyn_cast<SelectInst>(Ret))
    setExplicitlyUnknownBranchWeightsIfProfiled(*SI, PassName);
  return Ret;
}
 
Value *IRBuilderBase::CreateSelect(Value *C, Value *True, Value *False,
                                   const Twine &Name, Instruction *MDFrom) {
  return CreateSelectFMF(C, True, False, {}, Name, MDFrom);
}
 
Value *IRBuilderBase::CreateSelectFMF(Value *C, Value *True, Value *False,
                                      FMFSource FMFSource, const Twine &Name,
                                      Instruction *MDFrom) {
  if (auto *V = Folder.FoldSelect(C, True, False, FMFSource.get(FMF)))
    return V;
 
  SelectInst *Sel = SelectInst::Create(C, True, False);
  if (MDFrom) {
    MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof);
    MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable);
    Sel = addBranchMetadata(Sel, Prof, Unpred);
  }
  if (isa<FPMathOperator>(Sel))
    setFPAttrs(Sel, /*MDNode=*/nullptr, FMFSource.get(FMF));
  return Insert(Sel, Name);
}
 
Value *IRBuilderBase::CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name,
                                    bool IsNUW) {
  assert(LHS->getType() == RHS->getType() &&
         "Pointer subtraction operand types must match!");
  Value *LHSAddr = CreatePtrToAddr(LHS);
  Value *RHSAddr = CreatePtrToAddr(RHS);
  return CreateSub(LHSAddr, RHSAddr, Name, IsNUW);
}
Value *IRBuilderBase::CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS,
                                    const Twine &Name) {
  const DataLayout &DL = BB->getDataLayout();
  TypeSize ElemSize = DL.getTypeAllocSize(ElemTy);
  if (ElemSize == TypeSize::getFixed(1))
    return CreatePtrDiff(LHS, RHS, Name);
 
  Value *Diff = CreatePtrDiff(LHS, RHS);
  return CreateExactSDiv(Diff, CreateTypeSize(Diff->getType(), ElemSize), Name);
}
 
Value *IRBuilderBase::CreateLaunderInvariantGroup(Value *Ptr) {
  assert(isa<PointerType>(Ptr->getType()) &&
         "launder.invariant.group only applies to pointers.");
  auto *PtrType = Ptr->getType();
  Module *M = BB->getParent()->getParent();
  Function *FnLaunderInvariantGroup = Intrinsic::getOrInsertDeclaration(
      M, Intrinsic::launder_invariant_group, {PtrType});
 
  assert(FnLaunderInvariantGroup->getReturnType() == PtrType &&
         FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==
             PtrType &&
         "LaunderInvariantGroup should take and return the same type");
 
  return CreateCall(FnLaunderInvariantGroup, {Ptr});
}
 
Value *IRBuilderBase::CreateStripInvariantGroup(Value *Ptr) {
  assert(isa<PointerType>(Ptr->getType()) &&
         "strip.invariant.group only applies to pointers.");
 
  auto *PtrType = Ptr->getType();
  Module *M = BB->getParent()->getParent();
  Function *FnStripInvariantGroup = Intrinsic::getOrInsertDeclaration(
      M, Intrinsic::strip_invariant_group, {PtrType});
 
  assert(FnStripInvariantGroup->getReturnType() == PtrType &&
         FnStripInvariantGroup->getFunctionType()->getParamType(0) ==
             PtrType &&
         "StripInvariantGroup should take and return the same type");
 
  return CreateCall(FnStripInvariantGroup, {Ptr});
}
 
Value *IRBuilderBase::CreateVectorReverse(Value *V, const Twine &Name) {
  auto *Ty = cast<VectorType>(V->getType());
  if (isa<ScalableVectorType>(Ty)) {
    Module *M = BB->getParent()->getParent();
    Function *F =
        Intrinsic::getOrInsertDeclaration(M, Intrinsic::vector_reverse, Ty);
    return Insert(CallInst::Create(F, V), Name);
  }
  // Keep the original behaviour for fixed vector
  SmallVector<int, 8> ShuffleMask;
  int NumElts = Ty->getElementCount().getKnownMinValue();
  for (int i = 0; i < NumElts; ++i)
    ShuffleMask.push_back(NumElts - i - 1);
  return CreateShuffleVector(V, ShuffleMask, Name);
}
 
static SmallVector<int, 8> getSpliceMask(int64_t Imm, unsigned NumElts) {
  unsigned Idx = (NumElts + Imm) % NumElts;
  SmallVector<int, 8> Mask;
  for (unsigned I = 0; I < NumElts; ++I)
    Mask.push_back(Idx + I);
  return Mask;
}
 
Value *IRBuilderBase::CreateVectorSpliceLeft(Value *V1, Value *V2,
                                             Value *Offset, const Twine &Name) {
  assert(isa<VectorType>(V1->getType()) && "Unexpected type");
  assert(V1->getType() == V2->getType() &&
         "Splice expects matching operand types!");
 
  // Emit a shufflevector for fixed vectors with a constant offset
  if (auto *COffset = dyn_cast<ConstantInt>(Offset))
    if (auto *FVTy = dyn_cast<FixedVectorType>(V1->getType()))
      return CreateShuffleVector(
          V1, V2,
          getSpliceMask(COffset->getZExtValue(), FVTy->getNumElements()));
 
  return CreateIntrinsic(Intrinsic::vector_splice_left, V1->getType(),
                         {V1, V2, Offset}, {}, Name);
}
 
Value *IRBuilderBase::CreateVectorSpliceRight(Value *V1, Value *V2,
                                              Value *Offset,
                                              const Twine &Name) {
  assert(isa<VectorType>(V1->getType()) && "Unexpected type");
  assert(V1->getType() == V2->getType() &&
         "Splice expects matching operand types!");
 
  // Emit a shufflevector for fixed vectors with a constant offset
  if (auto *COffset = dyn_cast<ConstantInt>(Offset))
    if (auto *FVTy = dyn_cast<FixedVectorType>(V1->getType()))
      return CreateShuffleVector(
          V1, V2,
          getSpliceMask(-COffset->getZExtValue(), FVTy->getNumElements()));
 
  return CreateIntrinsic(Intrinsic::vector_splice_right, V1->getType(),
                         {V1, V2, Offset}, {}, Name);
}
 
Value *IRBuilderBase::CreateVectorSplat(unsigned NumElts, Value *V,
                                        const Twine &Name) {
  auto EC = ElementCount::getFixed(NumElts);
  return CreateVectorSplat(EC, V, Name);
}
 
Value *IRBuilderBase::CreateVectorSplat(ElementCount EC, Value *V,
                                        const Twine &Name) {
  assert(EC.isNonZero() && "Cannot splat to an empty vector!");
 
  // First insert it into a poison vector so we can shuffle it.
  Value *Poison = PoisonValue::get(VectorType::get(V->getType(), EC));
  V = CreateInsertElement(Poison, V, getInt64(0), Name + ".splatinsert");
 
  // Shuffle the value across the desired number of elements.
  SmallVector<int, 16> Zeros;
  Zeros.resize(EC.getKnownMinValue());
  return CreateShuffleVector(V, Zeros, Name + ".splat");
}
 
Value *IRBuilderBase::CreateVectorInterleave(ArrayRef<Value *> Ops,
                                             const Twine &Name) {
  assert(Ops.size() >= 2 && Ops.size() <= 8 &&
         "Unexpected number of operands to interleave");
 
  // Make sure all operands are the same type.
  assert(isa<VectorType>(Ops[0]->getType()) && "Unexpected type");
 
#ifndef NDEBUG
  for (unsigned I = 1; I < Ops.size(); I++) {
    assert(Ops[I]->getType() == Ops[0]->getType() &&
           "Vector interleave expects matching operand types!");
  }
#endif
 
  unsigned IID = Intrinsic::getInterleaveIntrinsicID(Ops.size());
  auto *SubvecTy = cast<VectorType>(Ops[0]->getType());
  Type *DestTy = VectorType::get(SubvecTy->getElementType(),
                                 SubvecTy->getElementCount() * Ops.size());
  return CreateIntrinsic(IID, {DestTy}, Ops, {}, Name);
}
 
Value *IRBuilderBase::CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base,
                                                     unsigned Dimension,
                                                     unsigned LastIndex,
                                                     MDNode *DbgInfo) {
  auto *BaseType = Base->getType();
  assert(isa<PointerType>(BaseType) &&
         "Invalid Base ptr type for preserve.array.access.index.");
 
  Value *LastIndexV = getInt32(LastIndex);
  Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
  SmallVector<Value *, 4> IdxList(Dimension, Zero);
  IdxList.push_back(LastIndexV);
 
  Type *ResultType = GetElementPtrInst::getGEPReturnType(Base, IdxList);
 
  Value *DimV = getInt32(Dimension);
  CallInst *Fn = CreateIntrinsicWithoutFolding(
      Intrinsic::preserve_array_access_index, {ResultType, BaseType},
      {Base, DimV, LastIndexV});
  Fn->addParamAttr(
      0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
  if (DbgInfo)
    Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
 
  return Fn;
}
 
Value *IRBuilderBase::CreatePreserveUnionAccessIndex(
    Value *Base, unsigned FieldIndex, MDNode *DbgInfo) {
  assert(isa<PointerType>(Base->getType()) &&
         "Invalid Base ptr type for preserve.union.access.index.");
  auto *BaseType = Base->getType();
 
  Value *DIIndex = getInt32(FieldIndex);
  CallInst *Fn =
      CreateIntrinsicWithoutFolding(Intrinsic::preserve_union_access_index,
                                    {BaseType, BaseType}, {Base, DIIndex});
  if (DbgInfo)
    Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
 
  return Fn;
}
 
Value *IRBuilderBase::CreatePreserveStructAccessIndex(
    Type *ElTy, Value *Base, unsigned Index, unsigned FieldIndex,
    MDNode *DbgInfo) {
  auto *BaseType = Base->getType();
  assert(isa<PointerType>(BaseType) &&
         "Invalid Base ptr type for preserve.struct.access.index.");
 
  Value *GEPIndex = getInt32(Index);
  Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0);
  Type *ResultType =
      GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex});
 
  Value *DIIndex = getInt32(FieldIndex);
  CallInst *Fn = CreateIntrinsicWithoutFolding(
      Intrinsic::preserve_struct_access_index, {ResultType, BaseType},
      {Base, GEPIndex, DIIndex});
  Fn->addParamAttr(
      0, Attribute::get(Fn->getContext(), Attribute::ElementType, ElTy));
  if (DbgInfo)
    Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo);
 
  return Fn;
}
 
Value *IRBuilderBase::createIsFPClass(Value *FPNum, unsigned Test) {
  ConstantInt *TestV = getInt32(Test);
  return CreateIntrinsic(Intrinsic::is_fpclass, {FPNum->getType()},
                         {FPNum, TestV});
}
 
CallInst *IRBuilderBase::CreateAlignmentAssumptionHelper(const DataLayout &DL,
                                                         Value *PtrValue,
                                                         Value *AlignValue,
                                                         Value *OffsetValue) {
  SmallVector<Value *, 4> Vals({PtrValue, AlignValue});
  if (OffsetValue)
    Vals.push_back(OffsetValue);
  OperandBundleDefT<Value *> AlignOpB("align", Vals);
  return CreateAssumption({AlignOpB});
}
 
CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
                                                   Value *PtrValue,
                                                   uint64_t Alignment,
                                                   Value *OffsetValue) {
  assert(isa<PointerType>(PtrValue->getType()) &&
         "trying to create an alignment assumption on a non-pointer?");
  assert(Alignment != 0 && "Invalid Alignment");
  Value *AlignValue = ConstantInt::get(getInt64Ty(), Alignment);
  return CreateAlignmentAssumptionHelper(DL, PtrValue, AlignValue, OffsetValue);
}
 
CallInst *IRBuilderBase::CreateAlignmentAssumption(const DataLayout &DL,
                                                   Value *PtrValue,
                                                   Value *Alignment,
                                                   Value *OffsetValue) {
  assert(isa<PointerType>(PtrValue->getType()) &&
         "trying to create an alignment assumption on a non-pointer?");
  return CreateAlignmentAssumptionHelper(DL, PtrValue, Alignment, OffsetValue);
}
 
CallInst *IRBuilderBase::CreateDereferenceableAssumption(Value *PtrValue,
                                                         Value *SizeValue) {
  assert(isa<PointerType>(PtrValue->getType()) &&
         "trying to create a deferenceable assumption on a non-pointer?");
  SmallVector<Value *, 4> Vals({PtrValue, SizeValue});
  OperandBundleDefT<Value *> DereferenceableOpB("dereferenceable", Vals);
  return CreateAssumption({DereferenceableOpB});
}
 
CallInst *IRBuilderBase::CreateNonnullAssumption(Value *PtrValue) {
  assert(isa<PointerType>(PtrValue->getType()) &&
         "trying to create a nonnull assumption on a non-pointer?");
  return CreateAssumption(OperandBundleDef("nonnull", PtrValue));
}
 
IRBuilderDefaultInserter::~IRBuilderDefaultInserter() = default;
IRBuilderCallbackInserter::~IRBuilderCallbackInserter() = default;
IRBuilderFolder::~IRBuilderFolder() = default;
void ConstantFolder::anchor() {}
void NoFolder::anchor() {}