PPCISelLowering.cpp

Go to the documentation of this file.

//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the PPCISelLowering class.
//
//===----------------------------------------------------------------------===//
 
#include "PPCISelLowering.h"
#include "MCTargetDesc/PPCMCTargetDesc.h"
#include "MCTargetDesc/PPCPredicates.h"
#include "PPC.h"
#include "PPCCallingConv.h"
#include "PPCFrameLowering.h"
#include "PPCInstrInfo.h"
#include "PPCMachineFunctionInfo.h"
#include "PPCPerfectShuffle.h"
#include "PPCRegisterInfo.h"
#include "PPCSelectionDAGInfo.h"
#include "PPCSubtarget.h"
#include "PPCTargetMachine.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/APSInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsPowerPC.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSectionXCOFF.h"
#include "llvm/MC/MCSymbolXCOFF.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <iterator>
#include <list>
#include <optional>
#include <utility>
#include <vector>
 
using namespace llvm;
 
#define DEBUG_TYPE "ppc-lowering"
 
static cl::opt<bool> DisableP10StoreForward(
    "disable-p10-store-forward",
    cl::desc("disable P10 store forward-friendly conversion"), cl::Hidden,
    cl::init(false));
 
static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc",
cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden);
 
static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref",
cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden);
 
static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned",
cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden);
 
static cl::opt<bool> DisableSCO("disable-ppc-sco",
cl::desc("disable sibling call optimization on ppc"), cl::Hidden);
 
static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32",
cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden);
 
static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables",
cl::desc("use absolute jump tables on ppc"), cl::Hidden);
 
static cl::opt<bool>
    DisablePerfectShuffle("ppc-disable-perfect-shuffle",
                          cl::desc("disable vector permute decomposition"),
                          cl::init(true), cl::Hidden);
 
cl::opt<bool> DisableAutoPairedVecSt(
    "disable-auto-paired-vec-st",
    cl::desc("disable automatically generated 32byte paired vector stores"),
    cl::init(true), cl::Hidden);
 
static cl::opt<unsigned> PPCMinimumJumpTableEntries(
    "ppc-min-jump-table-entries", cl::init(64), cl::Hidden,
    cl::desc("Set minimum number of entries to use a jump table on PPC"));
 
static cl::opt<unsigned> PPCMinimumBitTestCmps(
    "ppc-min-bit-test-cmps", cl::init(3), cl::Hidden,
    cl::desc("Set minimum of largest number of comparisons to use bit test for "
             "switch on PPC."));
 
static cl::opt<unsigned> PPCGatherAllAliasesMaxDepth(
    "ppc-gather-alias-max-depth", cl::init(18), cl::Hidden,
    cl::desc("max depth when checking alias info in GatherAllAliases()"));
 
static cl::opt<unsigned> PPCAIXTLSModelOptUseIEForLDLimit(
    "ppc-aix-shared-lib-tls-model-opt-limit", cl::init(1), cl::Hidden,
    cl::desc("Set inclusive limit count of TLS local-dynamic access(es) in a "
             "function to use initial-exec"));
 
STATISTIC(NumTailCalls, "Number of tail calls");
STATISTIC(NumSiblingCalls, "Number of sibling calls");
STATISTIC(ShufflesHandledWithVPERM,
          "Number of shuffles lowered to a VPERM or XXPERM");
STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed");
 
static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
 
static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
 
static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
                                       unsigned OpIdx, bool IsByte,
                                       const PPCInstrInfo *TII);
 
// A faster local-[exec|dynamic] TLS access sequence (enabled with the
// -maix-small-local-[exec|dynamic]-tls option) can be produced for TLS
// variables; consistent with the IBM XL compiler, we apply a max size of
// slightly under 32KB.
constexpr uint64_t AIXSmallTlsPolicySizeLimit = 32751;
 
// FIXME: Remove this once the bug has been fixed!
extern cl::opt<bool> ANDIGlueBug;
 
PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
                                     const PPCSubtarget &STI)
    : TargetLowering(TM, STI), Subtarget(STI) {
  // Initialize map that relates the PPC addressing modes to the computed flags
  // of a load/store instruction. The map is used to determine the optimal
  // addressing mode when selecting load and stores.
  initializeAddrModeMap();
  // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
  // arguments are at least 4/8 bytes aligned.
  bool isPPC64 = Subtarget.isPPC64();
  setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4));
  const MVT RegVT = Subtarget.getScalarIntVT();
 
  // Set up the register classes.
  addRegisterClass(MVT::i32, &PPC::GPRCRegClass);
  if (!useSoftFloat()) {
    if (hasSPE()) {
      addRegisterClass(MVT::f32, &PPC::GPRCRegClass);
      // EFPU2 APU only supports f32
      if (!Subtarget.hasEFPU2())
        addRegisterClass(MVT::f64, &PPC::SPERCRegClass);
    } else {
      addRegisterClass(MVT::f32, &PPC::F4RCRegClass);
      addRegisterClass(MVT::f64, &PPC::F8RCRegClass);
    }
  }
 
  setOperationAction(ISD::UADDO, RegVT, Custom);
  setOperationAction(ISD::USUBO, RegVT, Custom);
 
  // PowerPC uses addo_carry,subo_carry to propagate carry.
  setOperationAction(ISD::UADDO_CARRY, RegVT, Custom);
  setOperationAction(ISD::USUBO_CARRY, RegVT, Custom);
 
  // On P10, the default lowering generates better code using the
  // setbc instruction.
  if (!Subtarget.hasP10Vector()) {
    setOperationAction(ISD::SSUBO, MVT::i32, Custom);
    setOperationAction(ISD::SADDO, MVT::i32, Custom);
    if (isPPC64) {
      setOperationAction(ISD::SSUBO, MVT::i64, Custom);
      setOperationAction(ISD::SADDO, MVT::i64, Custom);
    }
  }
 
  // Match BITREVERSE to customized fast code sequence in the td file.
  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
  setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
 
  // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended.
  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom);
 
  // Custom lower inline assembly to check for special registers.
  setOperationAction(ISD::INLINEASM, MVT::Other, Custom);
  setOperationAction(ISD::INLINEASM_BR, MVT::Other, Custom);
 
  // PowerPC has an i16 but no i8 (or i1) SEXTLOAD.
  for (MVT VT : MVT::integer_valuetypes()) {
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand);
  }
 
  setTruncStoreAction(MVT::f128, MVT::f16, Expand);
  setOperationAction(ISD::FP_TO_FP16, MVT::f128, Expand);
 
  if (Subtarget.isISA3_0()) {
    setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f16, Legal);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Legal);
    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Legal);
    setTruncStoreAction(MVT::f64, MVT::f16, Legal);
    setTruncStoreAction(MVT::f32, MVT::f16, Legal);
  } else {
    // No extending loads from f16 or HW conversions back and forth.
    setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f16, Expand);
    setOperationAction(ISD::FP16_TO_FP, MVT::f128, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
    setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
    setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
    setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
    setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
    setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
    setTruncStoreAction(MVT::f64, MVT::f16, Expand);
    setTruncStoreAction(MVT::f32, MVT::f16, Expand);
  }
 
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
  // PowerPC has pre-inc load and store's.
  setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal);
  setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal);
  setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal);
  setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal);
  setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal);
  setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal);
  setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal);
  setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal);
  setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal);
  setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal);
  if (!Subtarget.hasSPE()) {
    setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal);
    setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal);
    setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal);
    setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal);
  }
 
  if (Subtarget.useCRBits()) {
    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 
    if (isPPC64 || Subtarget.hasFPCVT()) {
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Promote);
      AddPromotedToType(ISD::STRICT_SINT_TO_FP, MVT::i1, RegVT);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Promote);
      AddPromotedToType(ISD::STRICT_UINT_TO_FP, MVT::i1, RegVT);
 
      setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote);
      AddPromotedToType(ISD::SINT_TO_FP, MVT::i1, RegVT);
      setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);
      AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, RegVT);
 
      setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i1, Promote);
      AddPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::i1, RegVT);
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i1, Promote);
      AddPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::i1, RegVT);
 
      setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote);
      AddPromotedToType(ISD::FP_TO_SINT, MVT::i1, RegVT);
      setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);
      AddPromotedToType(ISD::FP_TO_UINT, MVT::i1, RegVT);
    } else {
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom);
    }
 
    // PowerPC does not support direct load/store of condition registers.
    setOperationAction(ISD::LOAD, MVT::i1, Custom);
    setOperationAction(ISD::STORE, MVT::i1, Custom);
 
    // FIXME: Remove this once the ANDI glue bug is fixed:
    if (ANDIGlueBug)
      setOperationAction(ISD::TRUNCATE, MVT::i1, Custom);
 
    for (MVT VT : MVT::integer_valuetypes()) {
      setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
      setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
      setTruncStoreAction(VT, MVT::i1, Expand);
    }
 
    addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass);
  }
 
  // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
  // PPC (the libcall is not available).
  setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom);
  setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::ppcf128, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::ppcf128, Custom);
 
  // We do not currently implement these libm ops for PowerPC.
  setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand);
  setOperationAction(ISD::FCEIL,  MVT::ppcf128, Expand);
  setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand);
  setOperationAction(ISD::FRINT,  MVT::ppcf128, Expand);
  setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand);
  setOperationAction(ISD::FREM, MVT::ppcf128, LibCall);
 
  // PowerPC has no SREM/UREM instructions unless we are on P9
  // On P9 we may use a hardware instruction to compute the remainder.
  // When the result of both the remainder and the division is required it is
  // more efficient to compute the remainder from the result of the division
  // rather than use the remainder instruction. The instructions are legalized
  // directly because the DivRemPairsPass performs the transformation at the IR
  // level.
  if (Subtarget.isISA3_0()) {
    setOperationAction(ISD::SREM, MVT::i32, Legal);
    setOperationAction(ISD::UREM, MVT::i32, Legal);
    setOperationAction(ISD::SREM, MVT::i64, Legal);
    setOperationAction(ISD::UREM, MVT::i64, Legal);
  } else {
    setOperationAction(ISD::SREM, MVT::i32, Expand);
    setOperationAction(ISD::UREM, MVT::i32, Expand);
    setOperationAction(ISD::SREM, MVT::i64, Expand);
    setOperationAction(ISD::UREM, MVT::i64, Expand);
  }
 
  // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM.
  setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
  setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
  setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
  setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
  setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
  setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
  setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
  setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
 
  // Handle constrained floating-point operations of scalar.
  // TODO: Handle SPE specific operation.
  setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
 
  setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
  setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
 
  if (!Subtarget.hasSPE()) {
    setOperationAction(ISD::STRICT_FMA, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FMA, MVT::f64, Legal);
  }
 
  if (Subtarget.hasVSX()) {
    setOperationAction(ISD::STRICT_FRINT, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FRINT, MVT::f64, Legal);
  }
 
  if (Subtarget.hasFSQRT()) {
    setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
  }
 
  if (Subtarget.hasFPRND()) {
    setOperationAction(ISD::STRICT_FFLOOR, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FCEIL,  MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FTRUNC, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FROUND, MVT::f32, Legal);
 
    setOperationAction(ISD::STRICT_FFLOOR, MVT::f64, Legal);
    setOperationAction(ISD::STRICT_FCEIL,  MVT::f64, Legal);
    setOperationAction(ISD::STRICT_FTRUNC, MVT::f64, Legal);
    setOperationAction(ISD::STRICT_FROUND, MVT::f64, Legal);
  }
 
  // We don't support sin/cos/sqrt/fmod/pow
  setOperationAction(ISD::FSIN , MVT::f64, Expand);
  setOperationAction(ISD::FCOS , MVT::f64, Expand);
  setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
  setOperationAction(ISD::FREM, MVT::f64, LibCall);
  setOperationAction(ISD::FPOW , MVT::f64, Expand);
  setOperationAction(ISD::FSIN , MVT::f32, Expand);
  setOperationAction(ISD::FCOS , MVT::f32, Expand);
  setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
  setOperationAction(ISD::FREM, MVT::f32, LibCall);
  setOperationAction(ISD::FPOW , MVT::f32, Expand);
 
  // MASS transformation for LLVM intrinsics with replicating fast-math flag
  // to be consistent to PPCGenScalarMASSEntries pass
  if (TM.getOptLevel() == CodeGenOptLevel::Aggressive) {
    setOperationAction(ISD::FSIN , MVT::f64, Custom);
    setOperationAction(ISD::FCOS , MVT::f64, Custom);
    setOperationAction(ISD::FPOW , MVT::f64, Custom);
    setOperationAction(ISD::FLOG, MVT::f64, Custom);
    setOperationAction(ISD::FLOG10, MVT::f64, Custom);
    setOperationAction(ISD::FEXP, MVT::f64, Custom);
    setOperationAction(ISD::FSIN , MVT::f32, Custom);
    setOperationAction(ISD::FCOS , MVT::f32, Custom);
    setOperationAction(ISD::FPOW , MVT::f32, Custom);
    setOperationAction(ISD::FLOG, MVT::f32, Custom);
    setOperationAction(ISD::FLOG10, MVT::f32, Custom);
    setOperationAction(ISD::FEXP, MVT::f32, Custom);
  }
 
  if (Subtarget.hasSPE()) {
    setOperationAction(ISD::FMA  , MVT::f64, Expand);
    setOperationAction(ISD::FMA  , MVT::f32, Expand);
  } else {
    setOperationAction(ISD::FMA  , MVT::f64, Legal);
    setOperationAction(ISD::FMA  , MVT::f32, Legal);
    setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
    setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
  }
 
  if (Subtarget.hasSPE())
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
 
  // If we're enabling GP optimizations, use hardware square root
  if (!Subtarget.hasFSQRT() && !(Subtarget.hasFRSQRTE() && Subtarget.hasFRE()))
    setOperationAction(ISD::FSQRT, MVT::f64, Expand);
 
  if (!Subtarget.hasFSQRT() &&
      !(Subtarget.hasFRSQRTES() && Subtarget.hasFRES()))
    setOperationAction(ISD::FSQRT, MVT::f32, Expand);
 
  if (Subtarget.hasFCPSGN()) {
    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal);
    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal);
  } else {
    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
  }
 
  if (Subtarget.hasFPRND()) {
    setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
    setOperationAction(ISD::FCEIL,  MVT::f64, Legal);
    setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
    setOperationAction(ISD::FROUND, MVT::f64, Legal);
 
    setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
    setOperationAction(ISD::FCEIL,  MVT::f32, Legal);
    setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
    setOperationAction(ISD::FROUND, MVT::f32, Legal);
  }
 
  // Prior to P10, PowerPC does not have BSWAP, but we can use vector BSWAP
  // instruction xxbrd to speed up scalar BSWAP64.
  if (Subtarget.isISA3_1()) {
    setOperationAction(ISD::BSWAP, MVT::i32, Legal);
    setOperationAction(ISD::BSWAP, MVT::i64, Legal);
  } else {
    setOperationAction(ISD::BSWAP, MVT::i32, Expand);
    setOperationAction(ISD::BSWAP, MVT::i64,
                       ((Subtarget.hasP8Vector()) && isPPC64) ? Custom
                                                              : Expand);
  }
 
  // CTPOP or CTTZ were introduced in P8/P9 respectively
  if (Subtarget.isISA3_0()) {
    setOperationAction(ISD::CTTZ , MVT::i32  , Legal);
    setOperationAction(ISD::CTTZ , MVT::i64  , Legal);
  } else {
    setOperationAction(ISD::CTTZ , MVT::i32  , Expand);
    setOperationAction(ISD::CTTZ , MVT::i64  , Expand);
  }
 
  if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
    setOperationAction(ISD::CTPOP, MVT::i32  , Legal);
    setOperationAction(ISD::CTPOP, MVT::i64  , Legal);
  } else {
    setOperationAction(ISD::CTPOP, MVT::i32  , Expand);
    setOperationAction(ISD::CTPOP, MVT::i64  , Expand);
  }
 
  // PowerPC does not have ROTR
  setOperationAction(ISD::ROTR, MVT::i32   , Expand);
  setOperationAction(ISD::ROTR, MVT::i64   , Expand);
 
  if (!Subtarget.useCRBits()) {
    // PowerPC does not have Select
    setOperationAction(ISD::SELECT, MVT::i32, Expand);
    setOperationAction(ISD::SELECT, MVT::i64, Expand);
    setOperationAction(ISD::SELECT, MVT::f32, Expand);
    setOperationAction(ISD::SELECT, MVT::f64, Expand);
  }
 
  // PowerPC wants to turn select_cc of FP into fsel when possible.
  setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
 
  // PowerPC wants to optimize integer setcc a bit
  if (!Subtarget.useCRBits())
    setOperationAction(ISD::SETCC, MVT::i32, Custom);
 
  if (Subtarget.hasFPU()) {
    setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Legal);
    setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Legal);
 
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Legal);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Legal);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Legal);
  }
 
  // PowerPC does not have BRCOND which requires SetCC
  if (!Subtarget.useCRBits())
    setOperationAction(ISD::BRCOND, MVT::Other, Expand);
 
  setOperationAction(ISD::BR_JT,  MVT::Other, Expand);
 
  if (Subtarget.hasSPE()) {
    // SPE has built-in conversions
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Legal);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Legal);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Legal);
    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal);
    setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal);
    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal);
 
    // SPE supports signaling compare of f32/f64.
    // But it doesn't comply IEEE-754 rules for comparing
    // special values like NaNs, Infs.
    setOperationAction(ISD::SETCC, MVT::f32, Custom);
    setOperationAction(ISD::SETCC, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
    setOperationAction(ISD::BR_CC, MVT::f32, Custom);
    setOperationAction(ISD::BR_CC, MVT::f64, Custom);
  } else {
    // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores.
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
 
    // PowerPC does not have [U|S]INT_TO_FP
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Expand);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Expand);
    setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand);
    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand);
  }
 
  if (Subtarget.hasDirectMove() && isPPC64) {
    setOperationAction(ISD::BITCAST, MVT::f32, Legal);
    setOperationAction(ISD::BITCAST, MVT::i32, Legal);
    setOperationAction(ISD::BITCAST, MVT::i64, Legal);
    setOperationAction(ISD::BITCAST, MVT::f64, Legal);
 
    setOperationAction(ISD::STRICT_LRINT, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_LRINT, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_LLRINT, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_LLRINT, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_LROUND, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_LROUND, MVT::f32, Custom);
    setOperationAction(ISD::STRICT_LLROUND, MVT::f64, Custom);
    setOperationAction(ISD::STRICT_LLROUND, MVT::f32, Custom);
  } else {
    setOperationAction(ISD::BITCAST, MVT::f32, Expand);
    setOperationAction(ISD::BITCAST, MVT::i32, Expand);
    setOperationAction(ISD::BITCAST, MVT::i64, Expand);
    setOperationAction(ISD::BITCAST, MVT::f64, Expand);
  }
 
  // We cannot sextinreg(i1).  Expand to shifts.
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
 
  // Custom handling for PowerPC ucmp instruction
  setOperationAction(ISD::UCMP, MVT::i32, Custom);
  setOperationAction(ISD::UCMP, MVT::i64, isPPC64 ? Custom : Expand);
  setOperationAction(ISD::ABDU, MVT::i32, Custom);
  setOperationAction(ISD::ABDU, MVT::i64, isPPC64 ? Custom : Expand);
 
  // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
  // SjLj exception handling but a light-weight setjmp/longjmp replacement to
  // support continuation, user-level threading, and etc.. As a result, no
  // other SjLj exception interfaces are implemented and please don't build
  // your own exception handling based on them.
  // LLVM/Clang supports zero-cost DWARF exception handling.
  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
 
  // We want to legalize GlobalAddress and ConstantPool nodes into the
  // appropriate instructions to materialize the address.
  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
  setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
  setOperationAction(ISD::BlockAddress,  MVT::i32, Custom);
  setOperationAction(ISD::ConstantPool,  MVT::i32, Custom);
  setOperationAction(ISD::JumpTable,     MVT::i32, Custom);
  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
  setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
  setOperationAction(ISD::BlockAddress,  MVT::i64, Custom);
  setOperationAction(ISD::ConstantPool,  MVT::i64, Custom);
  setOperationAction(ISD::JumpTable,     MVT::i64, Custom);
 
  // TRAP is legal.
  setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
  // TRAMPOLINE is custom lowered.
  setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
  setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
 
  // VASTART needs to be custom lowered to use the VarArgsFrameIndex
  setOperationAction(ISD::VASTART           , MVT::Other, Custom);
 
  if (Subtarget.is64BitELFABI()) {
    // VAARG always uses double-word chunks, so promote anything smaller.
    setOperationAction(ISD::VAARG, MVT::i1, Promote);
    AddPromotedToType(ISD::VAARG, MVT::i1, MVT::i64);
    setOperationAction(ISD::VAARG, MVT::i8, Promote);
    AddPromotedToType(ISD::VAARG, MVT::i8, MVT::i64);
    setOperationAction(ISD::VAARG, MVT::i16, Promote);
    AddPromotedToType(ISD::VAARG, MVT::i16, MVT::i64);
    setOperationAction(ISD::VAARG, MVT::i32, Promote);
    AddPromotedToType(ISD::VAARG, MVT::i32, MVT::i64);
    setOperationAction(ISD::VAARG, MVT::Other, Expand);
  } else if (Subtarget.is32BitELFABI()) {
    // VAARG is custom lowered with the 32-bit SVR4 ABI.
    setOperationAction(ISD::VAARG, MVT::Other, Custom);
    setOperationAction(ISD::VAARG, MVT::i64, Custom);
  } else
    setOperationAction(ISD::VAARG, MVT::Other, Expand);
 
  // VACOPY is custom lowered with the 32-bit SVR4 ABI.
  if (Subtarget.is32BitELFABI())
    setOperationAction(ISD::VACOPY            , MVT::Other, Custom);
  else
    setOperationAction(ISD::VACOPY            , MVT::Other, Expand);
 
  // Use the default implementation.
  setOperationAction(ISD::VAEND             , MVT::Other, Expand);
  setOperationAction(ISD::STACKSAVE         , MVT::Other, Expand);
  setOperationAction(ISD::STACKRESTORE      , MVT::Other, Custom);
  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32  , Custom);
  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64  , Custom);
  setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom);
  setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom);
  setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
  setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
 
  if (Subtarget.isISA3_0() && isPPC64) {
    setOperationAction(ISD::VP_STORE, MVT::v16i1, Custom);
    setOperationAction(ISD::VP_STORE, MVT::v8i1, Custom);
    setOperationAction(ISD::VP_STORE, MVT::v4i1, Custom);
    setOperationAction(ISD::VP_STORE, MVT::v2i1, Custom);
    setOperationAction(ISD::VP_LOAD, MVT::v16i1, Custom);
    setOperationAction(ISD::VP_LOAD, MVT::v8i1, Custom);
    setOperationAction(ISD::VP_LOAD, MVT::v4i1, Custom);
    setOperationAction(ISD::VP_LOAD, MVT::v2i1, Custom);
  }
 
  // We want to custom lower some of our intrinsics.
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f64, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::ppcf128, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom);
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2f64, Custom);
 
  // To handle counter-based loop conditions.
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom);
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
 
  setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom);
  setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom);
  setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom);
  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 
  // Comparisons that require checking two conditions.
  if (Subtarget.hasSPE()) {
    setCondCodeAction(ISD::SETO, MVT::f32, Expand);
    setCondCodeAction(ISD::SETO, MVT::f64, Expand);
    setCondCodeAction(ISD::SETUO, MVT::f32, Expand);
    setCondCodeAction(ISD::SETUO, MVT::f64, Expand);
  }
  setCondCodeAction(ISD::SETULT, MVT::f32, Expand);
  setCondCodeAction(ISD::SETULT, MVT::f64, Expand);
  setCondCodeAction(ISD::SETUGT, MVT::f32, Expand);
  setCondCodeAction(ISD::SETUGT, MVT::f64, Expand);
  setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand);
  setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand);
  setCondCodeAction(ISD::SETOGE, MVT::f32, Expand);
  setCondCodeAction(ISD::SETOGE, MVT::f64, Expand);
  setCondCodeAction(ISD::SETOLE, MVT::f32, Expand);
  setCondCodeAction(ISD::SETOLE, MVT::f64, Expand);
  setCondCodeAction(ISD::SETONE, MVT::f32, Expand);
  setCondCodeAction(ISD::SETONE, MVT::f64, Expand);
 
  setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
  setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
 
  if (Subtarget.has64BitSupport()) {
    // They also have instructions for converting between i64 and fp.
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Expand);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Expand);
    setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand);
    setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand);
    // This is just the low 32 bits of a (signed) fp->i64 conversion.
    // We cannot do this with Promote because i64 is not a legal type.
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
 
    if (Subtarget.hasLFIWAX() || isPPC64) {
      setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
    }
  } else {
    // PowerPC does not have FP_TO_UINT on 32-bit implementations.
    if (Subtarget.hasSPE()) {
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Legal);
      setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal);
    } else {
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Expand);
      setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand);
    }
  }
 
  // With the instructions enabled under FPCVT, we can do everything.
  if (Subtarget.hasFPCVT()) {
    if (Subtarget.has64BitSupport()) {
      setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
      setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
      setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
    }
 
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
    setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
  }
 
  if (Subtarget.use64BitRegs()) {
    // 64-bit PowerPC implementations can support i64 types directly
    addRegisterClass(MVT::i64, &PPC::G8RCRegClass);
    // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
    setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
    // 64-bit PowerPC wants to expand i128 shifts itself.
    setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
    setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
    setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
  } else {
    // 32-bit PowerPC wants to expand i64 shifts itself.
    setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
    setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
    setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
  }
 
  // PowerPC has better expansions for funnel shifts than the generic
  // TargetLowering::expandFunnelShift.
  if (Subtarget.has64BitSupport()) {
    setOperationAction(ISD::FSHL, MVT::i64, Custom);
    setOperationAction(ISD::FSHR, MVT::i64, Custom);
  }
  setOperationAction(ISD::FSHL, MVT::i32, Custom);
  setOperationAction(ISD::FSHR, MVT::i32, Custom);
 
  if (Subtarget.hasVSX()) {
    setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal);
    setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);
    setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);
    setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal);
    setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
    setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
    setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
    setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
    setOperationAction(ISD::FCANONICALIZE, MVT::f64, Legal);
    setOperationAction(ISD::FCANONICALIZE, MVT::f32, Legal);
  }
 
  if (Subtarget.hasAltivec()) {
    for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
      setOperationAction(ISD::AVGCEILS, VT, Legal);
      setOperationAction(ISD::AVGCEILU, VT, Legal);
      setOperationAction(ISD::SADDSAT, VT, Legal);
      setOperationAction(ISD::SSUBSAT, VT, Legal);
      setOperationAction(ISD::UADDSAT, VT, Legal);
      setOperationAction(ISD::USUBSAT, VT, Legal);
    }
    // First set operation action for all vector types to expand. Then we
    // will selectively turn on ones that can be effectively codegen'd.
    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
      // add/sub are legal for all supported vector VT's.
      setOperationAction(ISD::ADD, VT, Legal);
      setOperationAction(ISD::SUB, VT, Legal);
 
      // For v2i64, these are only valid with P8Vector. This is corrected after
      // the loop.
      if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) {
        setOperationAction(ISD::SMAX, VT, Legal);
        setOperationAction(ISD::SMIN, VT, Legal);
        setOperationAction(ISD::UMAX, VT, Legal);
        setOperationAction(ISD::UMIN, VT, Legal);
      }
      else {
        setOperationAction(ISD::SMAX, VT, Expand);
        setOperationAction(ISD::SMIN, VT, Expand);
        setOperationAction(ISD::UMAX, VT, Expand);
        setOperationAction(ISD::UMIN, VT, Expand);
      }
 
      if (Subtarget.hasVSX()) {
        setOperationAction(ISD::FMAXNUM_IEEE, VT, Legal);
        setOperationAction(ISD::FMINNUM_IEEE, VT, Legal);
        setOperationAction(ISD::FMAXNUM, VT, Legal);
        setOperationAction(ISD::FMINNUM, VT, Legal);
        setOperationAction(ISD::FCANONICALIZE, VT, Legal);
      }
 
      // Vector instructions introduced in P8
      if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
        setOperationAction(ISD::CTPOP, VT, Legal);
        setOperationAction(ISD::CTLZ, VT, Legal);
      }
      else {
        setOperationAction(ISD::CTPOP, VT, Expand);
        setOperationAction(ISD::CTLZ, VT, Expand);
      }
 
      // Vector instructions introduced in P9
      if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128))
        setOperationAction(ISD::CTTZ, VT, Legal);
      else
        setOperationAction(ISD::CTTZ, VT, Expand);
 
      // We promote all shuffles to v16i8.
      setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote);
      AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8);
 
      // We promote all non-typed operations to v4i32.
      setOperationAction(ISD::AND   , VT, Promote);
      AddPromotedToType (ISD::AND   , VT, MVT::v4i32);
      setOperationAction(ISD::OR    , VT, Promote);
      AddPromotedToType (ISD::OR    , VT, MVT::v4i32);
      setOperationAction(ISD::XOR   , VT, Promote);
      AddPromotedToType (ISD::XOR   , VT, MVT::v4i32);
      setOperationAction(ISD::LOAD  , VT, Promote);
      AddPromotedToType (ISD::LOAD  , VT, MVT::v4i32);
      setOperationAction(ISD::SELECT, VT, Promote);
      AddPromotedToType (ISD::SELECT, VT, MVT::v4i32);
      setOperationAction(ISD::VSELECT, VT, Legal);
      setOperationAction(ISD::SELECT_CC, VT, Promote);
      AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32);
      setOperationAction(ISD::STORE, VT, Promote);
      AddPromotedToType (ISD::STORE, VT, MVT::v4i32);
 
      // No other operations are legal.
      setOperationAction(ISD::MUL , VT, Expand);
      setOperationAction(ISD::SDIV, VT, Expand);
      setOperationAction(ISD::SREM, VT, Expand);
      setOperationAction(ISD::UDIV, VT, Expand);
      setOperationAction(ISD::UREM, VT, Expand);
      setOperationAction(ISD::FDIV, VT, Expand);
      setOperationAction(ISD::FREM, VT, Expand);
      setOperationAction(ISD::FNEG, VT, Expand);
      setOperationAction(ISD::FSQRT, VT, Expand);
      setOperationAction(ISD::FLOG, VT, Expand);
      setOperationAction(ISD::FLOG10, VT, Expand);
      setOperationAction(ISD::FLOG2, VT, Expand);
      setOperationAction(ISD::FEXP, VT, Expand);
      setOperationAction(ISD::FEXP2, VT, Expand);
      setOperationAction(ISD::FSIN, VT, Expand);
      setOperationAction(ISD::FCOS, VT, Expand);
      setOperationAction(ISD::FABS, VT, Expand);
      setOperationAction(ISD::FFLOOR, VT, Expand);
      setOperationAction(ISD::FCEIL,  VT, Expand);
      setOperationAction(ISD::FTRUNC, VT, Expand);
      setOperationAction(ISD::FRINT,  VT, Expand);
      setOperationAction(ISD::FLDEXP, VT, Expand);
      setOperationAction(ISD::FNEARBYINT, VT, Expand);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand);
      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
      setOperationAction(ISD::BUILD_VECTOR, VT, Expand);
      setOperationAction(ISD::MULHU, VT, Expand);
      setOperationAction(ISD::MULHS, VT, Expand);
      setOperationAction(ISD::UMUL_LOHI, VT, Expand);
      setOperationAction(ISD::SMUL_LOHI, VT, Expand);
      setOperationAction(ISD::UDIVREM, VT, Expand);
      setOperationAction(ISD::SDIVREM, VT, Expand);
      setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
      setOperationAction(ISD::FPOW, VT, Expand);
      setOperationAction(ISD::BSWAP, VT, Expand);
      setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
      setOperationAction(ISD::ROTL, VT, Expand);
      setOperationAction(ISD::ROTR, VT, Expand);
 
      for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
        setTruncStoreAction(VT, InnerVT, Expand);
        setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
      }
    }
    setOperationAction(ISD::SELECT_CC, MVT::v4i32, Expand);
    if (!Subtarget.hasP8Vector()) {
      setOperationAction(ISD::SMAX, MVT::v2i64, Expand);
      setOperationAction(ISD::SMIN, MVT::v2i64, Expand);
      setOperationAction(ISD::UMAX, MVT::v2i64, Expand);
      setOperationAction(ISD::UMIN, MVT::v2i64, Expand);
    }
 
    // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
    // with merges, splats, etc.
    setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom);
 
    // Vector truncates to sub-word integer that fit in an Altivec/VSX register
    // are cheap, so handle them before they get expanded to scalar.
    setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
    setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
    setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
    setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
    setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
 
    setOperationAction(ISD::AND   , MVT::v4i32, Legal);
    setOperationAction(ISD::OR    , MVT::v4i32, Legal);
    setOperationAction(ISD::XOR   , MVT::v4i32, Legal);
    setOperationAction(ISD::LOAD  , MVT::v4i32, Legal);
    setOperationAction(ISD::SELECT, MVT::v4i32,
                       Subtarget.useCRBits() ? Legal : Expand);
    setOperationAction(ISD::STORE , MVT::v4i32, Legal);
    setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Legal);
    setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Legal);
    setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Legal);
    setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Legal);
    setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
    setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
    setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
    setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
    setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal);
    setOperationAction(ISD::FCEIL, MVT::v4f32, Legal);
    setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal);
    setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal);
 
    // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8.
    setOperationAction(ISD::ROTL, MVT::v1i128, Custom);
    // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w).
    if (Subtarget.hasAltivec())
      for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8})
        setOperationAction(ISD::ROTL, VT, Legal);
    // With hasP8Altivec set, we can lower ISD::ROTL to vrld.
    if (Subtarget.hasP8Altivec())
      setOperationAction(ISD::ROTL, MVT::v2i64, Legal);
 
    addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass);
    addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass);
    addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass);
    addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass);
 
    setOperationAction(ISD::MUL, MVT::v4f32, Legal);
    setOperationAction(ISD::FMA, MVT::v4f32, Legal);
 
    if (Subtarget.hasVSX()) {
      setOperationAction(ISD::FDIV, MVT::v4f32, Legal);
      setOperationAction(ISD::FSQRT, MVT::v4f32, Legal);
      setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom);
    }
 
    if (Subtarget.hasP8Altivec())
      setOperationAction(ISD::MUL, MVT::v4i32, Legal);
    else
      setOperationAction(ISD::MUL, MVT::v4i32, Custom);
 
    if (Subtarget.isISA3_1()) {
      setOperationAction(ISD::MUL, MVT::v2i64, Legal);
      setOperationAction(ISD::MULHS, MVT::v2i64, Legal);
      setOperationAction(ISD::MULHU, MVT::v2i64, Legal);
      setOperationAction(ISD::MULHS, MVT::v4i32, Legal);
      setOperationAction(ISD::MULHU, MVT::v4i32, Legal);
      setOperationAction(ISD::UDIV, MVT::v2i64, Legal);
      setOperationAction(ISD::SDIV, MVT::v2i64, Legal);
      setOperationAction(ISD::UDIV, MVT::v4i32, Legal);
      setOperationAction(ISD::SDIV, MVT::v4i32, Legal);
      setOperationAction(ISD::UREM, MVT::v2i64, Legal);
      setOperationAction(ISD::SREM, MVT::v2i64, Legal);
      setOperationAction(ISD::UREM, MVT::v4i32, Legal);
      setOperationAction(ISD::SREM, MVT::v4i32, Legal);
      setOperationAction(ISD::UREM, MVT::v1i128, Legal);
      setOperationAction(ISD::SREM, MVT::v1i128, Legal);
      setOperationAction(ISD::UDIV, MVT::v1i128, Legal);
      setOperationAction(ISD::SDIV, MVT::v1i128, Legal);
      setOperationAction(ISD::ROTL, MVT::v1i128, Legal);
    }
 
    setOperationAction(ISD::MUL, MVT::v8i16, Legal);
    setOperationAction(ISD::MUL, MVT::v16i8, Custom);
 
    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom);
    setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom);
    // LE is P8+/64-bit so direct moves are supported and these operations
    // are legal. The custom transformation requires 64-bit since we need a
    // pair of stores that will cover a 128-bit load for P10.
    if (!DisableP10StoreForward && isPPC64 && !Subtarget.isLittleEndian()) {
      setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Custom);
      setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom);
      setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom);
    }
 
    setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom);
    setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom);
    setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom);
    setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
 
    // Altivec does not contain unordered floating-point compare instructions
    setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand);
    setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand);
    setCondCodeAction(ISD::SETO,   MVT::v4f32, Expand);
    setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand);
 
    if (Subtarget.hasVSX()) {
      setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal);
      if (Subtarget.hasP8Vector()) {
        setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal);
        setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal);
      }
      if (Subtarget.hasDirectMove() && isPPC64) {
        setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal);
        setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal);
        setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal);
        setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal);
        setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal);
        setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal);
        setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal);
        setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal);
      }
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal);
 
      // The nearbyint variants are not allowed to raise the inexact exception
      // so we can only code-gen them with fpexcept.ignore.
      setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f64, Custom);
      setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f32, Custom);
      setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v2f64, Custom);
      setOperationAction(ISD::STRICT_FNEARBYINT, MVT::v4f32, Custom);
 
      setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal);
      setOperationAction(ISD::FCEIL, MVT::v2f64, Legal);
      setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal);
      setOperationAction(ISD::FRINT, MVT::v2f64, Legal);
      setOperationAction(ISD::FROUND, MVT::v2f64, Legal);
      setOperationAction(ISD::FROUND, MVT::f64, Legal);
      setOperationAction(ISD::FRINT, MVT::f64, Legal);
 
      setOperationAction(ISD::FRINT, MVT::v4f32, Legal);
      setOperationAction(ISD::FROUND, MVT::v4f32, Legal);
      setOperationAction(ISD::FROUND, MVT::f32, Legal);
      setOperationAction(ISD::FRINT, MVT::f32, Legal);
 
      setOperationAction(ISD::MUL, MVT::v2f64, Legal);
      setOperationAction(ISD::FMA, MVT::v2f64, Legal);
 
      setOperationAction(ISD::FDIV, MVT::v2f64, Legal);
      setOperationAction(ISD::FSQRT, MVT::v2f64, Legal);
 
      // Share the Altivec comparison restrictions.
      setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand);
      setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand);
      setCondCodeAction(ISD::SETO,   MVT::v2f64, Expand);
      setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand);
 
      setOperationAction(ISD::LOAD, MVT::v2f64, Legal);
      setOperationAction(ISD::STORE, MVT::v2f64, Legal);
 
      setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
 
      if (Subtarget.hasP8Vector())
        addRegisterClass(MVT::f32, &PPC::VSSRCRegClass);
 
      addRegisterClass(MVT::f64, &PPC::VSFRCRegClass);
 
      addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass);
      addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass);
      addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass);
 
      if (Subtarget.hasP8Altivec()) {
        setOperationAction(ISD::SHL, MVT::v2i64, Legal);
        setOperationAction(ISD::SRA, MVT::v2i64, Legal);
        setOperationAction(ISD::SRL, MVT::v2i64, Legal);
 
        // 128 bit shifts can be accomplished via 3 instructions for SHL and
        // SRL, but not for SRA because of the instructions available:
        // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth
        // doing
        setOperationAction(ISD::SHL, MVT::v1i128, Expand);
        setOperationAction(ISD::SRL, MVT::v1i128, Expand);
        setOperationAction(ISD::SRA, MVT::v1i128, Expand);
 
        setOperationAction(ISD::SETCC, MVT::v2i64, Legal);
      }
      else {
        setOperationAction(ISD::SHL, MVT::v2i64, Expand);
        setOperationAction(ISD::SRA, MVT::v2i64, Expand);
        setOperationAction(ISD::SRL, MVT::v2i64, Expand);
 
        setOperationAction(ISD::SETCC, MVT::v2i64, Custom);
 
        // VSX v2i64 only supports non-arithmetic operations.
        setOperationAction(ISD::ADD, MVT::v2i64, Expand);
        setOperationAction(ISD::SUB, MVT::v2i64, Expand);
      }
 
      if (Subtarget.isISA3_1())
        setOperationAction(ISD::SETCC, MVT::v1i128, Legal);
      else
        setOperationAction(ISD::SETCC, MVT::v1i128, Expand);
 
      setOperationAction(ISD::LOAD, MVT::v2i64, Promote);
      AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64);
      setOperationAction(ISD::STORE, MVT::v2i64, Promote);
      AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64);
 
      setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
 
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i64, Legal);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i64, Legal);
      setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i64, Legal);
      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i64, Legal);
      setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal);
      setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal);
      setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
      setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
 
      // Custom handling for partial vectors of integers converted to
      // floating point. We already have optimal handling for v2i32 through
      // the DAG combine, so those aren't necessary.
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i8, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i8, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i16, Custom);
      setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i16, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i8, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i8, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i16, Custom);
      setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i16, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
 
      setOperationAction(ISD::FNEG, MVT::v4f32, Legal);
      setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
      setOperationAction(ISD::FABS, MVT::v4f32, Legal);
      setOperationAction(ISD::FABS, MVT::v2f64, Legal);
      setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal);
      setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal);
 
      setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom);
      setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom);
 
      // Handle constrained floating-point operations of vector.
      // The predictor is `hasVSX` because altivec instruction has
      // no exception but VSX vector instruction has.
      setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FMA, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FMAXNUM, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FMINNUM, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FRINT, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FFLOOR, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FCEIL,  MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FTRUNC, MVT::v4f32, Legal);
      setOperationAction(ISD::STRICT_FROUND, MVT::v4f32, Legal);
 
      setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FMA, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FMAXNUM, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FMINNUM, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FRINT, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FFLOOR, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FCEIL,  MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FTRUNC, MVT::v2f64, Legal);
      setOperationAction(ISD::STRICT_FROUND, MVT::v2f64, Legal);
 
      addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass);
      addRegisterClass(MVT::f128, &PPC::VRRCRegClass);
 
      for (MVT FPT : MVT::fp_valuetypes())
        setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand);
 
      // Expand the SELECT to SELECT_CC
      setOperationAction(ISD::SELECT, MVT::f128, Expand);
 
      setTruncStoreAction(MVT::f128, MVT::f64, Expand);
      setTruncStoreAction(MVT::f128, MVT::f32, Expand);
 
      // No implementation for these ops for PowerPC.
      setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
      setOperationAction(ISD::FSIN, MVT::f128, Expand);
      setOperationAction(ISD::FCOS, MVT::f128, Expand);
      setOperationAction(ISD::FPOW, MVT::f128, Expand);
      setOperationAction(ISD::FPOWI, MVT::f128, Expand);
      setOperationAction(ISD::FREM, MVT::f128, LibCall);
    }
 
    if (Subtarget.hasP8Altivec()) {
      addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass);
      addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass);
    }
 
    if (Subtarget.hasP9Vector()) {
      setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
      setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
 
      // Test data class instructions store results in CR bits.
      if (Subtarget.useCRBits()) {
        setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
        setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
        setOperationAction(ISD::IS_FPCLASS, MVT::f128, Custom);
        setOperationAction(ISD::IS_FPCLASS, MVT::ppcf128, Custom);
      }
 
      // 128 bit shifts can be accomplished via 3 instructions for SHL and
      // SRL, but not for SRA because of the instructions available:
      // VS{RL} and VS{RL}O.
      setOperationAction(ISD::SHL, MVT::v1i128, Legal);
      setOperationAction(ISD::SRL, MVT::v1i128, Legal);
      setOperationAction(ISD::SRA, MVT::v1i128, Expand);
 
      setOperationAction(ISD::FADD, MVT::f128, Legal);
      setOperationAction(ISD::FSUB, MVT::f128, Legal);
      setOperationAction(ISD::FDIV, MVT::f128, Legal);
      setOperationAction(ISD::FMUL, MVT::f128, Legal);
      setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal);
 
      setOperationAction(ISD::FMA, MVT::f128, Legal);
      setCondCodeAction(ISD::SETULT, MVT::f128, Expand);
      setCondCodeAction(ISD::SETUGT, MVT::f128, Expand);
      setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand);
      setCondCodeAction(ISD::SETOGE, MVT::f128, Expand);
      setCondCodeAction(ISD::SETOLE, MVT::f128, Expand);
      setCondCodeAction(ISD::SETONE, MVT::f128, Expand);
 
      setOperationAction(ISD::FTRUNC, MVT::f128, Legal);
      setOperationAction(ISD::FRINT, MVT::f128, Legal);
      setOperationAction(ISD::FFLOOR, MVT::f128, Legal);
      setOperationAction(ISD::FCEIL, MVT::f128, Legal);
      setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal);
      setOperationAction(ISD::FROUND, MVT::f128, Legal);
 
      setOperationAction(ISD::FP_ROUND, MVT::f64, Legal);
      setOperationAction(ISD::FP_ROUND, MVT::f32, Legal);
      setOperationAction(ISD::BITCAST, MVT::i128, Custom);
 
      // Handle constrained floating-point operations of fp128
      setOperationAction(ISD::STRICT_FADD, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FSUB, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FMUL, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FDIV, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FMA, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FSQRT, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
      setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
      setOperationAction(ISD::STRICT_FRINT, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FFLOOR, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FCEIL, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FTRUNC, MVT::f128, Legal);
      setOperationAction(ISD::STRICT_FROUND, MVT::f128, Legal);
      setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
      setOperationAction(ISD::BSWAP, MVT::v8i16, Legal);
      setOperationAction(ISD::BSWAP, MVT::v4i32, Legal);
      setOperationAction(ISD::BSWAP, MVT::v2i64, Legal);
      setOperationAction(ISD::BSWAP, MVT::v1i128, Legal);
    } else if (Subtarget.hasVSX()) {
      setOperationAction(ISD::LOAD, MVT::f128, Promote);
      setOperationAction(ISD::STORE, MVT::f128, Promote);
 
      AddPromotedToType(ISD::LOAD, MVT::f128, MVT::v4i32);
      AddPromotedToType(ISD::STORE, MVT::f128, MVT::v4i32);
 
      // Set FADD/FSUB as libcall to avoid the legalizer to expand the
      // fp_to_uint and int_to_fp.
      setOperationAction(ISD::FADD, MVT::f128, LibCall);
      setOperationAction(ISD::FSUB, MVT::f128, LibCall);
 
      setOperationAction(ISD::FMUL, MVT::f128, Expand);
      setOperationAction(ISD::FDIV, MVT::f128, Expand);
      setOperationAction(ISD::FNEG, MVT::f128, Expand);
      setOperationAction(ISD::FABS, MVT::f128, Expand);
      setOperationAction(ISD::FSQRT, MVT::f128, Expand);
      setOperationAction(ISD::FMA, MVT::f128, Expand);
      setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
 
      // Expand the fp_extend if the target type is fp128.
      setOperationAction(ISD::FP_EXTEND, MVT::f128, Expand);
      setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Expand);
 
      // Expand the fp_round if the source type is fp128.
      for (MVT VT : {MVT::f32, MVT::f64}) {
        setOperationAction(ISD::FP_ROUND, VT, Custom);
        setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
      }
 
      setOperationAction(ISD::SETCC, MVT::f128, Custom);
      setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
      setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
      setOperationAction(ISD::BR_CC, MVT::f128, Expand);
 
      // Lower following f128 select_cc pattern:
      // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE
      setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
 
      // We need to handle f128 SELECT_CC with integer result type.
      setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
      setOperationAction(ISD::SELECT_CC, MVT::i64, isPPC64 ? Custom : Expand);
    }
 
    if (Subtarget.hasP9Altivec()) {
      if (Subtarget.isISA3_1()) {
        setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Legal);
        setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Legal);
        setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Legal);
        setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Legal);
      } else {
        setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
        setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
      }
      setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8,  Legal);
      setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal);
      setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal);
      setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8,  Legal);
      setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal);
      setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal);
      setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal);
 
      setOperationAction(ISD::ABDU, MVT::v16i8, Legal);
      setOperationAction(ISD::ABDU, MVT::v8i16, Legal);
      setOperationAction(ISD::ABDU, MVT::v4i32, Legal);
      setOperationAction(ISD::ABDS, MVT::v4i32, Legal);
    }
 
    if (Subtarget.hasP10Vector()) {
      setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
    }
 
    setOperationAction(ISD::PARTIAL_REDUCE_UMLA, MVT::v16i32, Custom);
    setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_UMLA, MVT::v4i32, MVT::v8i16,
                              Legal);
    setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SMLA, MVT::v4i32, MVT::v8i16,
                              Legal);
    setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_UMLA, MVT::v4i32, MVT::v16i8,
                              Legal);
    setPartialReduceMLAAction(ISD::PARTIAL_REDUCE_SUMLA, MVT::v4i32, MVT::v16i8,
                              Legal);
  }
 
  if (Subtarget.pairedVectorMemops()) {
    addRegisterClass(MVT::v256i1, &PPC::VSRpRCRegClass);
    setOperationAction(ISD::LOAD, MVT::v256i1, Custom);
    setOperationAction(ISD::STORE, MVT::v256i1, Custom);
  }
  if (Subtarget.hasMMA()) {
    if (Subtarget.isISAFuture()) {
      addRegisterClass(MVT::v512i1, &PPC::WACCRCRegClass);
      addRegisterClass(MVT::v1024i1, &PPC::DMRRCRegClass);
      addRegisterClass(MVT::v2048i1, &PPC::DMRpRCRegClass);
      setOperationAction(ISD::LOAD, MVT::v1024i1, Custom);
      setOperationAction(ISD::STORE, MVT::v1024i1, Custom);
      setOperationAction(ISD::LOAD, MVT::v2048i1, Custom);
      setOperationAction(ISD::STORE, MVT::v2048i1, Custom);
    } else {
      addRegisterClass(MVT::v512i1, &PPC::UACCRCRegClass);
    }
    setOperationAction(ISD::LOAD, MVT::v512i1, Custom);
    setOperationAction(ISD::STORE, MVT::v512i1, Custom);
    setOperationAction(ISD::BUILD_VECTOR, MVT::v512i1, Custom);
  }
 
  if (Subtarget.has64BitSupport())
    setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
 
  if (Subtarget.isISA3_1())
    setOperationAction(ISD::SRA, MVT::v1i128, Legal);
 
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom);
 
  if (!isPPC64) {
    setOperationAction(ISD::ATOMIC_LOAD,  MVT::i64, Expand);
    setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand);
  }
 
  if (shouldInlineQuadwordAtomics()) {
    setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
    setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
    setOperationAction(ISD::INTRINSIC_VOID, MVT::i128, Custom);
  }
 
  setBooleanContents(ZeroOrOneBooleanContent);
 
  if (Subtarget.hasAltivec()) {
    // Altivec instructions set fields to all zeros or all ones.
    setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
  }
 
  if (shouldInlineQuadwordAtomics())
    setMaxAtomicSizeInBitsSupported(128);
  else if (isPPC64)
    setMaxAtomicSizeInBitsSupported(64);
  else
    setMaxAtomicSizeInBitsSupported(32);
 
  setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1);
 
  // We have target-specific dag combine patterns for the following nodes:
  setTargetDAGCombine({ISD::AND, ISD::ADD, ISD::XOR, ISD::SHL, ISD::SRA,
                       ISD::SRL, ISD::MUL, ISD::FMA, ISD::SINT_TO_FP,
                       ISD::BUILD_VECTOR});
  if (Subtarget.hasFPCVT())
    setTargetDAGCombine(ISD::UINT_TO_FP);
  setTargetDAGCombine({ISD::LOAD, ISD::STORE, ISD::BR_CC});
  if (Subtarget.useCRBits())
    setTargetDAGCombine(ISD::BRCOND);
  setTargetDAGCombine({ISD::BSWAP, ISD::INTRINSIC_WO_CHAIN,
                       ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID});
 
  setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND, ISD::ANY_EXTEND});
 
  setTargetDAGCombine({ISD::TRUNCATE, ISD::VECTOR_SHUFFLE});
 
  if (Subtarget.useCRBits()) {
    setTargetDAGCombine({ISD::SETCC, ISD::SELECT_CC});
  }
 
  if (Subtarget.hasP8Vector())
    setTargetDAGCombine(ISD::BITCAST);
 
  // With 32 condition bits, we don't need to sink (and duplicate) compares
  // aggressively in CodeGenPrep.
  if (Subtarget.useCRBits()) {
    setJumpIsExpensive();
  }
 
  // TODO: The default entry number is set to 64. This stops most jump table
  // generation on PPC. But it is good for current PPC HWs because the indirect
  // branch instruction mtctr to the jump table may lead to bad branch predict.
  // Re-evaluate this value on future HWs that can do better with mtctr.
  setMinimumJumpTableEntries(PPCMinimumJumpTableEntries);
 
  // The default minimum of largest number in a BitTest cluster is 3.
  setMinimumBitTestCmps(PPCMinimumBitTestCmps);
 
  setMinFunctionAlignment(Align(4));
  setMinCmpXchgSizeInBits(Subtarget.hasPartwordAtomics() ? 8 : 32);
 
  auto CPUDirective = Subtarget.getCPUDirective();
  switch (CPUDirective) {
  default: break;
  case PPC::DIR_970:
  case PPC::DIR_A2:
  case PPC::DIR_E500:
  case PPC::DIR_E500mc:
  case PPC::DIR_E5500:
  case PPC::DIR_PWR4:
  case PPC::DIR_PWR5:
  case PPC::DIR_PWR5X:
  case PPC::DIR_PWR6:
  case PPC::DIR_PWR6X:
  case PPC::DIR_PWR7:
  case PPC::DIR_PWR8:
  case PPC::DIR_PWR9:
  case PPC::DIR_PWR10:
  case PPC::DIR_PWR11:
  case PPC::DIR_PWR_FUTURE:
    setPrefLoopAlignment(Align(16));
    setPrefFunctionAlignment(Align(16));
    break;
  }
 
  if (Subtarget.enableMachineScheduler())
    setSchedulingPreference(Sched::Source);
  else
    setSchedulingPreference(Sched::Hybrid);
 
  computeRegisterProperties(STI.getRegisterInfo());
 
  // The Freescale cores do better with aggressive inlining of memcpy and
  // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
  if (CPUDirective == PPC::DIR_E500mc || CPUDirective == PPC::DIR_E5500) {
    MaxStoresPerMemset = 32;
    MaxStoresPerMemsetOptSize = 16;
    MaxStoresPerMemcpy = 32;
    MaxStoresPerMemcpyOptSize = 8;
    MaxStoresPerMemmove = 32;
    MaxStoresPerMemmoveOptSize = 8;
  } else if (CPUDirective == PPC::DIR_A2) {
    // The A2 also benefits from (very) aggressive inlining of memcpy and
    // friends. The overhead of a the function call, even when warm, can be
    // over one hundred cycles.
    MaxStoresPerMemset = 128;
    MaxStoresPerMemcpy = 128;
    MaxStoresPerMemmove = 128;
    MaxLoadsPerMemcmp = 128;
  } else {
    MaxLoadsPerMemcmp = 8;
    MaxLoadsPerMemcmpOptSize = 4;
  }
 
  // Enable generation of STXVP instructions by default for mcpu=future.
  if (CPUDirective == PPC::DIR_PWR_FUTURE &&
      DisableAutoPairedVecSt.getNumOccurrences() == 0)
    DisableAutoPairedVecSt = false;
 
  IsStrictFPEnabled = true;
 
  // Let the subtarget (CPU) decide if a predictable select is more expensive
  // than the corresponding branch. This information is used in CGP to decide
  // when to convert selects into branches.
  PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive();
 
  GatherAllAliasesMaxDepth = PPCGatherAllAliasesMaxDepth;
}
 
// *********************************** NOTE ************************************
// For selecting load and store instructions, the addressing modes are defined
// as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD
// patterns to match the load the store instructions.
//
// The TD definitions for the addressing modes correspond to their respective
// Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely
// on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the
// address mode flags of a particular node. Afterwards, the computed address
// flags are passed into getAddrModeForFlags() in order to retrieve the optimal
// addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement
// accordingly, based on the preferred addressing mode.
//
// Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode.
// MemOpFlags contains all the possible flags that can be used to compute the
// optimal addressing mode for load and store instructions.
// AddrMode contains all the possible load and store addressing modes available
// on Power (such as DForm, DSForm, DQForm, XForm, etc.)
//
// When adding new load and store instructions, it is possible that new address
// flags may need to be added into MemOpFlags, and a new addressing mode will
// need to be added to AddrMode. An entry of the new addressing mode (consisting
// of the minimal and main distinguishing address flags for the new load/store
// instructions) will need to be added into initializeAddrModeMap() below.
// Finally, when adding new addressing modes, the getAddrModeForFlags() will
// need to be updated to account for selecting the optimal addressing mode.
// *****************************************************************************
/// Initialize the map that relates the different addressing modes of the load
/// and store instructions to a set of flags. This ensures the load/store
/// instruction is correctly matched during instruction selection.
void PPCTargetLowering::initializeAddrModeMap() {
  AddrModesMap[PPC::AM_DForm] = {
      // LWZ, STW
      PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt,
      PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt,
      PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
      PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
      // LBZ, LHZ, STB, STH
      PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
      PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
      PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
      PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
      // LHA
      PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
      PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
      PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
      PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
      // LFS, LFD, STFS, STFD
      PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
      PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
      PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
      PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
  };
  AddrModesMap[PPC::AM_DSForm] = {
      // LWA
      PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt,
      PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
      PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
      // LD, STD
      PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt,
      PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt,
      PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt,
      // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64
      PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
      PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
      PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
  };
  AddrModesMap[PPC::AM_DQForm] = {
      // LXV, STXV
      PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
      PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
      PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
  };
  AddrModesMap[PPC::AM_PrefixDForm] = {PPC::MOF_RPlusSImm34 |
                                       PPC::MOF_SubtargetP10};
  // TODO: Add mapping for quadword load/store.
}
 
/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
/// the desired ByVal argument alignment.
static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
  if (MaxAlign == MaxMaxAlign)
    return;
  if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
    if (MaxMaxAlign >= 32 &&
        VTy->getPrimitiveSizeInBits().getFixedValue() >= 256)
      MaxAlign = Align(32);
    else if (VTy->getPrimitiveSizeInBits().getFixedValue() >= 128 &&
             MaxAlign < 16)
      MaxAlign = Align(16);
  } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
    Align EltAlign;
    getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign);
    if (EltAlign > MaxAlign)
      MaxAlign = EltAlign;
  } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
    for (auto *EltTy : STy->elements()) {
      Align EltAlign;
      getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign);
      if (EltAlign > MaxAlign)
        MaxAlign = EltAlign;
      if (MaxAlign == MaxMaxAlign)
        break;
    }
  }
}
 
/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
/// function arguments in the caller parameter area.
Align PPCTargetLowering::getByValTypeAlignment(Type *Ty,
                                               const DataLayout &DL) const {
  // 16byte and wider vectors are passed on 16byte boundary.
  // The rest is 8 on PPC64 and 4 on PPC32 boundary.
  Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
  if (Subtarget.hasAltivec())
    getMaxByValAlign(Ty, Alignment, Align(16));
  return Alignment;
}
 
bool PPCTargetLowering::useSoftFloat() const {
  return Subtarget.useSoftFloat();
}
 
bool PPCTargetLowering::hasSPE() const {
  return Subtarget.hasSPE();
}
 
bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
  return VT.isScalarInteger();
}
 
bool PPCTargetLowering::shallExtractConstSplatVectorElementToStore(
    Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const {
  if (!Subtarget.isPPC64() || !Subtarget.hasVSX())
    return false;
 
  if (auto *VTy = dyn_cast<VectorType>(VectorTy)) {
    if (VTy->getScalarType()->isIntegerTy()) {
      // ElemSizeInBits 8/16 can fit in immediate field, not needed here.
      if (ElemSizeInBits == 32) {
        Index = Subtarget.isLittleEndian() ? 2 : 1;
        return true;
      }
      if (ElemSizeInBits == 64) {
        Index = Subtarget.isLittleEndian() ? 1 : 0;
        return true;
      }
    }
  }
  return false;
}
 
EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
                                          EVT VT) const {
  if (!VT.isVector())
    return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
 
  return VT.changeVectorElementTypeToInteger();
}
 
bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const {
  assert(VT.isFloatingPoint() && "Non-floating-point FMA?");
  return true;
}
 
//===----------------------------------------------------------------------===//
// Node matching predicates, for use by the tblgen matching code.
//===----------------------------------------------------------------------===//
 
/// isFloatingPointZero - Return true if this is 0.0 or -0.0.
static bool isFloatingPointZero(SDValue Op) {
  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
    return CFP->getValueAPF().isZero();
  else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
    // Maybe this has already been legalized into the constant pool?
    if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op.getOperand(1)))
      if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
        return CFP->getValueAPF().isZero();
  }
  return false;
}
 
/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode.  Return
/// true if Op is undef or if it matches the specified value.
static bool isConstantOrUndef(int Op, int Val) {
  return Op < 0 || Op == Val;
}
 
/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
/// VPKUHUM instruction.
/// The ShuffleKind distinguishes between big-endian operations with
/// two different inputs (0), either-endian operations with two identical
/// inputs (1), and little-endian operations with two different inputs (2).
/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
                               SelectionDAG &DAG) {
  bool IsLE = DAG.getDataLayout().isLittleEndian();
  if (ShuffleKind == 0) {
    if (IsLE)
      return false;
    for (unsigned i = 0; i != 16; ++i)
      if (!isConstantOrUndef(N->getMaskElt(i), i*2+1))
        return false;
  } else if (ShuffleKind == 2) {
    if (!IsLE)
      return false;
    for (unsigned i = 0; i != 16; ++i)
      if (!isConstantOrUndef(N->getMaskElt(i), i*2))
        return false;
  } else if (ShuffleKind == 1) {
    unsigned j = IsLE ? 0 : 1;
    for (unsigned i = 0; i != 8; ++i)
      if (!isConstantOrUndef(N->getMaskElt(i),    i*2+j) ||
          !isConstantOrUndef(N->getMaskElt(i+8),  i*2+j))
        return false;
  }
  return true;
}
 
/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
/// VPKUWUM instruction.
/// The ShuffleKind distinguishes between big-endian operations with
/// two different inputs (0), either-endian operations with two identical
/// inputs (1), and little-endian operations with two different inputs (2).
/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
                               SelectionDAG &DAG) {
  bool IsLE = DAG.getDataLayout().isLittleEndian();
  if (ShuffleKind == 0) {
    if (IsLE)
      return false;
    for (unsigned i = 0; i != 16; i += 2)
      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2+2) ||
          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+3))
        return false;
  } else if (ShuffleKind == 2) {
    if (!IsLE)
      return false;
    for (unsigned i = 0; i != 16; i += 2)
      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2) ||
          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+1))
        return false;
  } else if (ShuffleKind == 1) {
    unsigned j = IsLE ? 0 : 2;
    for (unsigned i = 0; i != 8; i += 2)
      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2+j)   ||
          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+j+1) ||
          !isConstantOrUndef(N->getMaskElt(i+8),  i*2+j)   ||
          !isConstantOrUndef(N->getMaskElt(i+9),  i*2+j+1))
        return false;
  }
  return true;
}
 
/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the
/// current subtarget.
///
/// The ShuffleKind distinguishes between big-endian operations with
/// two different inputs (0), either-endian operations with two identical
/// inputs (1), and little-endian operations with two different inputs (2).
/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
                               SelectionDAG &DAG) {
  const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
  if (!Subtarget.hasP8Vector())
    return false;
 
  bool IsLE = DAG.getDataLayout().isLittleEndian();
  if (ShuffleKind == 0) {
    if (IsLE)
      return false;
    for (unsigned i = 0; i != 16; i += 4)
      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2+4) ||
          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+5) ||
          !isConstantOrUndef(N->getMaskElt(i+2),  i*2+6) ||
          !isConstantOrUndef(N->getMaskElt(i+3),  i*2+7))
        return false;
  } else if (ShuffleKind == 2) {
    if (!IsLE)
      return false;
    for (unsigned i = 0; i != 16; i += 4)
      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2) ||
          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+1) ||
          !isConstantOrUndef(N->getMaskElt(i+2),  i*2+2) ||
          !isConstantOrUndef(N->getMaskElt(i+3),  i*2+3))
        return false;
  } else if (ShuffleKind == 1) {
    unsigned j = IsLE ? 0 : 4;
    for (unsigned i = 0; i != 8; i += 4)
      if (!isConstantOrUndef(N->getMaskElt(i  ),  i*2+j)   ||
          !isConstantOrUndef(N->getMaskElt(i+1),  i*2+j+1) ||
          !isConstantOrUndef(N->getMaskElt(i+2),  i*2+j+2) ||
          !isConstantOrUndef(N->getMaskElt(i+3),  i*2+j+3) ||
          !isConstantOrUndef(N->getMaskElt(i+8),  i*2+j)   ||
          !isConstantOrUndef(N->getMaskElt(i+9),  i*2+j+1) ||
          !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) ||
          !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3))
        return false;
  }
  return true;
}
 
/// isVMerge - Common function, used to match vmrg* shuffles.
///
static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
                     unsigned LHSStart, unsigned RHSStart) {
  if (N->getValueType(0) != MVT::v16i8)
    return false;
  assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&
         "Unsupported merge size!");
 
  for (unsigned i = 0; i != 8/UnitSize; ++i)     // Step over units
    for (unsigned j = 0; j != UnitSize; ++j) {   // Step over bytes within unit
      if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j),
                             LHSStart+j+i*UnitSize) ||
          !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j),
                             RHSStart+j+i*UnitSize))
        return false;
    }
  return true;
}
 
/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes).
/// The ShuffleKind distinguishes between big-endian merges with two
/// different inputs (0), either-endian merges with two identical inputs (1),
/// and little-endian merges with two different inputs (2).  For the latter,
/// the input operands are swapped (see PPCInstrAltivec.td).
bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
                             unsigned ShuffleKind, SelectionDAG &DAG) {
  if (DAG.getDataLayout().isLittleEndian()) {
    if (ShuffleKind == 1) // unary
      return isVMerge(N, UnitSize, 0, 0);
    else if (ShuffleKind == 2) // swapped
      return isVMerge(N, UnitSize, 0, 16);
    else
      return false;
  } else {
    if (ShuffleKind == 1) // unary
      return isVMerge(N, UnitSize, 8, 8);
    else if (ShuffleKind == 0) // normal
      return isVMerge(N, UnitSize, 8, 24);
    else
      return false;
  }
}
 
/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes).
/// The ShuffleKind distinguishes between big-endian merges with two
/// different inputs (0), either-endian merges with two identical inputs (1),
/// and little-endian merges with two different inputs (2).  For the latter,
/// the input operands are swapped (see PPCInstrAltivec.td).
bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
                             unsigned ShuffleKind, SelectionDAG &DAG) {
  if (DAG.getDataLayout().isLittleEndian()) {
    if (ShuffleKind == 1) // unary
      return isVMerge(N, UnitSize, 8, 8);
    else if (ShuffleKind == 2) // swapped
      return isVMerge(N, UnitSize, 8, 24);
    else
      return false;
  } else {
    if (ShuffleKind == 1) // unary
      return isVMerge(N, UnitSize, 0, 0);
    else if (ShuffleKind == 0) // normal
      return isVMerge(N, UnitSize, 0, 16);
    else
      return false;
  }
}
 
/**
 * Common function used to match vmrgew and vmrgow shuffles
 *
 * The indexOffset determines whether to look for even or odd words in
 * the shuffle mask. This is based on the of the endianness of the target
 * machine.
 *   - Little Endian:
 *     - Use offset of 0 to check for odd elements
 *     - Use offset of 4 to check for even elements
 *   - Big Endian:
 *     - Use offset of 0 to check for even elements
 *     - Use offset of 4 to check for odd elements
 * A detailed description of the vector element ordering for little endian and
 * big endian can be found at
 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html
 * Targeting your applications - what little endian and big endian IBM XL C/C++
 * compiler differences mean to you
 *
 * The mask to the shuffle vector instruction specifies the indices of the
 * elements from the two input vectors to place in the result. The elements are
 * numbered in array-access order, starting with the first vector. These vectors
 * are always of type v16i8, thus each vector will contain 16 elements of size
 * 8. More info on the shuffle vector can be found in the
 * http://llvm.org/docs/LangRef.html#shufflevector-instruction
 * Language Reference.
 *
 * The RHSStartValue indicates whether the same input vectors are used (unary)
 * or two different input vectors are used, based on the following:
 *   - If the instruction uses the same vector for both inputs, the range of the
 *     indices will be 0 to 15. In this case, the RHSStart value passed should
 *     be 0.
 *   - If the instruction has two different vectors then the range of the
 *     indices will be 0 to 31. In this case, the RHSStart value passed should
 *     be 16 (indices 0-15 specify elements in the first vector while indices 16
 *     to 31 specify elements in the second vector).
 *
 * \param[in] N The shuffle vector SD Node to analyze
 * \param[in] IndexOffset Specifies whether to look for even or odd elements
 * \param[in] RHSStartValue Specifies the starting index for the righthand input
 * vector to the shuffle_vector instruction
 * \return true iff this shuffle vector represents an even or odd word merge
 */
static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset,
                     unsigned RHSStartValue) {
  if (N->getValueType(0) != MVT::v16i8)
    return false;
 
  for (unsigned i = 0; i < 2; ++i)
    for (unsigned j = 0; j < 4; ++j)
      if (!isConstantOrUndef(N->getMaskElt(i*4+j),
                             i*RHSStartValue+j+IndexOffset) ||
          !isConstantOrUndef(N->getMaskElt(i*4+j+8),
                             i*RHSStartValue+j+IndexOffset+8))
        return false;
  return true;
}
 
/**
 * Determine if the specified shuffle mask is suitable for the vmrgew or
 * vmrgow instructions.
 *
 * \param[in] N The shuffle vector SD Node to analyze
 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false)
 * \param[in] ShuffleKind Identify the type of merge:
 *   - 0 = big-endian merge with two different inputs;
 *   - 1 = either-endian merge with two identical inputs;
 *   - 2 = little-endian merge with two different inputs (inputs are swapped for
 *     little-endian merges).
 * \param[in] DAG The current SelectionDAG
 * \return true iff this shuffle mask
 */
bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
                              unsigned ShuffleKind, SelectionDAG &DAG) {
  if (DAG.getDataLayout().isLittleEndian()) {
    unsigned indexOffset = CheckEven ? 4 : 0;
    if (ShuffleKind == 1) // Unary
      return isVMerge(N, indexOffset, 0);
    else if (ShuffleKind == 2) // swapped
      return isVMerge(N, indexOffset, 16);
    else
      return false;
  }
  else {
    unsigned indexOffset = CheckEven ? 0 : 4;
    if (ShuffleKind == 1) // Unary
      return isVMerge(N, indexOffset, 0);
    else if (ShuffleKind == 0) // Normal
      return isVMerge(N, indexOffset, 16);
    else
      return false;
  }
  return false;
}
 
/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
/// amount, otherwise return -1.
/// The ShuffleKind distinguishes between big-endian operations with two
/// different inputs (0), either-endian operations with two identical inputs
/// (1), and little-endian operations with two different inputs (2).  For the
/// latter, the input operands are swapped (see PPCInstrAltivec.td).
int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
                             SelectionDAG &DAG) {
  if (N->getValueType(0) != MVT::v16i8)
    return -1;
 
  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
 
  // Find the first non-undef value in the shuffle mask.
  unsigned i;
  for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i)
    /*search*/;
 
  if (i == 16) return -1;  // all undef.
 
  // Otherwise, check to see if the rest of the elements are consecutively
  // numbered from this value.
  unsigned ShiftAmt = SVOp->getMaskElt(i);
  if (ShiftAmt < i) return -1;
 
  ShiftAmt -= i;
  bool isLE = DAG.getDataLayout().isLittleEndian();
 
  if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
    // Check the rest of the elements to see if they are consecutive.
    for (++i; i != 16; ++i)
      if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i))
        return -1;
  } else if (ShuffleKind == 1) {
    // Check the rest of the elements to see if they are consecutive.
    for (++i; i != 16; ++i)
      if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15))
        return -1;
  } else
    return -1;
 
  if (isLE)
    ShiftAmt = 16 - ShiftAmt;
 
  return ShiftAmt;
}
 
/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
/// specifies a splat of a single element that is suitable for input to
/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.).
bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
  EVT VT = N->getValueType(0);
  if (VT == MVT::v2i64 || VT == MVT::v2f64)
    return EltSize == 8 && N->getMaskElt(0) == N->getMaskElt(1);
 
  assert(VT == MVT::v16i8 && isPowerOf2_32(EltSize) &&
         EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes");
 
  // The consecutive indices need to specify an element, not part of two
  // different elements.  So abandon ship early if this isn't the case.
  if (N->getMaskElt(0) % EltSize != 0)
    return false;
 
  // This is a splat operation if each element of the permute is the same, and
  // if the value doesn't reference the second vector.
  unsigned ElementBase = N->getMaskElt(0);
 
  // FIXME: Handle UNDEF elements too!
  if (ElementBase >= 16)
    return false;
 
  // Check that the indices are consecutive, in the case of a multi-byte element
  // splatted with a v16i8 mask.
  for (unsigned i = 1; i != EltSize; ++i)
    if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase))
      return false;
 
  for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
    // An UNDEF element is a sequence of UNDEF bytes.
    if (N->getMaskElt(i) < 0) {
      for (unsigned j = 1; j != EltSize; ++j)
        if (N->getMaskElt(i + j) >= 0)
          return false;
    } else
      for (unsigned j = 0; j != EltSize; ++j)
        if (N->getMaskElt(i + j) != N->getMaskElt(j))
          return false;
  }
  return true;
}
 
/// Check that the mask is shuffling N byte elements. Within each N byte
/// element of the mask, the indices could be either in increasing or
/// decreasing order as long as they are consecutive.
/// \param[in] N the shuffle vector SD Node to analyze
/// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/
/// Word/DoubleWord/QuadWord).
/// \param[in] StepLen the delta indices number among the N byte element, if
/// the mask is in increasing/decreasing order then it is 1/-1.
/// \return true iff the mask is shuffling N byte elements.
static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width,
                                   int StepLen) {
  assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) &&
         "Unexpected element width.");
  assert((StepLen == 1 || StepLen == -1) && "Unexpected element width.");
 
  unsigned NumOfElem = 16 / Width;
  unsigned MaskVal[16]; //  Width is never greater than 16
  for (unsigned i = 0; i < NumOfElem; ++i) {
    MaskVal[0] = N->getMaskElt(i * Width);
    if ((StepLen == 1) && (MaskVal[0] % Width)) {
      return false;
    } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) {
      return false;
    }
 
    for (unsigned int j = 1; j < Width; ++j) {
      MaskVal[j] = N->getMaskElt(i * Width + j);
      if (MaskVal[j] != MaskVal[j-1] + StepLen) {
        return false;
      }
    }
  }
 
  return true;
}
 
bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
                          unsigned &InsertAtByte, bool &Swap, bool IsLE) {
  if (!isNByteElemShuffleMask(N, 4, 1))
    return false;
 
  // Now we look at mask elements 0,4,8,12
  unsigned M0 = N->getMaskElt(0) / 4;
  unsigned M1 = N->getMaskElt(4) / 4;
  unsigned M2 = N->getMaskElt(8) / 4;
  unsigned M3 = N->getMaskElt(12) / 4;
  unsigned LittleEndianShifts[] = { 2, 1, 0, 3 };
  unsigned BigEndianShifts[] = { 3, 0, 1, 2 };
 
  // Below, let H and L be arbitrary elements of the shuffle mask
  // where H is in the range [4,7] and L is in the range [0,3].
  // H, 1, 2, 3 or L, 5, 6, 7
  if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) ||
      (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) {
    ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3];
    InsertAtByte = IsLE ? 12 : 0;
    Swap = M0 < 4;
    return true;
  }
  // 0, H, 2, 3 or 4, L, 6, 7
  if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) ||
      (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) {
    ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3];
    InsertAtByte = IsLE ? 8 : 4;
    Swap = M1 < 4;
    return true;
  }
  // 0, 1, H, 3 or 4, 5, L, 7
  if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) ||
      (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) {
    ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3];
    InsertAtByte = IsLE ? 4 : 8;
    Swap = M2 < 4;
    return true;
  }
  // 0, 1, 2, H or 4, 5, 6, L
  if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) ||
      (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) {
    ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3];
    InsertAtByte = IsLE ? 0 : 12;
    Swap = M3 < 4;
    return true;
  }
 
  // If both vector operands for the shuffle are the same vector, the mask will
  // contain only elements from the first one and the second one will be undef.
  if (N->getOperand(1).isUndef()) {
    ShiftElts = 0;
    Swap = true;
    unsigned XXINSERTWSrcElem = IsLE ? 2 : 1;
    if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) {
      InsertAtByte = IsLE ? 12 : 0;
      return true;
    }
    if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) {
      InsertAtByte = IsLE ? 8 : 4;
      return true;
    }
    if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) {
      InsertAtByte = IsLE ? 4 : 8;
      return true;
    }
    if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) {
      InsertAtByte = IsLE ? 0 : 12;
      return true;
    }
  }
 
  return false;
}
 
bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
                               bool &Swap, bool IsLE) {
  assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
  // Ensure each byte index of the word is consecutive.
  if (!isNByteElemShuffleMask(N, 4, 1))
    return false;
 
  // Now we look at mask elements 0,4,8,12, which are the beginning of words.
  unsigned M0 = N->getMaskElt(0) / 4;
  unsigned M1 = N->getMaskElt(4) / 4;
  unsigned M2 = N->getMaskElt(8) / 4;
  unsigned M3 = N->getMaskElt(12) / 4;
 
  // If both vector operands for the shuffle are the same vector, the mask will
  // contain only elements from the first one and the second one will be undef.
  if (N->getOperand(1).isUndef()) {
    assert(M0 < 4 && "Indexing into an undef vector?");
    if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4)
      return false;
 
    ShiftElts = IsLE ? (4 - M0) % 4 : M0;
    Swap = false;
    return true;
  }
 
  // Ensure each word index of the ShuffleVector Mask is consecutive.
  if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8)
    return false;
 
  if (IsLE) {
    if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) {
      // Input vectors don't need to be swapped if the leading element
      // of the result is one of the 3 left elements of the second vector
      // (or if there is no shift to be done at all).
      Swap = false;
      ShiftElts = (8 - M0) % 8;
    } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) {
      // Input vectors need to be swapped if the leading element
      // of the result is one of the 3 left elements of the first vector
      // (or if we're shifting by 4 - thereby simply swapping the vectors).
      Swap = true;
      ShiftElts = (4 - M0) % 4;
    }
 
    return true;
  } else {                                          // BE
    if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) {
      // Input vectors don't need to be swapped if the leading element
      // of the result is one of the 4 elements of the first vector.
      Swap = false;
      ShiftElts = M0;
    } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) {
      // Input vectors need to be swapped if the leading element
      // of the result is one of the 4 elements of the right vector.
      Swap = true;
      ShiftElts = M0 - 4;
    }
 
    return true;
  }
}
 
bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) {
  assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
 
  if (!isNByteElemShuffleMask(N, Width, -1))
    return false;
 
  for (int i = 0; i < 16; i += Width)
    if (N->getMaskElt(i) != i + Width - 1)
      return false;
 
  return true;
}
 
bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) {
  return isXXBRShuffleMaskHelper(N, 2);
}
 
bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) {
  return isXXBRShuffleMaskHelper(N, 4);
}
 
bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) {
  return isXXBRShuffleMaskHelper(N, 8);
}
 
bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) {
  return isXXBRShuffleMaskHelper(N, 16);
}
 
/// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap
/// if the inputs to the instruction should be swapped and set \p DM to the
/// value for the immediate.
/// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI
/// AND element 0 of the result comes from the first input (LE) or second input
/// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered.
/// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle
/// mask.
bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM,
                               bool &Swap, bool IsLE) {
  assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
 
  // Ensure each byte index of the double word is consecutive.
  if (!isNByteElemShuffleMask(N, 8, 1))
    return false;
 
  unsigned M0 = N->getMaskElt(0) / 8;
  unsigned M1 = N->getMaskElt(8) / 8;
  assert(((M0 | M1) < 4) && "A mask element out of bounds?");
 
  // If both vector operands for the shuffle are the same vector, the mask will
  // contain only elements from the first one and the second one will be undef.
  if (N->getOperand(1).isUndef()) {
    if ((M0 | M1) < 2) {
      DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1);
      Swap = false;
      return true;
    } else
      return false;
  }
 
  if (IsLE) {
    if (M0 > 1 && M1 < 2) {
      Swap = false;
    } else if (M0 < 2 && M1 > 1) {
      M0 = (M0 + 2) % 4;
      M1 = (M1 + 2) % 4;
      Swap = true;
    } else
      return false;
 
    // Note: if control flow comes here that means Swap is already set above
    DM = (((~M1) & 1) << 1) + ((~M0) & 1);
    return true;
  } else { // BE
    if (M0 < 2 && M1 > 1) {
      Swap = false;
    } else if (M0 > 1 && M1 < 2) {
      M0 = (M0 + 2) % 4;
      M1 = (M1 + 2) % 4;
      Swap = true;
    } else
      return false;
 
    // Note: if control flow comes here that means Swap is already set above
    DM = (M0 << 1) + (M1 & 1);
    return true;
  }
}
 
 
/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
/// appropriate for PPC mnemonics (which have a big endian bias - namely
/// elements are counted from the left of the vector register).
unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
                                         SelectionDAG &DAG) {
  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
  assert(isSplatShuffleMask(SVOp, EltSize));
  EVT VT = SVOp->getValueType(0);
 
  if (VT == MVT::v2i64 || VT == MVT::v2f64)
    return DAG.getDataLayout().isLittleEndian() ? 1 - SVOp->getMaskElt(0)
                                                : SVOp->getMaskElt(0);
 
  if (DAG.getDataLayout().isLittleEndian())
    return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize);
  else
    return SVOp->getMaskElt(0) / EltSize;
}
 
/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
/// by using a vspltis[bhw] instruction of the specified element size, return
/// the constant being splatted.  The ByteSize field indicates the number of
/// bytes of each element [124] -> [bhw].
SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
  SDValue OpVal;
 
  // If ByteSize of the splat is bigger than the element size of the
  // build_vector, then we have a case where we are checking for a splat where
  // multiple elements of the buildvector are folded together into a single
  // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8).
  unsigned EltSize = 16/N->getNumOperands();
  if (EltSize < ByteSize) {
    unsigned Multiple = ByteSize/EltSize;   // Number of BV entries per spltval.
    SDValue UniquedVals[4];
    assert(Multiple > 1 && Multiple <= 4 && "How can this happen?");
 
    // See if all of the elements in the buildvector agree across.
    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
      if (N->getOperand(i).isUndef()) continue;
      // If the element isn't a constant, bail fully out.
      if (!isa<ConstantSDNode>(N->getOperand(i))) return SDValue();
 
      if (!UniquedVals[i&(Multiple-1)].getNode())
        UniquedVals[i&(Multiple-1)] = N->getOperand(i);
      else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i))
        return SDValue();  // no match.
    }
 
    // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains
    // either constant or undef values that are identical for each chunk.  See
    // if these chunks can form into a larger vspltis*.
 
    // Check to see if all of the leading entries are either 0 or -1.  If
    // neither, then this won't fit into the immediate field.
    bool LeadingZero = true;
    bool LeadingOnes = true;
    for (unsigned i = 0; i != Multiple-1; ++i) {
      if (!UniquedVals[i].getNode()) continue;  // Must have been undefs.
 
      LeadingZero &= isNullConstant(UniquedVals[i]);
      LeadingOnes &= isAllOnesConstant(UniquedVals[i]);
    }
    // Finally, check the least significant entry.
    if (LeadingZero) {
      if (!UniquedVals[Multiple-1].getNode())
        return DAG.getTargetConstant(0, SDLoc(N), MVT::i32);  // 0,0,0,undef
      int Val = UniquedVals[Multiple - 1]->getAsZExtVal();
      if (Val < 16)                                   // 0,0,0,4 -> vspltisw(4)
        return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
    }
    if (LeadingOnes) {
      if (!UniquedVals[Multiple-1].getNode())
        return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef
      int Val =cast<ConstantSDNode>(UniquedVals[Multiple-1])->getSExtValue();
      if (Val >= -16)                            // -1,-1,-1,-2 -> vspltisw(-2)
        return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32);
    }
 
    return SDValue();
  }
 
  // Check to see if this buildvec has a single non-undef value in its elements.
  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
    if (N->getOperand(i).isUndef()) continue;
    if (!OpVal.getNode())
      OpVal = N->getOperand(i);
    else if (OpVal != N->getOperand(i))
      return SDValue();
  }
 
  if (!OpVal.getNode()) return SDValue();  // All UNDEF: use implicit def.
 
  unsigned ValSizeInBytes = EltSize;
  uint64_t Value = 0;
  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) {
    Value = CN->getZExtValue();
  } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) {
    assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!");
    Value = llvm::bit_cast<uint32_t>(CN->getValueAPF().convertToFloat());
  }
 
  // If the splat value is larger than the element value, then we can never do
  // this splat.  The only case that we could fit the replicated bits into our
  // immediate field for would be zero, and we prefer to use vxor for it.
  if (ValSizeInBytes < ByteSize) return SDValue();
 
  // If the element value is larger than the splat value, check if it consists
  // of a repeated bit pattern of size ByteSize.
  if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8))
    return SDValue();
 
  // Properly sign extend the value.
  int MaskVal = SignExtend32(Value, ByteSize * 8);
 
  // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
  if (MaskVal == 0) return SDValue();
 
  // Finally, if this value fits in a 5 bit sext field, return it
  if (SignExtend32<5>(MaskVal) == MaskVal)
    return DAG.getSignedTargetConstant(MaskVal, SDLoc(N), MVT::i32);
  return SDValue();
}
 
//===----------------------------------------------------------------------===//
//  Addressing Mode Selection
//===----------------------------------------------------------------------===//
 
/// isIntS16Immediate - This method tests to see if the node is either a 32-bit
/// or 64-bit immediate, and if the value can be accurately represented as a
/// sign extension from a 16-bit value.  If so, this returns true and the
/// immediate.
bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) {
  if (!isa<ConstantSDNode>(N))
    return false;
 
  Imm = (int16_t)N->getAsZExtVal();
  if (N->getValueType(0) == MVT::i32)
    return Imm == (int32_t)N->getAsZExtVal();
  else
    return Imm == (int64_t)N->getAsZExtVal();
}
bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) {
  return isIntS16Immediate(Op.getNode(), Imm);
}
 
/// Used when computing address flags for selecting loads and stores.
/// If we have an OR, check if the LHS and RHS are provably disjoint.
/// An OR of two provably disjoint values is equivalent to an ADD.
/// Most PPC load/store instructions compute the effective address as a sum,
/// so doing this conversion is useful.
static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) {
  if (N.getOpcode() != ISD::OR)
    return false;
  KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
  if (!LHSKnown.Zero.getBoolValue())
    return false;
  KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1));
  return (~(LHSKnown.Zero | RHSKnown.Zero) == 0);
}
 
/// SelectAddressEVXRegReg - Given the specified address, check to see if it can
/// be represented as an indexed [r+r] operation.
bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
                                               SDValue &Index,
                                               SelectionDAG &DAG) const {
  for (SDNode *U : N->users()) {
    if (MemSDNode *Memop = dyn_cast<MemSDNode>(U)) {
      if (Memop->getMemoryVT() == MVT::f64) {
          Base = N.getOperand(0);
          Index = N.getOperand(1);
          return true;
      }
    }
  }
  return false;
}
 
/// isIntS34Immediate - This method tests if value of node given can be
/// accurately represented as a sign extension from a 34-bit value.  If so,
/// this returns true and the immediate.
bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) {
  if (!isa<ConstantSDNode>(N))
    return false;
 
  Imm = cast<ConstantSDNode>(N)->getSExtValue();
  return isInt<34>(Imm);
}
bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) {
  return isIntS34Immediate(Op.getNode(), Imm);
}
 
/// SelectAddressRegReg - Given the specified addressed, check to see if it
/// can be represented as an indexed [r+r] operation.  Returns false if it
/// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is
/// non-zero and N can be represented by a base register plus a signed 16-bit
/// displacement, make a more precise judgement by checking (displacement % \p
/// EncodingAlignment).
bool PPCTargetLowering::SelectAddressRegReg(
    SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG,
    MaybeAlign EncodingAlignment) const {
  // If we have a PC Relative target flag don't select as [reg+reg]. It will be
  // a [pc+imm].
  if (SelectAddressPCRel(N, Base))
    return false;
 
  int16_t Imm = 0;
  if (N.getOpcode() == ISD::ADD) {
    // Is there any SPE load/store (f64), which can't handle 16bit offset?
    // SPE load/store can only handle 8-bit offsets.
    if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG))
        return true;
    if (isIntS16Immediate(N.getOperand(1), Imm) &&
        (!EncodingAlignment || isAligned(*EncodingAlignment, Imm)))
      return false; // r+i
    if (N.getOperand(1).getOpcode() == PPCISD::Lo)
      return false;    // r+i
 
    Base = N.getOperand(0);
    Index = N.getOperand(1);
    return true;
  } else if (N.getOpcode() == ISD::OR) {
    if (isIntS16Immediate(N.getOperand(1), Imm) &&
        (!EncodingAlignment || isAligned(*EncodingAlignment, Imm)))
      return false; // r+i can fold it if we can.
 
    // If this is an or of disjoint bitfields, we can codegen this as an add
    // (for better address arithmetic) if the LHS and RHS of the OR are provably
    // disjoint.
    KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
 
    if (LHSKnown.Zero.getBoolValue()) {
      KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1));
      // If all of the bits are known zero on the LHS or RHS, the add won't
      // carry.
      if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) {
        Base = N.getOperand(0);
        Index = N.getOperand(1);
        return true;
      }
    }
  }
 
  return false;
}
 
// If we happen to be doing an i64 load or store into a stack slot that has
// less than a 4-byte alignment, then the frame-index elimination may need to
// use an indexed load or store instruction (because the offset may not be a
// multiple of 4). The extra register needed to hold the offset comes from the
// register scavenger, and it is possible that the scavenger will need to use
// an emergency spill slot. As a result, we need to make sure that a spill slot
// is allocated when doing an i64 load/store into a less-than-4-byte-aligned
// stack slot.
static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
  // FIXME: This does not handle the LWA case.
  if (VT != MVT::i64)
    return;
 
  // NOTE: We'll exclude negative FIs here, which come from argument
  // lowering, because there are no known test cases triggering this problem
  // using packed structures (or similar). We can remove this exclusion if
  // we find such a test case. The reason why this is so test-case driven is
  // because this entire 'fixup' is only to prevent crashes (from the
  // register scavenger) on not-really-valid inputs. For example, if we have:
  //   %a = alloca i1
  //   %b = bitcast i1* %a to i64*
  //   store i64* a, i64 b
  // then the store should really be marked as 'align 1', but is not. If it
  // were marked as 'align 1' then the indexed form would have been
  // instruction-selected initially, and the problem this 'fixup' is preventing
  // won't happen regardless.
  if (FrameIdx < 0)
    return;
 
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
 
  if (MFI.getObjectAlign(FrameIdx) >= Align(4))
    return;
 
  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
  FuncInfo->setHasNonRISpills();
}
 
/// Returns true if the address N can be represented by a base register plus
/// a signed 16-bit displacement [r+imm], and if it is not better
/// represented as reg+reg.  If \p EncodingAlignment is non-zero, only accept
/// displacements that are multiples of that value.
bool PPCTargetLowering::SelectAddressRegImm(
    SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG,
    MaybeAlign EncodingAlignment) const {
  // FIXME dl should come from parent load or store, not from address
  SDLoc dl(N);
 
  // If we have a PC Relative target flag don't select as [reg+imm]. It will be
  // a [pc+imm].
  if (SelectAddressPCRel(N, Base))
    return false;
 
  // If this can be more profitably realized as r+r, fail.
  if (SelectAddressRegReg(N, Disp, Base, DAG, EncodingAlignment))
    return false;
 
  if (N.getOpcode() == ISD::ADD) {
    int16_t imm = 0;
    if (isIntS16Immediate(N.getOperand(1), imm) &&
        (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) {
      Disp = DAG.getSignedTargetConstant(imm, dl, N.getValueType());
      if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
        Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
        fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
      } else {
        Base = N.getOperand(0);
      }
      return true; // [r+i]
    } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) {
      // Match LOAD (ADD (X, Lo(G))).
      assert(!N.getOperand(1).getConstantOperandVal(1) &&
             "Cannot handle constant offsets yet!");
      Disp = N.getOperand(1).getOperand(0);  // The global address.
      assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
             Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
             Disp.getOpcode() == ISD::TargetConstantPool ||
             Disp.getOpcode() == ISD::TargetJumpTable);
      Base = N.getOperand(0);
      return true;  // [&g+r]
    }
  } else if (N.getOpcode() == ISD::OR) {
    int16_t imm = 0;
    if (isIntS16Immediate(N.getOperand(1), imm) &&
        (!EncodingAlignment || isAligned(*EncodingAlignment, imm))) {
      // If this is an or of disjoint bitfields, we can codegen this as an add
      // (for better address arithmetic) if the LHS and RHS of the OR are
      // provably disjoint.
      KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
 
      if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
        // If all of the bits are known zero on the LHS or RHS, the add won't
        // carry.
        if (FrameIndexSDNode *FI =
              dyn_cast<FrameIndexSDNode>(N.getOperand(0))) {
          Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
          fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
        } else {
          Base = N.getOperand(0);
        }
        Disp = DAG.getTargetConstant(imm, dl, N.getValueType());
        return true;
      }
    }
  } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) {
    // Loading from a constant address.
 
    // If this address fits entirely in a 16-bit sext immediate field, codegen
    // this as "d, 0"
    int16_t Imm;
    if (isIntS16Immediate(CN, Imm) &&
        (!EncodingAlignment || isAligned(*EncodingAlignment, Imm))) {
      Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0));
      Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
                             CN->getValueType(0));
      return true;
    }
 
    // Handle 32-bit sext immediates with LIS + addr mode.
    if ((CN->getValueType(0) == MVT::i32 ||
         (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
        (!EncodingAlignment ||
         isAligned(*EncodingAlignment, CN->getZExtValue()))) {
      int Addr = (int)CN->getZExtValue();
 
      // Otherwise, break this down into an LIS + disp.
      Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32);
 
      Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl,
                                   MVT::i32);
      unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
      Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0);
      return true;
    }
  }
 
  Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout()));
  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N)) {
    Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
    fixupFuncForFI(DAG, FI->getIndex(), N.getValueType());
  } else
    Base = N;
  return true;      // [r+0]
}
 
/// Similar to the 16-bit case but for instructions that take a 34-bit
/// displacement field (prefixed loads/stores).
bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
                                              SDValue &Base,
                                              SelectionDAG &DAG) const {
  // Only on 64-bit targets.
  if (N.getValueType() != MVT::i64)
    return false;
 
  SDLoc dl(N);
  int64_t Imm = 0;
 
  if (N.getOpcode() == ISD::ADD) {
    if (!isIntS34Immediate(N.getOperand(1), Imm))
      return false;
    Disp = DAG.getSignedTargetConstant(Imm, dl, N.getValueType());
    if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0)))
      Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
    else
      Base = N.getOperand(0);
    return true;
  }
 
  if (N.getOpcode() == ISD::OR) {
    if (!isIntS34Immediate(N.getOperand(1), Imm))
      return false;
    // If this is an or of disjoint bitfields, we can codegen this as an add
    // (for better address arithmetic) if the LHS and RHS of the OR are
    // provably disjoint.
    KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
    if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL)
      return false;
    if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(N.getOperand(0)))
      Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType());
    else
      Base = N.getOperand(0);
    Disp = DAG.getSignedTargetConstant(Imm, dl, N.getValueType());
    return true;
  }
 
  if (isIntS34Immediate(N, Imm)) { // If the address is a 34-bit const.
    Disp = DAG.getSignedTargetConstant(Imm, dl, N.getValueType());
    Base = DAG.getRegister(PPC::ZERO8, N.getValueType());
    return true;
  }
 
  return false;
}
 
/// SelectAddressRegRegOnly - Given the specified addressed, force it to be
/// represented as an indexed [r+r] operation.
bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
                                                SDValue &Index,
                                                SelectionDAG &DAG) const {
  // Check to see if we can easily represent this as an [r+r] address.  This
  // will fail if it thinks that the address is more profitably represented as
  // reg+imm, e.g. where imm = 0.
  if (SelectAddressRegReg(N, Base, Index, DAG))
    return true;
 
  // If the address is the result of an add, we will utilize the fact that the
  // address calculation includes an implicit add.  However, we can reduce
  // register pressure if we do not materialize a constant just for use as the
  // index register.  We only get rid of the add if it is not an add of a
  // value and a 16-bit signed constant and both have a single use.
  int16_t imm = 0;
  if (N.getOpcode() == ISD::ADD &&
      (!isIntS16Immediate(N.getOperand(1), imm) ||
       !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) {
    Base = N.getOperand(0);
    Index = N.getOperand(1);
    return true;
  }
 
  // Otherwise, do it the hard way, using R0 as the base register.
  Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
                         N.getValueType());
  Index = N;
  return true;
}
 
template <typename Ty> static bool isValidPCRelNode(SDValue N) {
  Ty *PCRelCand = dyn_cast<Ty>(N);
  return PCRelCand && (PPCInstrInfo::hasPCRelFlag(PCRelCand->getTargetFlags()));
}
 
/// Returns true if this address is a PC Relative address.
/// PC Relative addresses are marked with the flag PPCII::MO_PCREL_FLAG
/// or if the node opcode is PPCISD::MAT_PCREL_ADDR.
bool PPCTargetLowering::SelectAddressPCRel(SDValue N, SDValue &Base) const {
  // This is a materialize PC Relative node. Always select this as PC Relative.
  Base = N;
  if (N.getOpcode() == PPCISD::MAT_PCREL_ADDR)
    return true;
  if (isValidPCRelNode<ConstantPoolSDNode>(N) ||
      isValidPCRelNode<GlobalAddressSDNode>(N) ||
      isValidPCRelNode<JumpTableSDNode>(N) ||
      isValidPCRelNode<BlockAddressSDNode>(N))
    return true;
  return false;
}
 
/// Returns true if we should use a direct load into vector instruction
/// (such as lxsd or lfd), instead of a load into gpr + direct move sequence.
static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) {
 
  // If there are any other uses other than scalar to vector, then we should
  // keep it as a scalar load -> direct move pattern to prevent multiple
  // loads.
  LoadSDNode *LD = dyn_cast<LoadSDNode>(N);
  if (!LD)
    return false;
 
  EVT MemVT = LD->getMemoryVT();
  if (!MemVT.isSimple())
    return false;
  switch(MemVT.getSimpleVT().SimpleTy) {
  case MVT::i64:
    break;
  case MVT::i32:
    if (!ST.hasP8Vector())
      return false;
    break;
  case MVT::i16:
  case MVT::i8:
    if (!ST.hasP9Vector())
      return false;
    break;
  default:
    return false;
  }
 
  SDValue LoadedVal(N, 0);
  if (!LoadedVal.hasOneUse())
    return false;
 
  for (SDUse &Use : LD->uses())
    if (Use.getResNo() == 0 &&
        Use.getUser()->getOpcode() != ISD::SCALAR_TO_VECTOR &&
        Use.getUser()->getOpcode() != PPCISD::SCALAR_TO_VECTOR_PERMUTED)
      return false;
 
  return true;
}
 
/// getPreIndexedAddressParts - returns true by value, base pointer and
/// offset pointer and addressing mode by reference if the node's address
/// can be legally represented as pre-indexed load / store address.
bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
                                                  SDValue &Offset,
                                                  ISD::MemIndexedMode &AM,
                                                  SelectionDAG &DAG) const {
  if (DisablePPCPreinc) return false;
 
  bool isLoad = true;
  SDValue Ptr;
  EVT VT;
  Align Alignment;
  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
    Ptr = LD->getBasePtr();
    VT = LD->getMemoryVT();
    Alignment = LD->getAlign();
  } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
    Ptr = ST->getBasePtr();
    VT  = ST->getMemoryVT();
    Alignment = ST->getAlign();
    isLoad = false;
  } else
    return false;
 
  // Do not generate pre-inc forms for specific loads that feed scalar_to_vector
  // instructions because we can fold these into a more efficient instruction
  // instead, (such as LXSD).
  if (isLoad && usePartialVectorLoads(N, Subtarget)) {
    return false;
  }
 
  // PowerPC doesn't have preinc load/store instructions for vectors
  if (VT.isVector())
    return false;
 
  if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) {
    // Common code will reject creating a pre-inc form if the base pointer
    // is a frame index, or if N is a store and the base pointer is either
    // the same as or a predecessor of the value being stored.  Check for
    // those situations here, and try with swapped Base/Offset instead.
    bool Swap = false;
 
    if (isa<FrameIndexSDNode>(Base) || isa<RegisterSDNode>(Base))
      Swap = true;
    else if (!isLoad) {
      SDValue Val = cast<StoreSDNode>(N)->getValue();
      if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode()))
        Swap = true;
    }
 
    if (Swap)
      std::swap(Base, Offset);
 
    AM = ISD::PRE_INC;
    return true;
  }
 
  // LDU/STU can only handle immediates that are a multiple of 4.
  if (VT != MVT::i64) {
    if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, std::nullopt))
      return false;
  } else {
    // LDU/STU need an address with at least 4-byte alignment.
    if (Alignment < Align(4))
      return false;
 
    if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, Align(4)))
      return false;
  }
 
  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
    // PPC64 doesn't have lwau, but it does have lwaux.  Reject preinc load of
    // sext i32 to i64 when addr mode is r+i.
    if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 &&
        LD->getExtensionType() == ISD::SEXTLOAD &&
        isa<ConstantSDNode>(Offset))
      return false;
  }
 
  AM = ISD::PRE_INC;
  return true;
}
 
//===----------------------------------------------------------------------===//
//  LowerOperation implementation
//===----------------------------------------------------------------------===//
 
/// Return true if we should reference labels using a PICBase, set the HiOpFlags
/// and LoOpFlags to the target MO flags.
static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget,
                               unsigned &HiOpFlags, unsigned &LoOpFlags,
                               const GlobalValue *GV = nullptr) {
  HiOpFlags = PPCII::MO_HA;
  LoOpFlags = PPCII::MO_LO;
 
  // Don't use the pic base if not in PIC relocation model.
  if (IsPIC) {
    HiOpFlags = PPCII::MO_PIC_HA_FLAG;
    LoOpFlags = PPCII::MO_PIC_LO_FLAG;
  }
}
 
static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
                             SelectionDAG &DAG) {
  SDLoc DL(HiPart);
  EVT PtrVT = HiPart.getValueType();
  SDValue Zero = DAG.getConstant(0, DL, PtrVT);
 
  SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero);
  SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero);
 
  // With PIC, the first instruction is actually "GR+hi(&G)".
  if (isPIC)
    Hi = DAG.getNode(ISD::ADD, DL, PtrVT,
                     DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi);
 
  // Generate non-pic code that has direct accesses to the constant pool.
  // The address of the global is just (hi(&g)+lo(&g)).
  return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo);
}
 
static void setUsesTOCBasePtr(MachineFunction &MF) {
  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
  FuncInfo->setUsesTOCBasePtr();
}
 
static void setUsesTOCBasePtr(SelectionDAG &DAG) {
  setUsesTOCBasePtr(DAG.getMachineFunction());
}
 
SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
                                       SDValue GA) const {
  EVT VT = Subtarget.getScalarIntVT();
  SDValue Reg = Subtarget.isPPC64() ? DAG.getRegister(PPC::X2, VT)
                : Subtarget.isAIXABI()
                    ? DAG.getRegister(PPC::R2, VT)
                    : DAG.getNode(PPCISD::GlobalBaseReg, dl, VT);
  SDValue Ops[] = { GA, Reg };
  return DAG.getMemIntrinsicNode(
      PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT,
      MachinePointerInfo::getGOT(DAG.getMachineFunction()), std::nullopt,
      MachineMemOperand::MOLoad);
}
 
SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
                                             SelectionDAG &DAG) const {
  EVT PtrVT = Op.getValueType();
  ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
  const Constant *C = CP->getConstVal();
 
  // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
  // The actual address of the GlobalValue is stored in the TOC.
  if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
    if (Subtarget.isUsingPCRelativeCalls()) {
      SDLoc DL(CP);
      EVT Ty = getPointerTy(DAG.getDataLayout());
      SDValue ConstPool = DAG.getTargetConstantPool(
          C, Ty, CP->getAlign(), CP->getOffset(), PPCII::MO_PCREL_FLAG);
      return DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, ConstPool);
    }
    setUsesTOCBasePtr(DAG);
    SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0);
    return getTOCEntry(DAG, SDLoc(CP), GA);
  }
 
  unsigned MOHiFlag, MOLoFlag;
  bool IsPIC = isPositionIndependent();
  getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag);
 
  if (IsPIC && Subtarget.isSVR4ABI()) {
    SDValue GA =
        DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), PPCII::MO_PIC_FLAG);
    return getTOCEntry(DAG, SDLoc(CP), GA);
  }
 
  SDValue CPIHi =
      DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0, MOHiFlag);
  SDValue CPILo =
      DAG.getTargetConstantPool(C, PtrVT, CP->getAlign(), 0, MOLoFlag);
  return LowerLabelRef(CPIHi, CPILo, IsPIC, DAG);
}
 
// For 64-bit PowerPC, prefer the more compact relative encodings.
// This trades 32 bits per jump table entry for one or two instructions
// on the jump site.
unsigned PPCTargetLowering::getJumpTableEncoding() const {
  if (isJumpTableRelative())
    return MachineJumpTableInfo::EK_LabelDifference32;
 
  return TargetLowering::getJumpTableEncoding();
}
 
bool PPCTargetLowering::isJumpTableRelative() const {
  if (UseAbsoluteJumpTables)
    return false;
  if (Subtarget.isPPC64() || Subtarget.isAIXABI())
    return true;
  return TargetLowering::isJumpTableRelative();
}
 
SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
                                                    SelectionDAG &DAG) const {
  if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
    return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
 
  switch (getTargetMachine().getCodeModel()) {
  case CodeModel::Small:
  case CodeModel::Medium:
    return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
  default:
    return DAG.getNode(PPCISD::GlobalBaseReg, SDLoc(),
                       getPointerTy(DAG.getDataLayout()));
  }
}
 
const MCExpr *
PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
                                                unsigned JTI,
                                                MCContext &Ctx) const {
  if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
    return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
 
  switch (getTargetMachine().getCodeModel()) {
  case CodeModel::Small:
  case CodeModel::Medium:
    return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
  default:
    return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);
  }
}
 
SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
  EVT PtrVT = Op.getValueType();
  JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
 
  // isUsingPCRelativeCalls() returns true when PCRelative is enabled
  if (Subtarget.isUsingPCRelativeCalls()) {
    SDLoc DL(JT);
    EVT Ty = getPointerTy(DAG.getDataLayout());
    SDValue GA =
        DAG.getTargetJumpTable(JT->getIndex(), Ty, PPCII::MO_PCREL_FLAG);
    SDValue MatAddr = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
    return MatAddr;
  }
 
  // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
  // The actual address of the GlobalValue is stored in the TOC.
  if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
    setUsesTOCBasePtr(DAG);
    SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
    return getTOCEntry(DAG, SDLoc(JT), GA);
  }
 
  unsigned MOHiFlag, MOLoFlag;
  bool IsPIC = isPositionIndependent();
  getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag);
 
  if (IsPIC && Subtarget.isSVR4ABI()) {
    SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT,
                                        PPCII::MO_PIC_FLAG);
    return getTOCEntry(DAG, SDLoc(GA), GA);
  }
 
  SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag);
  SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag);
  return LowerLabelRef(JTIHi, JTILo, IsPIC, DAG);
}
 
SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
                                             SelectionDAG &DAG) const {
  EVT PtrVT = Op.getValueType();
  BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Op);
  const BlockAddress *BA = BASDN->getBlockAddress();
 
  // isUsingPCRelativeCalls() returns true when PCRelative is enabled
  if (Subtarget.isUsingPCRelativeCalls()) {
    SDLoc DL(BASDN);
    EVT Ty = getPointerTy(DAG.getDataLayout());
    SDValue GA = DAG.getTargetBlockAddress(BA, Ty, BASDN->getOffset(),
                                           PPCII::MO_PCREL_FLAG);
    SDValue MatAddr = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
    return MatAddr;
  }
 
  // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
  // The actual BlockAddress is stored in the TOC.
  if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
    setUsesTOCBasePtr(DAG);
    SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset());
    return getTOCEntry(DAG, SDLoc(BASDN), GA);
  }
 
  // 32-bit position-independent ELF stores the BlockAddress in the .got.
  if (Subtarget.is32BitELFABI() && isPositionIndependent())
    return getTOCEntry(
        DAG, SDLoc(BASDN),
        DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()));
 
  unsigned MOHiFlag, MOLoFlag;
  bool IsPIC = isPositionIndependent();
  getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag);
  SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag);
  SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag);
  return LowerLabelRef(TgtBAHi, TgtBALo, IsPIC, DAG);
}
 
SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
                                              SelectionDAG &DAG) const {
  if (Subtarget.isAIXABI())
    return LowerGlobalTLSAddressAIX(Op, DAG);
 
  return LowerGlobalTLSAddressLinux(Op, DAG);
}
 
/// updateForAIXShLibTLSModelOpt - Helper to initialize TLS model opt settings,
/// and then apply the update.
static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
                                         SelectionDAG &DAG,
                                         const TargetMachine &TM) {
  // Initialize TLS model opt setting lazily:
  // (1) Use initial-exec for single TLS var references within current function.
  // (2) Use local-dynamic for multiple TLS var references within current
  // function.
  PPCFunctionInfo *FuncInfo =
      DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
  if (!FuncInfo->isAIXFuncTLSModelOptInitDone()) {
    SmallPtrSet<const GlobalValue *, 8> TLSGV;
    // Iterate over all instructions within current function, collect all TLS
    // global variables (global variables taken as the first parameter to
    // Intrinsic::threadlocal_address).
    const Function &Func = DAG.getMachineFunction().getFunction();
    for (const BasicBlock &BB : Func)
      for (const Instruction &I : BB)
        if (I.getOpcode() == Instruction::Call)
          if (const CallInst *CI = dyn_cast<const CallInst>(&I))
            if (Function *CF = CI->getCalledFunction())
              if (CF->isDeclaration() &&
                  CF->getIntrinsicID() == Intrinsic::threadlocal_address)
                if (const GlobalValue *GV =
                        dyn_cast<GlobalValue>(I.getOperand(0))) {
                  TLSModel::Model GVModel = TM.getTLSModel(GV);
                  if (GVModel == TLSModel::LocalDynamic)
                    TLSGV.insert(GV);
                }
 
    unsigned TLSGVCnt = TLSGV.size();
    LLVM_DEBUG(dbgs() << format("LocalDynamic TLSGV count:%d\n", TLSGVCnt));
    if (TLSGVCnt <= PPCAIXTLSModelOptUseIEForLDLimit)
      FuncInfo->setAIXFuncUseTLSIEForLD();
    FuncInfo->setAIXFuncTLSModelOptInitDone();
  }
 
  if (FuncInfo->isAIXFuncUseTLSIEForLD()) {
    LLVM_DEBUG(
        dbgs() << DAG.getMachineFunction().getName()
               << " function is using the TLS-IE model for TLS-LD access.\n");
    Model = TLSModel::InitialExec;
  }
}
 
SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
                                                    SelectionDAG &DAG) const {
  GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
 
  if (DAG.getTarget().useEmulatedTLS())
    report_fatal_error("Emulated TLS is not yet supported on AIX");
 
  SDLoc dl(GA);
  const GlobalValue *GV = GA->getGlobal();
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  bool Is64Bit = Subtarget.isPPC64();
  TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
 
  // Apply update to the TLS model.
  if (Subtarget.hasAIXShLibTLSModelOpt())
    updateForAIXShLibTLSModelOpt(Model, DAG, getTargetMachine());
 
  // TLS variables are accessed through TOC entries.
  // To support this, set the DAG to use the TOC base pointer.
  setUsesTOCBasePtr(DAG);
 
  bool IsTLSLocalExecModel = Model == TLSModel::LocalExec;
 
  if (IsTLSLocalExecModel || Model == TLSModel::InitialExec) {
    bool HasAIXSmallLocalExecTLS = Subtarget.hasAIXSmallLocalExecTLS();
    bool HasAIXSmallTLSGlobalAttr = false;
    SDValue VariableOffsetTGA =
        DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TPREL_FLAG);
    SDValue VariableOffset = getTOCEntry(DAG, dl, VariableOffsetTGA);
    SDValue TLSReg;
 
    if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
      if (GVar->hasAttribute("aix-small-tls"))
        HasAIXSmallTLSGlobalAttr = true;
 
    if (Is64Bit) {
      // For local-exec and initial-exec on AIX (64-bit), the sequence generated
      // involves a load of the variable offset (from the TOC), followed by an
      // add of the loaded variable offset to R13 (the thread pointer).
      // This code sequence looks like:
      //    ld reg1,var[TC](2)
      //    add reg2, reg1, r13     // r13 contains the thread pointer
      TLSReg = DAG.getRegister(PPC::X13, MVT::i64);
 
      // With the -maix-small-local-exec-tls option, or with the "aix-small-tls"
      // global variable attribute, produce a faster access sequence for
      // local-exec TLS variables where the offset from the TLS base is encoded
      // as an immediate operand.
      //
      // We only utilize the faster local-exec access sequence when the TLS
      // variable has a size within the policy limit. We treat types that are
      // not sized or are empty as being over the policy size limit.
      if ((HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr) &&
          IsTLSLocalExecModel) {
        Type *GVType = GV->getValueType();
        if (GVType->isSized() && !GVType->isEmptyTy() &&
            GV->getDataLayout().getTypeAllocSize(GVType) <=
                AIXSmallTlsPolicySizeLimit)
          return DAG.getNode(PPCISD::Lo, dl, PtrVT, VariableOffsetTGA, TLSReg);
      }
    } else {
      // For local-exec and initial-exec on AIX (32-bit), the sequence generated
      // involves loading the variable offset from the TOC, generating a call to
      // .__get_tpointer to get the thread pointer (which will be in R3), and
      // adding the two together:
      //    lwz reg1,var[TC](2)
      //    bla .__get_tpointer
      //    add reg2, reg1, r3
      TLSReg = DAG.getNode(PPCISD::GET_TPOINTER, dl, PtrVT);
 
      // We do not implement the 32-bit version of the faster access sequence
      // for local-exec that is controlled by the -maix-small-local-exec-tls
      // option, or the "aix-small-tls" global variable attribute.
      if (HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr)
        report_fatal_error("The small-local-exec TLS access sequence is "
                           "currently only supported on AIX (64-bit mode).");
    }
    return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, VariableOffset);
  }
 
  if (Model == TLSModel::LocalDynamic) {
    bool HasAIXSmallLocalDynamicTLS = Subtarget.hasAIXSmallLocalDynamicTLS();
 
    // We do not implement the 32-bit version of the faster access sequence
    // for local-dynamic that is controlled by -maix-small-local-dynamic-tls.
    if (!Is64Bit && HasAIXSmallLocalDynamicTLS)
      report_fatal_error("The small-local-dynamic TLS access sequence is "
                         "currently only supported on AIX (64-bit mode).");
 
    // For local-dynamic on AIX, we need to generate one TOC entry for each
    // variable offset, and a single module-handle TOC entry for the entire
    // file.
 
    SDValue VariableOffsetTGA =
        DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSLD_FLAG);
    SDValue VariableOffset = getTOCEntry(DAG, dl, VariableOffsetTGA);
 
    Module *M = DAG.getMachineFunction().getFunction().getParent();
    GlobalVariable *TLSGV =
        dyn_cast_or_null<GlobalVariable>(M->getOrInsertGlobal(
            StringRef("_$TLSML"), PointerType::getUnqual(*DAG.getContext())));
    TLSGV->setThreadLocalMode(GlobalVariable::LocalDynamicTLSModel);
    assert(TLSGV && "Not able to create GV for _$TLSML.");
    SDValue ModuleHandleTGA =
        DAG.getTargetGlobalAddress(TLSGV, dl, PtrVT, 0, PPCII::MO_TLSLDM_FLAG);
    SDValue ModuleHandleTOC = getTOCEntry(DAG, dl, ModuleHandleTGA);
    SDValue ModuleHandle =
        DAG.getNode(PPCISD::TLSLD_AIX, dl, PtrVT, ModuleHandleTOC);
 
    // With the -maix-small-local-dynamic-tls option, produce a faster access
    // sequence for local-dynamic TLS variables where the offset from the
    // module-handle is encoded as an immediate operand.
    //
    // We only utilize the faster local-dynamic access sequence when the TLS
    // variable has a size within the policy limit. We treat types that are
    // not sized or are empty as being over the policy size limit.
    if (HasAIXSmallLocalDynamicTLS) {
      Type *GVType = GV->getValueType();
      if (GVType->isSized() && !GVType->isEmptyTy() &&
          GV->getDataLayout().getTypeAllocSize(GVType) <=
              AIXSmallTlsPolicySizeLimit)
        return DAG.getNode(PPCISD::Lo, dl, PtrVT, VariableOffsetTGA,
                           ModuleHandle);
    }
 
    return DAG.getNode(ISD::ADD, dl, PtrVT, ModuleHandle, VariableOffset);
  }
 
  // If Local- or Initial-exec or Local-dynamic is not possible or specified,
  // all GlobalTLSAddress nodes are lowered using the general-dynamic model. We
  // need to generate two TOC entries, one for the variable offset, one for the
  // region handle. The global address for the TOC entry of the region handle is
  // created with the MO_TLSGDM_FLAG flag and the global address for the TOC
  // entry of the variable offset is created with MO_TLSGD_FLAG.
  SDValue VariableOffsetTGA =
      DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGD_FLAG);
  SDValue RegionHandleTGA =
      DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLSGDM_FLAG);
  SDValue VariableOffset = getTOCEntry(DAG, dl, VariableOffsetTGA);
  SDValue RegionHandle = getTOCEntry(DAG, dl, RegionHandleTGA);
  return DAG.getNode(PPCISD::TLSGD_AIX, dl, PtrVT, VariableOffset,
                     RegionHandle);
}
 
SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op,
                                                      SelectionDAG &DAG) const {
  // FIXME: TLS addresses currently use medium model code sequences,
  // which is the most useful form.  Eventually support for small and
  // large models could be added if users need it, at the cost of
  // additional complexity.
  GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
  if (DAG.getTarget().useEmulatedTLS())
    return LowerToTLSEmulatedModel(GA, DAG);
 
  SDLoc dl(GA);
  const GlobalValue *GV = GA->getGlobal();
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  bool is64bit = Subtarget.isPPC64();
  const Module *M = DAG.getMachineFunction().getFunction().getParent();
  PICLevel::Level picLevel = M->getPICLevel();
 
  const TargetMachine &TM = getTargetMachine();
  TLSModel::Model Model = TM.getTLSModel(GV);
 
  if (Model == TLSModel::LocalExec) {
    if (Subtarget.isUsingPCRelativeCalls()) {
      SDValue TLSReg = DAG.getRegister(PPC::X13, MVT::i64);
      SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
                                               PPCII::MO_TPREL_PCREL_FLAG);
      SDValue MatAddr =
          DAG.getNode(PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, dl, PtrVT, TGA);
      return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, MatAddr);
    }
 
    SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
                                               PPCII::MO_TPREL_HA);
    SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
                                               PPCII::MO_TPREL_LO);
    SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64)
                             : DAG.getRegister(PPC::R2, MVT::i32);
 
    SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg);
    return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi);
  }
 
  if (Model == TLSModel::InitialExec) {
    bool IsPCRel = Subtarget.isUsingPCRelativeCalls();
    SDValue TGA = DAG.getTargetGlobalAddress(
        GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0);
    SDValue TGATLS = DAG.getTargetGlobalAddress(
        GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_TLS_PCREL_FLAG : PPCII::MO_TLS);
    SDValue TPOffset;
    if (IsPCRel) {
      SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, dl, PtrVT, TGA);
      TPOffset = DAG.getLoad(MVT::i64, dl, DAG.getEntryNode(), MatPCRel,
                             MachinePointerInfo());
    } else {
      SDValue GOTPtr;
      if (is64bit) {
        setUsesTOCBasePtr(DAG);
        SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
        GOTPtr =
            DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, PtrVT, GOTReg, TGA);
      } else {
        if (!TM.isPositionIndependent())
          GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT);
        else if (picLevel == PICLevel::SmallPIC)
          GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
        else
          GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
      }
      TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, PtrVT, TGA, GOTPtr);
    }
    return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS);
  }
 
  if (Model == TLSModel::GeneralDynamic) {
    if (Subtarget.isUsingPCRelativeCalls()) {
      SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
                                               PPCII::MO_GOT_TLSGD_PCREL_FLAG);
      return DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA);
    }
 
    SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
    SDValue GOTPtr;
    if (is64bit) {
      setUsesTOCBasePtr(DAG);
      SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
      GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT,
                                   GOTReg, TGA);
    } else {
      if (picLevel == PICLevel::SmallPIC)
        GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
      else
        GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
    }
    return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT,
                       GOTPtr, TGA, TGA);
  }
 
  if (Model == TLSModel::LocalDynamic) {
    if (Subtarget.isUsingPCRelativeCalls()) {
      SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
                                               PPCII::MO_GOT_TLSLD_PCREL_FLAG);
      SDValue MatPCRel =
          DAG.getNode(PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, dl, PtrVT, TGA);
      return DAG.getNode(PPCISD::PADDI_DTPREL, dl, PtrVT, MatPCRel, TGA);
    }
 
    SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0);
    SDValue GOTPtr;
    if (is64bit) {
      setUsesTOCBasePtr(DAG);
      SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64);
      GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT,
                           GOTReg, TGA);
    } else {
      if (picLevel == PICLevel::SmallPIC)
        GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT);
      else
        GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT);
    }
    SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl,
                                  PtrVT, GOTPtr, TGA, TGA);
    SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl,
                                      PtrVT, TLSAddr, TGA);
    return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA);
  }
 
  llvm_unreachable("Unknown TLS model!");
}
 
SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
                                              SelectionDAG &DAG) const {
  EVT PtrVT = Op.getValueType();
  GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Op);
  SDLoc DL(GSDN);
  const GlobalValue *GV = GSDN->getGlobal();
 
  // 64-bit SVR4 ABI & AIX ABI code is always position-independent.
  // The actual address of the GlobalValue is stored in the TOC.
  if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
    if (Subtarget.isUsingPCRelativeCalls()) {
      EVT Ty = getPointerTy(DAG.getDataLayout());
      if (isAccessedAsGotIndirect(Op)) {
        SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(),
                                                PPCII::MO_GOT_PCREL_FLAG);
        SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
        SDValue Load = DAG.getLoad(MVT::i64, DL, DAG.getEntryNode(), MatPCRel,
                                   MachinePointerInfo());
        return Load;
      } else {
        SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(),
                                                PPCII::MO_PCREL_FLAG);
        return DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA);
      }
    }
    setUsesTOCBasePtr(DAG);
    SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset());
    return getTOCEntry(DAG, DL, GA);
  }
 
  unsigned MOHiFlag, MOLoFlag;
  bool IsPIC = isPositionIndependent();
  getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag, GV);
 
  if (IsPIC && Subtarget.isSVR4ABI()) {
    SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT,
                                            GSDN->getOffset(),
                                            PPCII::MO_PIC_FLAG);
    return getTOCEntry(DAG, DL, GA);
  }
 
  SDValue GAHi =
    DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag);
  SDValue GALo =
    DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag);
 
  return LowerLabelRef(GAHi, GALo, IsPIC, DAG);
}
 
SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
  bool IsStrict = Op->isStrictFPOpcode();
  const SDNodeFlags Flags = Op.getNode()->getFlags();
  ISD::CondCode CC =
      cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get();
  SDValue LHS = Op.getOperand(IsStrict ? 1 : 0);
  SDValue RHS = Op.getOperand(IsStrict ? 2 : 1);
  SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
  EVT LHSVT = LHS.getValueType();
  SDLoc dl(Op);
 
  // Soften the setcc with libcall if it is fp128 or it is SPE and fp32/fp64.
  if (LHSVT == MVT::f128 ||
      (Subtarget.hasSPE() && (LHSVT == MVT::f32 || LHSVT == MVT::f64) &&
       (!Flags.hasNoNaNs() || !Flags.hasNoInfs()))) {
    assert(!Subtarget.hasP9Vector() &&
           "SETCC for f128 is already legal under Power9!");
    softenSetCCOperands(DAG, LHSVT, LHS, RHS, CC, dl, LHS, RHS, Chain,
                        Op->getOpcode() == ISD::STRICT_FSETCCS);
    if (RHS.getNode())
      LHS = DAG.getNode(ISD::SETCC, dl, Op.getValueType(), LHS, RHS,
                        DAG.getCondCode(CC));
    if (IsStrict)
      return DAG.getMergeValues({LHS, Chain}, dl);
    return LHS;
  } else if (LHSVT == MVT::f32 || LHSVT == MVT::f64) {
    return Op;
  }
 
  assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!");
 
  if (Op.getValueType() == MVT::v2i64) {
    // When the operands themselves are v2i64 values, we need to do something
    // special because VSX has no underlying comparison operations for these.
    if (LHS.getValueType() == MVT::v2i64) {
      // Equality can be handled by casting to the legal type for Altivec
      // comparisons, everything else needs to be expanded.
      if (CC != ISD::SETEQ && CC != ISD::SETNE)
        return SDValue();
      SDValue SetCC32 = DAG.getSetCC(
          dl, MVT::v4i32, DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, LHS),
          DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, RHS), CC);
      int ShuffV[] = {1, 0, 3, 2};
      SDValue Shuff =
          DAG.getVectorShuffle(MVT::v4i32, dl, SetCC32, SetCC32, ShuffV);
      return DAG.getBitcast(MVT::v2i64,
                            DAG.getNode(CC == ISD::SETEQ ? ISD::AND : ISD::OR,
                                        dl, MVT::v4i32, Shuff, SetCC32));
    }
 
    // We handle most of these in the usual way.
    return Op;
  }
 
  // If we're comparing for equality to zero, expose the fact that this is
  // implemented as a ctlz/srl pair on ppc, so that the dag combiner can
  // fold the new nodes.
  if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG))
    return V;
 
  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
    // Leave comparisons against 0 and -1 alone for now, since they're usually
    // optimized.  FIXME: revisit this when we can custom lower all setcc
    // optimizations.
    if (C->isAllOnes() || C->isZero())
      return SDValue();
  }
 
  // If we have an integer seteq/setne, turn it into a compare against zero
  // by xor'ing the rhs with the lhs, which is faster than setting a
  // condition register, reading it back out, and masking the correct bit.  The
  // normal approach here uses sub to do this instead of xor.  Using xor exposes
  // the result to other bit-twiddling opportunities.
  if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
    EVT VT = Op.getValueType();
    SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, LHS, RHS);
    return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC);
  }
  return SDValue();
}
 
SDValue PPCTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
  const SDNodeFlags Flags = Op->getFlags();
  SDValue Chain = Op.getOperand(0);
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
  SDValue LHS = Op.getOperand(2);
  SDValue RHS = Op.getOperand(3);
  SDValue Dest = Op.getOperand(4);
  EVT LHSVT = LHS.getValueType();
  SDLoc dl(Op);
 
  assert(Subtarget.hasSPE() && "LowerBR_CC used only for targets with SPE");
 
  if ((LHSVT == MVT::f32 || LHSVT == MVT::f64) && Flags.hasNoNaNs() &&
      Flags.hasNoInfs())
    return Op;
 
  softenSetCCOperands(DAG, LHSVT, LHS, RHS, CC, dl, LHS, RHS);
 
  // If softenSetCCOperands returned a scalar, we need to compare the result
  // against zero to select between true and false values.
  if (!RHS) {
    RHS = DAG.getConstant(0, dl, LHSVT);
    CC = ISD::SETNE;
  }
 
  return DAG.getNode(ISD::BR_CC, dl, Op.getValueType(), Chain,
                     DAG.getCondCode(CC), LHS, RHS, Dest);
}
 
SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
  SDNode *Node = Op.getNode();
  EVT VT = Node->getValueType(0);
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  SDValue InChain = Node->getOperand(0);
  SDValue VAListPtr = Node->getOperand(1);
  const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
  SDLoc dl(Node);
 
  assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only");
 
  // gpr_index
  SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
                                    VAListPtr, MachinePointerInfo(SV), MVT::i8);
  InChain = GprIndex.getValue(1);
 
  if (VT == MVT::i64) {
    // Check if GprIndex is even
    SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex,
                                 DAG.getConstant(1, dl, MVT::i32));
    SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd,
                                DAG.getConstant(0, dl, MVT::i32), ISD::SETNE);
    SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex,
                                          DAG.getConstant(1, dl, MVT::i32));
    // Align GprIndex to be even if it isn't
    GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne,
                           GprIndex);
  }
 
  // fpr index is 1 byte after gpr
  SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
                               DAG.getConstant(1, dl, MVT::i32));
 
  // fpr
  SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain,
                                    FprPtr, MachinePointerInfo(SV), MVT::i8);
  InChain = FprIndex.getValue(1);
 
  SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
                                       DAG.getConstant(8, dl, MVT::i32));
 
  SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr,
                                        DAG.getConstant(4, dl, MVT::i32));
 
  // areas
  SDValue OverflowArea =
      DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, MachinePointerInfo());
  InChain = OverflowArea.getValue(1);
 
  SDValue RegSaveArea =
      DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, MachinePointerInfo());
  InChain = RegSaveArea.getValue(1);
 
  // select overflow_area if index > 8
  SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex,
                            DAG.getConstant(8, dl, MVT::i32), ISD::SETLT);
 
  // adjustment constant gpr_index * 4/8
  SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32,
                                    VT.isInteger() ? GprIndex : FprIndex,
                                    DAG.getConstant(VT.isInteger() ? 4 : 8, dl,
                                                    MVT::i32));
 
  // OurReg = RegSaveArea + RegConstant
  SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea,
                               RegConstant);
 
  // Floating types are 32 bytes into RegSaveArea
  if (VT.isFloatingPoint())
    OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg,
                         DAG.getConstant(32, dl, MVT::i32));
 
  // increase {f,g}pr_index by 1 (or 2 if VT is i64)
  SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32,
                                   VT.isInteger() ? GprIndex : FprIndex,
                                   DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl,
                                                   MVT::i32));
 
  InChain = DAG.getTruncStore(InChain, dl, IndexPlus1,
                              VT.isInteger() ? VAListPtr : FprPtr,
                              MachinePointerInfo(SV), MVT::i8);
 
  // determine if we should load from reg_save_area or overflow_area
  SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea);
 
  // increase overflow_area by 4/8 if gpr/fpr > 8
  SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea,
                                          DAG.getConstant(VT.isInteger() ? 4 : 8,
                                          dl, MVT::i32));
 
  OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea,
                             OverflowAreaPlusN);
 
  InChain = DAG.getTruncStore(InChain, dl, OverflowArea, OverflowAreaPtr,
                              MachinePointerInfo(), MVT::i32);
 
  return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo());
}
 
SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
  assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only");
 
  // We have to copy the entire va_list struct:
  // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
  return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2),
                       DAG.getConstant(12, SDLoc(Op), MVT::i32), Align(8),
                       Align(8), false, true, /*CI=*/nullptr, std::nullopt,
                       MachinePointerInfo(), MachinePointerInfo());
}
 
SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
                                                  SelectionDAG &DAG) const {
  return Op.getOperand(0);
}
 
SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>();
 
  assert((Op.getOpcode() == ISD::INLINEASM ||
          Op.getOpcode() == ISD::INLINEASM_BR) &&
         "Expecting Inline ASM node.");
 
  // If an LR store is already known to be required then there is not point in
  // checking this ASM as well.
  if (MFI.isLRStoreRequired())
    return Op;
 
  // Inline ASM nodes have an optional last operand that is an incoming Flag of
  // type MVT::Glue. We want to ignore this last operand if that is the case.
  unsigned NumOps = Op.getNumOperands();
  if (Op.getOperand(NumOps - 1).getValueType() == MVT::Glue)
    --NumOps;
 
  // Check all operands that may contain the LR.
  for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
    const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
    unsigned NumVals = Flags.getNumOperandRegisters();
    ++i; // Skip the ID value.
 
    switch (Flags.getKind()) {
    default:
      llvm_unreachable("Bad flags!");
    case InlineAsm::Kind::RegUse:
    case InlineAsm::Kind::Imm:
    case InlineAsm::Kind::Mem:
      i += NumVals;
      break;
    case InlineAsm::Kind::Clobber:
    case InlineAsm::Kind::RegDef:
    case InlineAsm::Kind::RegDefEarlyClobber: {
      for (; NumVals; --NumVals, ++i) {
        Register Reg = cast<RegisterSDNode>(Op.getOperand(i))->getReg();
        if (Reg != PPC::LR && Reg != PPC::LR8)
          continue;
        MFI.setLRStoreRequired();
        return Op;
      }
      break;
    }
    }
  }
 
  return Op;
}
 
SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDValue Chain = Op.getOperand(0);
  SDValue Trmp = Op.getOperand(1); // trampoline
  SDValue FPtr = Op.getOperand(2); // nested function
  SDValue Nest = Op.getOperand(3); // 'nest' parameter value
  SDLoc dl(Op);
 
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
  if (Subtarget.isAIXABI()) {
    // On AIX we create a trampoline descriptor by combining the
    // entry point and TOC from the global descriptor (FPtr) with the
    // nest argument as the environment pointer.
    uint64_t PointerSize = Subtarget.isPPC64() ? 8 : 4;
    MaybeAlign PointerAlign(PointerSize);
    auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
                        ? (MachineMemOperand::MODereferenceable |
                           MachineMemOperand::MOInvariant)
                        : MachineMemOperand::MONone;
 
    uint64_t TOCPointerOffset = 1 * PointerSize;
    uint64_t EnvPointerOffset = 2 * PointerSize;
    SDValue SDTOCPtrOffset = DAG.getConstant(TOCPointerOffset, dl, PtrVT);
    SDValue SDEnvPtrOffset = DAG.getConstant(EnvPointerOffset, dl, PtrVT);
 
    const Value *TrampolineAddr =
        cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
    const Function *Func =
        cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
 
    SDValue OutChains[3];
 
    // Copy the entry point address from the global descriptor to the
    // trampoline buffer.
    SDValue LoadEntryPoint =
        DAG.getLoad(PtrVT, dl, Chain, FPtr, MachinePointerInfo(Func, 0),
                    PointerAlign, MMOFlags);
    SDValue EPLoadChain = LoadEntryPoint.getValue(1);
    OutChains[0] = DAG.getStore(EPLoadChain, dl, LoadEntryPoint, Trmp,
                                MachinePointerInfo(TrampolineAddr, 0));
 
    // Copy the TOC pointer from the global descriptor to the trampoline
    // buffer.
    SDValue TOCFromDescriptorPtr =
        DAG.getNode(ISD::ADD, dl, PtrVT, FPtr, SDTOCPtrOffset);
    SDValue TOCReg = DAG.getLoad(PtrVT, dl, Chain, TOCFromDescriptorPtr,
                                 MachinePointerInfo(Func, TOCPointerOffset),
                                 PointerAlign, MMOFlags);
    SDValue TrampolineTOCPointer =
        DAG.getNode(ISD::ADD, dl, PtrVT, Trmp, SDTOCPtrOffset);
    SDValue TOCLoadChain = TOCReg.getValue(1);
    OutChains[1] =
        DAG.getStore(TOCLoadChain, dl, TOCReg, TrampolineTOCPointer,
                     MachinePointerInfo(TrampolineAddr, TOCPointerOffset));
 
    // Store the nest argument into the environment pointer in the trampoline
    // buffer.
    SDValue EnvPointer = DAG.getNode(ISD::ADD, dl, PtrVT, Trmp, SDEnvPtrOffset);
    OutChains[2] =
        DAG.getStore(Chain, dl, Nest, EnvPointer,
                     MachinePointerInfo(TrampolineAddr, EnvPointerOffset));
 
    SDValue TokenFactor =
        DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    return TokenFactor;
  }
 
  bool isPPC64 = (PtrVT == MVT::i64);
  Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
 
  TargetLowering::ArgListTy Args;
  Args.emplace_back(Trmp, IntPtrTy);
  // TrampSize == (isPPC64 ? 48 : 40);
  Args.emplace_back(
      DAG.getConstant(isPPC64 ? 48 : 40, dl, Subtarget.getScalarIntVT()),
      IntPtrTy);
  Args.emplace_back(FPtr, IntPtrTy);
  Args.emplace_back(Nest, IntPtrTy);
 
  // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
  TargetLowering::CallLoweringInfo CLI(DAG);
  CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
      CallingConv::C, Type::getVoidTy(*DAG.getContext()),
      DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args));
 
  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
  return CallResult.second;
}
 
SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
  EVT PtrVT = getPointerTy(MF.getDataLayout());
 
  SDLoc dl(Op);
 
  if (Subtarget.isPPC64() || Subtarget.isAIXABI()) {
    // vastart just stores the address of the VarArgsFrameIndex slot into the
    // memory location argument.
    SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
    const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
    return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
                        MachinePointerInfo(SV));
  }
 
  // For the 32-bit SVR4 ABI we follow the layout of the va_list struct.
  // We suppose the given va_list is already allocated.
  //
  // typedef struct {
  //  char gpr;     /* index into the array of 8 GPRs
  //                 * stored in the register save area
  //                 * gpr=0 corresponds to r3,
  //                 * gpr=1 to r4, etc.
  //                 */
  //  char fpr;     /* index into the array of 8 FPRs
  //                 * stored in the register save area
  //                 * fpr=0 corresponds to f1,
  //                 * fpr=1 to f2, etc.
  //                 */
  //  char *overflow_arg_area;
  //                /* location on stack that holds
  //                 * the next overflow argument
  //                 */
  //  char *reg_save_area;
  //               /* where r3:r10 and f1:f8 (if saved)
  //                * are stored
  //                */
  // } va_list[1];
 
  SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32);
  SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32);
  SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(),
                                            PtrVT);
  SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
                                 PtrVT);
 
  uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
  SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT);
 
  uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
  SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT);
 
  uint64_t FPROffset = 1;
  SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT);
 
  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
 
  // Store first byte : number of int regs
  SDValue firstStore =
      DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, Op.getOperand(1),
                        MachinePointerInfo(SV), MVT::i8);
  uint64_t nextOffset = FPROffset;
  SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1),
                                  ConstFPROffset);
 
  // Store second byte : number of float regs
  SDValue secondStore =
      DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr,
                        MachinePointerInfo(SV, nextOffset), MVT::i8);
  nextOffset += StackOffset;
  nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset);
 
  // Store second word : arguments given on stack
  SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr,
                                    MachinePointerInfo(SV, nextOffset));
  nextOffset += FrameOffset;
  nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset);
 
  // Store third word : arguments given in registers
  return DAG.getStore(thirdStore, dl, FR, nextPtr,
                      MachinePointerInfo(SV, nextOffset));
}
 
/// FPR - The set of FP registers that should be allocated for arguments
/// on Darwin and AIX.
static const MCPhysReg FPR[] = {PPC::F1,  PPC::F2,  PPC::F3, PPC::F4, PPC::F5,
                                PPC::F6,  PPC::F7,  PPC::F8, PPC::F9, PPC::F10,
                                PPC::F11, PPC::F12, PPC::F13};
 
/// CalculateStackSlotSize - Calculates the size reserved for this argument on
/// the stack.
static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
                                       unsigned PtrByteSize) {
  unsigned ArgSize = ArgVT.getStoreSize();
  if (Flags.isByVal())
    ArgSize = Flags.getByValSize();
 
  // Round up to multiples of the pointer size, except for array members,
  // which are always packed.
  if (!Flags.isInConsecutiveRegs())
    ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
 
  return ArgSize;
}
 
/// CalculateStackSlotAlignment - Calculates the alignment of this argument
/// on the stack.
static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
                                         ISD::ArgFlagsTy Flags,
                                         unsigned PtrByteSize) {
  Align Alignment(PtrByteSize);
 
  // Altivec parameters are padded to a 16 byte boundary.
  if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
      ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
      ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
      ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
    Alignment = Align(16);
 
  // ByVal parameters are aligned as requested.
  if (Flags.isByVal()) {
    auto BVAlign = Flags.getNonZeroByValAlign();
    if (BVAlign > PtrByteSize) {
      if (BVAlign.value() % PtrByteSize != 0)
        llvm_unreachable(
            "ByVal alignment is not a multiple of the pointer size");
 
      Alignment = BVAlign;
    }
  }
 
  // Array members are always packed to their original alignment.
  if (Flags.isInConsecutiveRegs()) {
    // If the array member was split into multiple registers, the first
    // needs to be aligned to the size of the full type.  (Except for
    // ppcf128, which is only aligned as its f64 components.)
    if (Flags.isSplit() && OrigVT != MVT::ppcf128)
      Alignment = Align(OrigVT.getStoreSize());
    else
      Alignment = Align(ArgVT.getStoreSize());
  }
 
  return Alignment;
}
 
/// CalculateStackSlotUsed - Return whether this argument will use its
/// stack slot (instead of being passed in registers).  ArgOffset,
/// AvailableFPRs, and AvailableVRs must hold the current argument
/// position, and will be updated to account for this argument.
static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
                                   unsigned PtrByteSize, unsigned LinkageSize,
                                   unsigned ParamAreaSize, unsigned &ArgOffset,
                                   unsigned &AvailableFPRs,
                                   unsigned &AvailableVRs) {
  bool UseMemory = false;
 
  // Respect alignment of argument on the stack.
  Align Alignment =
      CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
  ArgOffset = alignTo(ArgOffset, Alignment);
  // If there's no space left in the argument save area, we must
  // use memory (this check also catches zero-sized arguments).
  if (ArgOffset >= LinkageSize + ParamAreaSize)
    UseMemory = true;
 
  // Allocate argument on the stack.
  ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
  if (Flags.isInConsecutiveRegsLast())
    ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
  // If we overran the argument save area, we must use memory
  // (this check catches arguments passed partially in memory)
  if (ArgOffset > LinkageSize + ParamAreaSize)
    UseMemory = true;
 
  // However, if the argument is actually passed in an FPR or a VR,
  // we don't use memory after all.
  if (!Flags.isByVal()) {
    if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
      if (AvailableFPRs > 0) {
        --AvailableFPRs;
        return false;
      }
    if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
        ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
        ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
        ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
      if (AvailableVRs > 0) {
        --AvailableVRs;
        return false;
      }
  }
 
  return UseMemory;
}
 
/// EnsureStackAlignment - Round stack frame size up from NumBytes to
/// ensure minimum alignment required for target.
static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
                                     unsigned NumBytes) {
  return alignTo(NumBytes, Lowering->getStackAlign());
}
 
SDValue PPCTargetLowering::LowerFormalArguments(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  if (Subtarget.isAIXABI())
    return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG,
                                    InVals);
  if (Subtarget.is64BitELFABI())
    return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
                                       InVals);
  assert(Subtarget.is32BitELFABI());
  return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
                                     InVals);
}
 
SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
 
  // 32-bit SVR4 ABI Stack Frame Layout:
  //              +-----------------------------------+
  //        +-->  |            Back chain             |
  //        |     +-----------------------------------+
  //        |     | Floating-point register save area |
  //        |     +-----------------------------------+
  //        |     |    General register save area     |
  //        |     +-----------------------------------+
  //        |     |          CR save word             |
  //        |     +-----------------------------------+
  //        |     |         VRSAVE save word          |
  //        |     +-----------------------------------+
  //        |     |         Alignment padding         |
  //        |     +-----------------------------------+
  //        |     |     Vector register save area     |
  //        |     +-----------------------------------+
  //        |     |       Local variable space        |
  //        |     +-----------------------------------+
  //        |     |        Parameter list area        |
  //        |     +-----------------------------------+
  //        |     |           LR save word            |
  //        |     +-----------------------------------+
  // SP-->  +---  |            Back chain             |
  //              +-----------------------------------+
  //
  // Specifications:
  //   System V Application Binary Interface PowerPC Processor Supplement
  //   AltiVec Technology Programming Interface Manual
 
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
 
  EVT PtrVT = getPointerTy(MF.getDataLayout());
  // Potential tail calls could cause overwriting of argument stack slots.
  bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
                       (CallConv == CallingConv::Fast));
  const Align PtrAlign(4);
 
  // Assign locations to all of the incoming arguments.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                 *DAG.getContext());
 
  // Reserve space for the linkage area on the stack.
  unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
  CCInfo.AllocateStack(LinkageSize, PtrAlign);
  CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4);
 
  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
    CCValAssign &VA = ArgLocs[i];
 
    // Arguments stored in registers.
    if (VA.isRegLoc()) {
      const TargetRegisterClass *RC;
      EVT ValVT = VA.getValVT();
 
      switch (ValVT.getSimpleVT().SimpleTy) {
        default:
          llvm_unreachable("ValVT not supported by formal arguments Lowering");
        case MVT::i1:
        case MVT::i32:
          RC = &PPC::GPRCRegClass;
          break;
        case MVT::f32:
          if (Subtarget.hasP8Vector())
            RC = &PPC::VSSRCRegClass;
          else if (Subtarget.hasSPE())
            RC = &PPC::GPRCRegClass;
          else
            RC = &PPC::F4RCRegClass;
          break;
        case MVT::f64:
          if (Subtarget.hasVSX())
            RC = &PPC::VSFRCRegClass;
          else if (Subtarget.hasSPE())
            // SPE passes doubles in GPR pairs.
            RC = &PPC::GPRCRegClass;
          else
            RC = &PPC::F8RCRegClass;
          break;
        case MVT::v16i8:
        case MVT::v8i16:
        case MVT::v4i32:
          RC = &PPC::VRRCRegClass;
          break;
        case MVT::v4f32:
          RC = &PPC::VRRCRegClass;
          break;
        case MVT::v2f64:
        case MVT::v2i64:
          RC = &PPC::VRRCRegClass;
          break;
      }
 
      SDValue ArgValue;
      // Transform the arguments stored in physical registers into
      // virtual ones.
      if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) {
        assert(i + 1 < e && "No second half of double precision argument");
        Register RegLo = MF.addLiveIn(VA.getLocReg(), RC);
        Register RegHi = MF.addLiveIn(ArgLocs[++i].getLocReg(), RC);
        SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32);
        SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32);
        if (!Subtarget.isLittleEndian())
          std::swap (ArgValueLo, ArgValueHi);
        ArgValue = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, ArgValueLo,
                               ArgValueHi);
      } else {
        Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
        ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
                                      ValVT == MVT::i1 ? MVT::i32 : ValVT);
        if (ValVT == MVT::i1)
          ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue);
      }
 
      InVals.push_back(ArgValue);
    } else {
      // Argument stored in memory.
      assert(VA.isMemLoc());
 
      // Get the extended size of the argument type in stack
      unsigned ArgSize = VA.getLocVT().getStoreSize();
      // Get the actual size of the argument type
      unsigned ObjSize = VA.getValVT().getStoreSize();
      unsigned ArgOffset = VA.getLocMemOffset();
      // Stack objects in PPC32 are right justified.
      ArgOffset += ArgSize - ObjSize;
      int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, isImmutable);
 
      // Create load nodes to retrieve arguments from the stack.
      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
      InVals.push_back(
          DAG.getLoad(VA.getValVT(), dl, Chain, FIN, MachinePointerInfo()));
    }
  }
 
  // Assign locations to all of the incoming aggregate by value arguments.
  // Aggregates passed by value are stored in the local variable space of the
  // caller's stack frame, right above the parameter list area.
  SmallVector<CCValAssign, 16> ByValArgLocs;
  CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
                      ByValArgLocs, *DAG.getContext());
 
  // Reserve stack space for the allocations in CCInfo.
  CCByValInfo.AllocateStack(CCInfo.getStackSize(), PtrAlign);
 
  CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal);
 
  // Area that is at least reserved in the caller of this function.
  unsigned MinReservedArea = CCByValInfo.getStackSize();
  MinReservedArea = std::max(MinReservedArea, LinkageSize);
 
  // Set the size that is at least reserved in caller of this function.  Tail
  // call optimized function's reserved stack space needs to be aligned so that
  // taking the difference between two stack areas will result in an aligned
  // stack.
  MinReservedArea =
      EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea);
  FuncInfo->setMinReservedArea(MinReservedArea);
 
  SmallVector<SDValue, 8> MemOps;
 
  // If the function takes variable number of arguments, make a frame index for
  // the start of the first vararg value... for expansion of llvm.va_start.
  if (isVarArg) {
    static const MCPhysReg GPArgRegs[] = {
      PPC::R3, PPC::R4, PPC::R5, PPC::R6,
      PPC::R7, PPC::R8, PPC::R9, PPC::R10,
    };
    const unsigned NumGPArgRegs = std::size(GPArgRegs);
 
    static const MCPhysReg FPArgRegs[] = {
      PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
      PPC::F8
    };
    unsigned NumFPArgRegs = std::size(FPArgRegs);
 
    if (useSoftFloat() || hasSPE())
       NumFPArgRegs = 0;
 
    FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs));
    FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs));
 
    // Make room for NumGPArgRegs and NumFPArgRegs.
    int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 +
                NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8;
 
    FuncInfo->setVarArgsStackOffset(MFI.CreateFixedObject(
        PtrVT.getSizeInBits() / 8, CCInfo.getStackSize(), true));
 
    FuncInfo->setVarArgsFrameIndex(
        MFI.CreateStackObject(Depth, Align(8), false));
    SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
 
    // The fixed integer arguments of a variadic function are stored to the
    // VarArgsFrameIndex on the stack so that they may be loaded by
    // dereferencing the result of va_next.
    for (MCPhysReg GPArgReg : GPArgRegs) {
      // Get an existing live-in vreg, or add a new one.
      Register VReg = MF.getRegInfo().getLiveInVirtReg(GPArgReg);
      if (!VReg)
        VReg = MF.addLiveIn(GPArgReg, &PPC::GPRCRegClass);
 
      SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
      SDValue Store =
          DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo());
      MemOps.push_back(Store);
      // Increment the address by four for the next argument to store
      SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT);
      FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
    }
 
    // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6
    // is set.
    // The double arguments are stored to the VarArgsFrameIndex
    // on the stack.
    for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) {
      // Get an existing live-in vreg, or add a new one.
      Register VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]);
      if (!VReg)
        VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass);
 
      SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64);
      SDValue Store =
          DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo());
      MemOps.push_back(Store);
      // Increment the address by eight for the next argument to store
      SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl,
                                         PtrVT);
      FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
    }
  }
 
  if (!MemOps.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
 
  return Chain;
}
 
// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
// value to MVT::i64 and then truncate to the correct register size.
SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags,
                                             EVT ObjectVT, SelectionDAG &DAG,
                                             SDValue ArgVal,
                                             const SDLoc &dl) const {
  if (Flags.isSExt())
    ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal,
                         DAG.getValueType(ObjectVT));
  else if (Flags.isZExt())
    ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal,
                         DAG.getValueType(ObjectVT));
 
  return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal);
}
 
SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  // TODO: add description of PPC stack frame format, or at least some docs.
  //
  bool isELFv2ABI = Subtarget.isELFv2ABI();
  bool isLittleEndian = Subtarget.isLittleEndian();
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
 
  assert(!(CallConv == CallingConv::Fast && isVarArg) &&
         "fastcc not supported on varargs functions");
 
  EVT PtrVT = getPointerTy(MF.getDataLayout());
  // Potential tail calls could cause overwriting of argument stack slots.
  bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
                       (CallConv == CallingConv::Fast));
  unsigned PtrByteSize = 8;
  unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
 
  static const MCPhysReg GPR[] = {
    PPC::X3, PPC::X4, PPC::X5, PPC::X6,
    PPC::X7, PPC::X8, PPC::X9, PPC::X10,
  };
  static const MCPhysReg VR[] = {
    PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
    PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
  };
 
  const unsigned Num_GPR_Regs = std::size(GPR);
  const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
  const unsigned Num_VR_Regs = std::size(VR);
 
  // Do a first pass over the arguments to determine whether the ABI
  // guarantees that our caller has allocated the parameter save area
  // on its stack frame.  In the ELFv1 ABI, this is always the case;
  // in the ELFv2 ABI, it is true if this is a vararg function or if
  // any parameter is located in a stack slot.
 
  bool HasParameterArea = !isELFv2ABI || isVarArg;
  unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize;
  unsigned NumBytes = LinkageSize;
  unsigned AvailableFPRs = Num_FPR_Regs;
  unsigned AvailableVRs = Num_VR_Regs;
  for (const ISD::InputArg &In : Ins) {
    if (In.Flags.isNest())
      continue;
 
    if (CalculateStackSlotUsed(In.VT, In.ArgVT, In.Flags, PtrByteSize,
                               LinkageSize, ParamAreaSize, NumBytes,
                               AvailableFPRs, AvailableVRs))
      HasParameterArea = true;
  }
 
  // Add DAG nodes to load the arguments or copy them out of registers.  On
  // entry to a function on PPC, the arguments start after the linkage area,
  // although the first ones are often in registers.
 
  unsigned ArgOffset = LinkageSize;
  unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
  SmallVector<SDValue, 8> MemOps;
  Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
  unsigned CurArgIdx = 0;
  for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
    SDValue ArgVal;
    bool needsLoad = false;
    EVT ObjectVT = Ins[ArgNo].VT;
    EVT OrigVT = Ins[ArgNo].ArgVT;
    unsigned ObjSize = ObjectVT.getStoreSize();
    unsigned ArgSize = ObjSize;
    ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
    if (Ins[ArgNo].isOrigArg()) {
      std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
      CurArgIdx = Ins[ArgNo].getOrigArgIndex();
    }
    // We re-align the argument offset for each argument, except when using the
    // fast calling convention, when we need to make sure we do that only when
    // we'll actually use a stack slot.
    unsigned CurArgOffset;
    Align Alignment;
    auto ComputeArgOffset = [&]() {
      /* Respect alignment of argument on the stack.  */
      Alignment =
          CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize);
      ArgOffset = alignTo(ArgOffset, Alignment);
      CurArgOffset = ArgOffset;
    };
 
    if (CallConv != CallingConv::Fast) {
      ComputeArgOffset();
 
      /* Compute GPR index associated with argument offset.  */
      GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
      GPR_idx = std::min(GPR_idx, Num_GPR_Regs);
    }
 
    // FIXME the codegen can be much improved in some cases.
    // We do not have to keep everything in memory.
    if (Flags.isByVal()) {
      assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit");
 
      if (CallConv == CallingConv::Fast)
        ComputeArgOffset();
 
      // ObjSize is the true size, ArgSize rounded up to multiple of registers.
      ObjSize = Flags.getByValSize();
      ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
      // Empty aggregate parameters do not take up registers.  Examples:
      //   struct { } a;
      //   union  { } b;
      //   int c[0];
      // etc.  However, we have to provide a place-holder in InVals, so
      // pretend we have an 8-byte item at the current address for that
      // purpose.
      if (!ObjSize) {
        int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true);
        SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
        InVals.push_back(FIN);
        continue;
      }
 
      // Create a stack object covering all stack doublewords occupied
      // by the argument.  If the argument is (fully or partially) on
      // the stack, or if the argument is fully in registers but the
      // caller has allocated the parameter save anyway, we can refer
      // directly to the caller's stack frame.  Otherwise, create a
      // local copy in our own frame.
      int FI;
      if (HasParameterArea ||
          ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize)
        FI = MFI.CreateFixedObject(ArgSize, ArgOffset, false, true);
      else
        FI = MFI.CreateStackObject(ArgSize, Alignment, false);
      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
 
      // Handle aggregates smaller than 8 bytes.
      if (ObjSize < PtrByteSize) {
        // The value of the object is its address, which differs from the
        // address of the enclosing doubleword on big-endian systems.
        SDValue Arg = FIN;
        if (!isLittleEndian) {
          SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT);
          Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff);
        }
        InVals.push_back(Arg);
 
        if (GPR_idx != Num_GPR_Regs) {
          Register VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
          FuncInfo->addLiveInAttr(VReg, Flags);
          SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
          EVT ObjType = EVT::getIntegerVT(*DAG.getContext(), ObjSize * 8);
          SDValue Store =
              DAG.getTruncStore(Val.getValue(1), dl, Val, Arg,
                                MachinePointerInfo(&*FuncArg), ObjType);
          MemOps.push_back(Store);
        }
        // Whether we copied from a register or not, advance the offset
        // into the parameter save area by a full doubleword.
        ArgOffset += PtrByteSize;
        continue;
      }
 
      // The value of the object is its address, which is the address of
      // its first stack doubleword.
      InVals.push_back(FIN);
 
      // Store whatever pieces of the object are in registers to memory.
      for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
        if (GPR_idx == Num_GPR_Regs)
          break;
 
        Register VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
        FuncInfo->addLiveInAttr(VReg, Flags);
        SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
        SDValue Addr = FIN;
        if (j) {
          SDValue Off = DAG.getConstant(j, dl, PtrVT);
          Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off);
        }
        unsigned StoreSizeInBits = std::min(PtrByteSize, (ObjSize - j)) * 8;
        EVT ObjType = EVT::getIntegerVT(*DAG.getContext(), StoreSizeInBits);
        SDValue Store =
            DAG.getTruncStore(Val.getValue(1), dl, Val, Addr,
                              MachinePointerInfo(&*FuncArg, j), ObjType);
        MemOps.push_back(Store);
        ++GPR_idx;
      }
      ArgOffset += ArgSize;
      continue;
    }
 
    switch (ObjectVT.getSimpleVT().SimpleTy) {
    default: llvm_unreachable("Unhandled argument type!");
    case MVT::i1:
    case MVT::i32:
    case MVT::i64:
      if (Flags.isNest()) {
        // The 'nest' parameter, if any, is passed in R11.
        Register VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass);
        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
 
        if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
          ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
 
        break;
      }
 
      // These can be scalar arguments or elements of an integer array type
      // passed directly.  Clang may use those instead of "byval" aggregate
      // types to avoid forcing arguments to memory unnecessarily.
      if (GPR_idx != Num_GPR_Regs) {
        Register VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
        FuncInfo->addLiveInAttr(VReg, Flags);
        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
 
        if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
          // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
          // value to MVT::i64 and then truncate to the correct register size.
          ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
      } else {
        if (CallConv == CallingConv::Fast)
          ComputeArgOffset();
 
        needsLoad = true;
        ArgSize = PtrByteSize;
      }
      if (CallConv != CallingConv::Fast || needsLoad)
        ArgOffset += 8;
      break;
 
    case MVT::f32:
    case MVT::f64:
      // These can be scalar arguments or elements of a float array type
      // passed directly.  The latter are used to implement ELFv2 homogenous
      // float aggregates.
      if (FPR_idx != Num_FPR_Regs) {
        unsigned VReg;
 
        if (ObjectVT == MVT::f32)
          VReg = MF.addLiveIn(FPR[FPR_idx],
                              Subtarget.hasP8Vector()
                                  ? &PPC::VSSRCRegClass
                                  : &PPC::F4RCRegClass);
        else
          VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX()
                                                ? &PPC::VSFRCRegClass
                                                : &PPC::F8RCRegClass);
 
        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
        ++FPR_idx;
      } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) {
        // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
        // once we support fp <-> gpr moves.
 
        // This can only ever happen in the presence of f32 array types,
        // since otherwise we never run out of FPRs before running out
        // of GPRs.
        Register VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass);
        FuncInfo->addLiveInAttr(VReg, Flags);
        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
 
        if (ObjectVT == MVT::f32) {
          if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
            ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal,
                                 DAG.getConstant(32, dl, MVT::i32));
          ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal);
        }
 
        ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal);
      } else {
        if (CallConv == CallingConv::Fast)
          ComputeArgOffset();
 
        needsLoad = true;
      }
 
      // When passing an array of floats, the array occupies consecutive
      // space in the argument area; only round up to the next doubleword
      // at the end of the array.  Otherwise, each float takes 8 bytes.
      if (CallConv != CallingConv::Fast || needsLoad) {
        ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize;
        ArgOffset += ArgSize;
        if (Flags.isInConsecutiveRegsLast())
          ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
      }
      break;
    case MVT::v4f32:
    case MVT::v4i32:
    case MVT::v8i16:
    case MVT::v16i8:
    case MVT::v2f64:
    case MVT::v2i64:
    case MVT::v1i128:
    case MVT::f128:
      // These can be scalar arguments or elements of a vector array type
      // passed directly.  The latter are used to implement ELFv2 homogenous
      // vector aggregates.
      if (VR_idx != Num_VR_Regs) {
        Register VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
        ++VR_idx;
      } else {
        if (CallConv == CallingConv::Fast)
          ComputeArgOffset();
        needsLoad = true;
      }
      if (CallConv != CallingConv::Fast || needsLoad)
        ArgOffset += 16;
      break;
    }
 
    // We need to load the argument to a virtual register if we determined
    // above that we ran out of physical registers of the appropriate type.
    if (needsLoad) {
      if (ObjSize < ArgSize && !isLittleEndian)
        CurArgOffset += ArgSize - ObjSize;
      int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, isImmutable);
      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
      ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo());
    }
 
    InVals.push_back(ArgVal);
  }
 
  // Area that is at least reserved in the caller of this function.
  unsigned MinReservedArea;
  if (HasParameterArea)
    MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize);
  else
    MinReservedArea = LinkageSize;
 
  // Set the size that is at least reserved in caller of this function.  Tail
  // call optimized functions' reserved stack space needs to be aligned so that
  // taking the difference between two stack areas will result in an aligned
  // stack.
  MinReservedArea =
      EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea);
  FuncInfo->setMinReservedArea(MinReservedArea);
 
  // If the function takes variable number of arguments, make a frame index for
  // the start of the first vararg value... for expansion of llvm.va_start.
  // On ELFv2ABI spec, it writes:
  // C programs that are intended to be *portable* across different compilers
  // and architectures must use the header file <stdarg.h> to deal with variable
  // argument lists.
  if (isVarArg && MFI.hasVAStart()) {
    int Depth = ArgOffset;
 
    FuncInfo->setVarArgsFrameIndex(
      MFI.CreateFixedObject(PtrByteSize, Depth, true));
    SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
 
    // If this function is vararg, store any remaining integer argument regs
    // to their spots on the stack so that they may be loaded by dereferencing
    // the result of va_next.
    for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
         GPR_idx < Num_GPR_Regs; ++GPR_idx) {
      Register VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
      SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
      SDValue Store =
          DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo());
      MemOps.push_back(Store);
      // Increment the address by four for the next argument to store
      SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT);
      FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
    }
  }
 
  if (!MemOps.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
 
  return Chain;
}
 
/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be
/// adjusted to accommodate the arguments for the tailcall.
static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
                                   unsigned ParamSize) {
 
  if (!isTailCall) return 0;
 
  PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
  unsigned CallerMinReservedArea = FI->getMinReservedArea();
  int SPDiff = (int)CallerMinReservedArea - (int)ParamSize;
  // Remember only if the new adjustment is bigger.
  if (SPDiff < FI->getTailCallSPDelta())
    FI->setTailCallSPDelta(SPDiff);
 
  return SPDiff;
}
 
static bool isFunctionGlobalAddress(const GlobalValue *CalleeGV);
 
static bool callsShareTOCBase(const Function *Caller,
                              const GlobalValue *CalleeGV,
                              const TargetMachine &TM) {
  // It does not make sense to call callsShareTOCBase() with a caller that
  // is PC Relative since PC Relative callers do not have a TOC.
#ifndef NDEBUG
  const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller);
  assert(!STICaller->isUsingPCRelativeCalls() &&
         "PC Relative callers do not have a TOC and cannot share a TOC Base");
#endif
 
  // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
  // don't have enough information to determine if the caller and callee share
  // the same  TOC base, so we have to pessimistically assume they don't for
  // correctness.
  if (!CalleeGV)
    return false;
 
  // If the callee is preemptable, then the static linker will use a plt-stub
  // which saves the toc to the stack, and needs a nop after the call
  // instruction to convert to a toc-restore.
  if (!TM.shouldAssumeDSOLocal(CalleeGV))
    return false;
 
  // Functions with PC Relative enabled may clobber the TOC in the same DSO.
  // We may need a TOC restore in the situation where the caller requires a
  // valid TOC but the callee is PC Relative and does not.
  const Function *F = dyn_cast<Function>(CalleeGV);
  const GlobalAlias *Alias = dyn_cast<GlobalAlias>(CalleeGV);
 
  // If we have an Alias we can try to get the function from there.
  if (Alias) {
    const GlobalObject *GlobalObj = Alias->getAliaseeObject();
    F = dyn_cast<Function>(GlobalObj);
  }
 
  // If we still have no valid function pointer we do not have enough
  // information to determine if the callee uses PC Relative calls so we must
  // assume that it does.
  if (!F)
    return false;
 
  // If the callee uses PC Relative we cannot guarantee that the callee won't
  // clobber the TOC of the caller and so we must assume that the two
  // functions do not share a TOC base.
  const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(*F);
  if (STICallee->isUsingPCRelativeCalls())
    return false;
 
  // If the GV is not a strong definition then we need to assume it can be
  // replaced by another function at link time. The function that replaces
  // it may not share the same TOC as the caller since the callee may be
  // replaced by a PC Relative version of the same function.
  if (!CalleeGV->isStrongDefinitionForLinker())
    return false;
 
  // The medium and large code models are expected to provide a sufficiently
  // large TOC to provide all data addressing needs of a module with a
  // single TOC.
  if (CodeModel::Medium == TM.getCodeModel() ||
      CodeModel::Large == TM.getCodeModel())
    return true;
 
  // Any explicitly-specified sections and section prefixes must also match.
  // Also, if we're using -ffunction-sections, then each function is always in
  // a different section (the same is true for COMDAT functions).
  if (TM.getFunctionSections() || CalleeGV->hasComdat() ||
      Caller->hasComdat() || CalleeGV->getSection() != Caller->getSection())
    return false;
  if (const auto *F = dyn_cast<Function>(CalleeGV)) {
    if (F->getSectionPrefix() != Caller->getSectionPrefix())
      return false;
  }
 
  return true;
}
 
static bool
needStackSlotPassParameters(const PPCSubtarget &Subtarget,
                            const SmallVectorImpl<ISD::OutputArg> &Outs) {
  assert(Subtarget.is64BitELFABI());
 
  const unsigned PtrByteSize = 8;
  const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
 
  static const MCPhysReg GPR[] = {
    PPC::X3, PPC::X4, PPC::X5, PPC::X6,
    PPC::X7, PPC::X8, PPC::X9, PPC::X10,
  };
  static const MCPhysReg VR[] = {
    PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
    PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
  };
 
  const unsigned NumGPRs = std::size(GPR);
  const unsigned NumFPRs = 13;
  const unsigned NumVRs = std::size(VR);
  const unsigned ParamAreaSize = NumGPRs * PtrByteSize;
 
  unsigned NumBytes = LinkageSize;
  unsigned AvailableFPRs = NumFPRs;
  unsigned AvailableVRs = NumVRs;
 
  for (const ISD::OutputArg& Param : Outs) {
    if (Param.Flags.isNest()) continue;
 
    if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize,
                               LinkageSize, ParamAreaSize, NumBytes,
                               AvailableFPRs, AvailableVRs))
      return true;
  }
  return false;
}
 
static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) {
  if (CB.arg_size() != CallerFn->arg_size())
    return false;
 
  auto CalleeArgIter = CB.arg_begin();
  auto CalleeArgEnd = CB.arg_end();
  Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin();
 
  for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) {
    const Value* CalleeArg = *CalleeArgIter;
    const Value* CallerArg = &(*CallerArgIter);
    if (CalleeArg == CallerArg)
      continue;
 
    // e.g. @caller([4 x i64] %a, [4 x i64] %b) {
    //        tail call @callee([4 x i64] undef, [4 x i64] %b)
    //      }
    // 1st argument of callee is undef and has the same type as caller.
    if (CalleeArg->getType() == CallerArg->getType() &&
        isa<UndefValue>(CalleeArg))
      continue;
 
    return false;
  }
 
  return true;
}
 
// Returns true if TCO is possible between the callers and callees
// calling conventions.
static bool
areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC,
                                    CallingConv::ID CalleeCC) {
  // Tail calls are possible with fastcc and ccc.
  auto isTailCallableCC  = [] (CallingConv::ID CC){
      return  CC == CallingConv::C || CC == CallingConv::Fast;
  };
  if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC))
    return false;
 
  // We can safely tail call both fastcc and ccc callees from a c calling
  // convention caller. If the caller is fastcc, we may have less stack space
  // than a non-fastcc caller with the same signature so disable tail-calls in
  // that case.
  return CallerCC == CallingConv::C || CallerCC == CalleeCC;
}
 
bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
    const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
    CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
    bool isCalleeExternalSymbol) const {
  bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
 
  if (DisableSCO && !TailCallOpt) return false;
 
  // Variadic argument functions are not supported.
  if (isVarArg) return false;
 
  // Check that the calling conventions are compatible for tco.
  if (!areCallingConvEligibleForTCO_64SVR4(CallerCC, CalleeCC))
    return false;
 
  // Caller contains any byval parameter is not supported.
  if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
    return false;
 
  // Callee contains any byval parameter is not supported, too.
  // Note: This is a quick work around, because in some cases, e.g.
  // caller's stack size > callee's stack size, we are still able to apply
  // sibling call optimization. For example, gcc is able to do SCO for caller1
  // in the following example, but not for caller2.
  //   struct test {
  //     long int a;
  //     char ary[56];
  //   } gTest;
  //   __attribute__((noinline)) int callee(struct test v, struct test *b) {
  //     b->a = v.a;
  //     return 0;
  //   }
  //   void caller1(struct test a, struct test c, struct test *b) {
  //     callee(gTest, b); }
  //   void caller2(struct test *b) { callee(gTest, b); }
  if (any_of(Outs, [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); }))
    return false;
 
  // If callee and caller use different calling conventions, we cannot pass
  // parameters on stack since offsets for the parameter area may be different.
  if (CallerCC != CalleeCC && needStackSlotPassParameters(Subtarget, Outs))
    return false;
 
  // All variants of 64-bit ELF ABIs without PC-Relative addressing require that
  // the caller and callee share the same TOC for TCO/SCO. If the caller and
  // callee potentially have different TOC bases then we cannot tail call since
  // we need to restore the TOC pointer after the call.
  // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977
  // We cannot guarantee this for indirect calls or calls to external functions.
  // When PC-Relative addressing is used, the concept of the TOC is no longer
  // applicable so this check is not required.
  // Check first for indirect calls.
  if (!Subtarget.isUsingPCRelativeCalls() &&
      !isFunctionGlobalAddress(CalleeGV) && !isCalleeExternalSymbol)
    return false;
 
  // Check if we share the TOC base.
  if (!Subtarget.isUsingPCRelativeCalls() &&
      !callsShareTOCBase(CallerFunc, CalleeGV, getTargetMachine()))
    return false;
 
  // TCO allows altering callee ABI, so we don't have to check further.
  if (CalleeCC == CallingConv::Fast && TailCallOpt)
    return true;
 
  if (DisableSCO) return false;
 
  // If callee use the same argument list that caller is using, then we can
  // apply SCO on this case. If it is not, then we need to check if callee needs
  // stack for passing arguments.
  // PC Relative tail calls may not have a CallBase.
  // If there is no CallBase we cannot verify if we have the same argument
  // list so assume that we don't have the same argument list.
  if (CB && !hasSameArgumentList(CallerFunc, *CB) &&
      needStackSlotPassParameters(Subtarget, Outs))
    return false;
  else if (!CB && needStackSlotPassParameters(Subtarget, Outs))
    return false;
 
  return true;
}
 
/// IsEligibleForTailCallOptimization - Check whether the call is eligible
/// for tail call optimization. Targets which want to do tail call
/// optimization should implement this function.
bool PPCTargetLowering::IsEligibleForTailCallOptimization(
    const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
    CallingConv::ID CallerCC, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins) const {
  if (!getTargetMachine().Options.GuaranteedTailCallOpt)
    return false;
 
  // Variable argument functions are not supported.
  if (isVarArg)
    return false;
 
  if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) {
    // Functions containing by val parameters are not supported.
    if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
      return false;
 
    // Non-PIC/GOT tail calls are supported.
    if (getTargetMachine().getRelocationModel() != Reloc::PIC_)
      return true;
 
    // At the moment we can only do local tail calls (in same module, hidden
    // or protected) if we are generating PIC.
    if (CalleeGV)
      return CalleeGV->hasHiddenVisibility() ||
             CalleeGV->hasProtectedVisibility();
  }
 
  return false;
}
 
/// isCallCompatibleAddress - Return the immediate to use if the specified
/// 32-bit value is representable in the immediate field of a BxA instruction.
static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
  ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
  if (!C) return nullptr;
 
  int Addr = C->getZExtValue();
  if ((Addr & 3) != 0 ||  // Low 2 bits are implicitly zero.
      SignExtend32<26>(Addr) != Addr)
    return nullptr;  // Top 6 bits have to be sext of immediate.
 
  return DAG
      .getSignedConstant(
          (int)C->getZExtValue() >> 2, SDLoc(Op),
          DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()))
      .getNode();
}
 
namespace {
 
struct TailCallArgumentInfo {
  SDValue Arg;
  SDValue FrameIdxOp;
  int FrameIdx = 0;
 
  TailCallArgumentInfo() = default;
};
 
} // end anonymous namespace
 
/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot.
static void StoreTailCallArgumentsToStackSlot(
    SelectionDAG &DAG, SDValue Chain,
    const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs,
    SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) {
  for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) {
    SDValue Arg = TailCallArgs[i].Arg;
    SDValue FIN = TailCallArgs[i].FrameIdxOp;
    int FI = TailCallArgs[i].FrameIdx;
    // Store relative to framepointer.
    MemOpChains.push_back(DAG.getStore(
        Chain, dl, Arg, FIN,
        MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)));
  }
}
 
/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to
/// the appropriate stack slot for the tail call optimized function call.
static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
                                             SDValue OldRetAddr, SDValue OldFP,
                                             int SPDiff, const SDLoc &dl) {
  if (SPDiff) {
    // Calculate the new stack slot for the return address.
    MachineFunction &MF = DAG.getMachineFunction();
    const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
    const PPCFrameLowering *FL = Subtarget.getFrameLowering();
    int SlotSize = Subtarget.isPPC64() ? 8 : 4;
    int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset();
    int NewRetAddr = MF.getFrameInfo().CreateFixedObject(SlotSize,
                                                         NewRetAddrLoc, true);
    SDValue NewRetAddrFrIdx =
        DAG.getFrameIndex(NewRetAddr, Subtarget.getScalarIntVT());
    Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx,
                         MachinePointerInfo::getFixedStack(MF, NewRetAddr));
  }
  return Chain;
}
 
/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate
/// the position of the argument.
static void CalculateTailCallArgDest(
    SelectionDAG &DAG, MachineFunction &MF, bool IsPPC64, SDValue Arg,
    int SPDiff, unsigned ArgOffset,
    SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
  int Offset = ArgOffset + SPDiff;
  uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8;
  int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
  EVT VT = IsPPC64 ? MVT::i64 : MVT::i32;
  SDValue FIN = DAG.getFrameIndex(FI, VT);
  TailCallArgumentInfo Info;
  Info.Arg = Arg;
  Info.FrameIdxOp = FIN;
  Info.FrameIdx = FI;
  TailCallArguments.push_back(Info);
}
 
/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address
/// stack slot. Returns the chain as result and the loaded frame pointers in
/// LROpOut/FPOpout. Used when tail calling.
SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(
    SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
    SDValue &FPOpOut, const SDLoc &dl) const {
  if (SPDiff) {
    // Load the LR and FP stack slot for later adjusting.
    LROpOut = getReturnAddrFrameIndex(DAG);
    LROpOut = DAG.getLoad(Subtarget.getScalarIntVT(), dl, Chain, LROpOut,
                          MachinePointerInfo());
    Chain = SDValue(LROpOut.getNode(), 1);
  }
  return Chain;
}
 
/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
/// by "Src" to address "Dst" of size "Size".  Alignment information is
/// specified by the specific parameter attribute. The copy will be passed as
/// a byval function parameter.
/// Sometimes what we are copying is the end of a larger object, the part that
/// does not fit in registers.
static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
                                         SDValue Chain, ISD::ArgFlagsTy Flags,
                                         SelectionDAG &DAG, const SDLoc &dl) {
  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
  Align Alignment = Flags.getNonZeroByValAlign();
  return DAG.getMemcpy(
      Chain, dl, Dst, Src, SizeNode, Alignment, Alignment, false, false,
      /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());
}
 
/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
/// tail calls.
static void LowerMemOpCallTo(
    SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
    SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64,
    bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
    SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) {
  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
  if (!isTailCall) {
    if (isVector) {
      SDValue StackPtr;
      if (isPPC64)
        StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
      else
        StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
      PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
                           DAG.getConstant(ArgOffset, dl, PtrVT));
    }
    MemOpChains.push_back(
        DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()));
    // Calculate and remember argument location.
  } else
    CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset,
                             TailCallArguments);
}
 
static void
PrepareTailCall(SelectionDAG &DAG, SDValue &InGlue, SDValue &Chain,
                const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp,
                SDValue FPOp,
                SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
  // Emit a sequence of copyto/copyfrom virtual registers for arguments that
  // might overwrite each other in case of tail call optimization.
  SmallVector<SDValue, 8> MemOpChains2;
  // Do not flag preceding copytoreg stuff together with the following stuff.
  InGlue = SDValue();
  StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments,
                                    MemOpChains2, dl);
  if (!MemOpChains2.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);
 
  // Store the return address to the appropriate stack slot.
  Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, LROp, FPOp, SPDiff, dl);
 
  // Emit callseq_end just before tailcall node.
  Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InGlue, dl);
  InGlue = Chain.getValue(1);
}
 
// Is this global address that of a function that can be called by name? (as
// opposed to something that must hold a descriptor for an indirect call).
static bool isFunctionGlobalAddress(const GlobalValue *GV) {
  if (GV) {
    if (GV->isThreadLocal())
      return false;
 
    return GV->getValueType()->isFunctionTy();
  }
 
  return false;
}
 
SDValue PPCTargetLowering::LowerCallResult(
    SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                    *DAG.getContext());
 
  CCRetInfo.AnalyzeCallResult(
      Ins, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
               ? RetCC_PPC_Cold
               : RetCC_PPC);
 
  // Copy all of the result registers out of their specified physreg.
  for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
    CCValAssign &VA = RVLocs[i];
    assert(VA.isRegLoc() && "Can only return in registers!");
 
    SDValue Val;
 
    if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
      SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
                                      InGlue);
      Chain = Lo.getValue(1);
      InGlue = Lo.getValue(2);
      VA = RVLocs[++i]; // skip ahead to next loc
      SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
                                      InGlue);
      Chain = Hi.getValue(1);
      InGlue = Hi.getValue(2);
      if (!Subtarget.isLittleEndian())
        std::swap (Lo, Hi);
      Val = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, Lo, Hi);
    } else {
      Val = DAG.getCopyFromReg(Chain, dl,
                               VA.getLocReg(), VA.getLocVT(), InGlue);
      Chain = Val.getValue(1);
      InGlue = Val.getValue(2);
    }
 
    switch (VA.getLocInfo()) {
    default: llvm_unreachable("Unknown loc info!");
    case CCValAssign::Full: break;
    case CCValAssign::AExt:
      Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
      break;
    case CCValAssign::ZExt:
      Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
      break;
    case CCValAssign::SExt:
      Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val,
                        DAG.getValueType(VA.getValVT()));
      Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
      break;
    }
 
    InVals.push_back(Val);
  }
 
  return Chain;
}
 
static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG,
                           const PPCSubtarget &Subtarget, bool isPatchPoint) {
  auto *G = dyn_cast<GlobalAddressSDNode>(Callee);
  const GlobalValue *GV = G ? G->getGlobal() : nullptr;
 
  // PatchPoint calls are not indirect.
  if (isPatchPoint)
    return false;
 
  if (isFunctionGlobalAddress(GV) || isa<ExternalSymbolSDNode>(Callee))
    return false;
 
  // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not
  // becuase the immediate function pointer points to a descriptor instead of
  // a function entry point. The ELFv2 ABI cannot use a BLA because the function
  // pointer immediate points to the global entry point, while the BLA would
  // need to jump to the local entry point (see rL211174).
  if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() &&
      isBLACompatibleAddress(Callee, DAG))
    return false;
 
  return true;
}
 
// AIX and 64-bit ELF ABIs w/o PCRel require a TOC save/restore around calls.
static inline bool isTOCSaveRestoreRequired(const PPCSubtarget &Subtarget) {
  return Subtarget.isAIXABI() ||
         (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls());
}
 
static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags,
                              const Function &Caller, const SDValue &Callee,
                              const PPCSubtarget &Subtarget,
                              const TargetMachine &TM,
                              bool IsStrictFPCall = false) {
  if (CFlags.IsTailCall)
    return PPCISD::TC_RETURN;
 
  unsigned RetOpc = 0;
  // This is a call through a function pointer.
  if (CFlags.IsIndirect) {
    // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross
    // indirect calls. The save of the caller's TOC pointer to the stack will be
    // inserted into the DAG as part of call lowering. The restore of the TOC
    // pointer is modeled by using a pseudo instruction for the call opcode that
    // represents the 2 instruction sequence of an indirect branch and link,
    // immediately followed by a load of the TOC pointer from the stack save
    // slot into gpr2. For 64-bit ELFv2 ABI with PCRel, do not restore the TOC
    // as it is not saved or used.
    if (Subtarget.usePointerGlueHelper())
      RetOpc = PPCISD::BL_LOAD_TOC;
    else
      RetOpc = isTOCSaveRestoreRequired(Subtarget) ? PPCISD::BCTRL_LOAD_TOC
                                                   : PPCISD::BCTRL;
  } else if (Subtarget.isUsingPCRelativeCalls()) {
    assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI.");
    RetOpc = PPCISD::CALL_NOTOC;
  } else if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) {
    // The ABIs that maintain a TOC pointer accross calls need to have a nop
    // immediately following the call instruction if the caller and callee may
    // have different TOC bases. At link time if the linker determines the calls
    // may not share a TOC base, the call is redirected to a trampoline inserted
    // by the linker. The trampoline will (among other things) save the callers
    // TOC pointer at an ABI designated offset in the linkage area and the
    // linker will rewrite the nop to be a load of the TOC pointer from the
    // linkage area into gpr2.
    auto *G = dyn_cast<GlobalAddressSDNode>(Callee);
    const GlobalValue *GV = G ? G->getGlobal() : nullptr;
    RetOpc =
        callsShareTOCBase(&Caller, GV, TM) ? PPCISD::CALL : PPCISD::CALL_NOP;
  } else
    RetOpc = PPCISD::CALL;
  if (IsStrictFPCall) {
    switch (RetOpc) {
    default:
      llvm_unreachable("Unknown call opcode");
    case PPCISD::BCTRL_LOAD_TOC:
      RetOpc = PPCISD::BCTRL_LOAD_TOC_RM;
      break;
    case PPCISD::BCTRL:
      RetOpc = PPCISD::BCTRL_RM;
      break;
    case PPCISD::BL_LOAD_TOC:
      RetOpc = PPCISD::BL_LOAD_TOC_RM;
      break;
    case PPCISD::CALL_NOTOC:
      RetOpc = PPCISD::CALL_NOTOC_RM;
      break;
    case PPCISD::CALL:
      RetOpc = PPCISD::CALL_RM;
      break;
    case PPCISD::CALL_NOP:
      RetOpc = PPCISD::CALL_NOP_RM;
      break;
    }
  }
  return RetOpc;
}
 
static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
                               const SDLoc &dl, const PPCSubtarget &Subtarget) {
  if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI())
    if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG))
      return SDValue(Dest, 0);
 
  // Returns true if the callee is local, and false otherwise.
  auto isLocalCallee = [&]() {
    const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
    const GlobalValue *GV = G ? G->getGlobal() : nullptr;
 
    return DAG.getTarget().shouldAssumeDSOLocal(GV) &&
           !isa_and_nonnull<GlobalIFunc>(GV);
  };
 
  // The PLT is only used in 32-bit ELF PIC mode.  Attempting to use the PLT in
  // a static relocation model causes some versions of GNU LD (2.17.50, at
  // least) to force BSS-PLT, instead of secure-PLT, even if all objects are
  // built with secure-PLT.
  bool UsePlt =
      Subtarget.is32BitELFABI() && !isLocalCallee() &&
      Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_;
 
  const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) {
    const TargetMachine &TM = Subtarget.getTargetMachine();
    const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering();
    auto *S =
        static_cast<MCSymbolXCOFF *>(TLOF->getFunctionEntryPointSymbol(GV, TM));
 
    MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
    return DAG.getMCSymbol(S, PtrVT);
  };
 
  auto *G = dyn_cast<GlobalAddressSDNode>(Callee);
  const GlobalValue *GV = G ? G->getGlobal() : nullptr;
  if (isFunctionGlobalAddress(GV)) {
    const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
 
    if (Subtarget.isAIXABI()) {
      return getAIXFuncEntryPointSymbolSDNode(GV);
    }
    return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0,
                                      UsePlt ? PPCII::MO_PLT : 0);
  }
 
  if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
    const char *SymName = S->getSymbol();
    if (Subtarget.isAIXABI()) {
      // If there exists a user-declared function whose name is the same as the
      // ExternalSymbol's, then we pick up the user-declared version.
      const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
      if (const Function *F =
              dyn_cast_or_null<Function>(Mod->getNamedValue(SymName)))
        return getAIXFuncEntryPointSymbolSDNode(F);
 
      // On AIX, direct function calls reference the symbol for the function's
      // entry point, which is named by prepending a "." before the function's
      // C-linkage name. A Qualname is returned here because an external
      // function entry point is a csect with XTY_ER property.
      const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) {
        auto &Context = DAG.getMachineFunction().getContext();
        MCSectionXCOFF *Sec = Context.getXCOFFSection(
            (Twine(".") + Twine(SymName)).str(), SectionKind::getMetadata(),
            XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER));
        return Sec->getQualNameSymbol();
      };
 
      SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data();
    }
    return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(),
                                       UsePlt ? PPCII::MO_PLT : 0);
  }
 
  // No transformation needed.
  assert(Callee.getNode() && "What no callee?");
  return Callee;
}
 
static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) {
  assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START &&
         "Expected a CALLSEQ_STARTSDNode.");
 
  // The last operand is the chain, except when the node has glue. If the node
  // has glue, then the last operand is the glue, and the chain is the second
  // last operand.
  SDValue LastValue = CallSeqStart.getValue(CallSeqStart->getNumValues() - 1);
  if (LastValue.getValueType() != MVT::Glue)
    return LastValue;
 
  return CallSeqStart.getValue(CallSeqStart->getNumValues() - 2);
}
 
// Creates the node that moves a functions address into the count register
// to prepare for an indirect call instruction.
static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee,
                                SDValue &Glue, SDValue &Chain,
                                const SDLoc &dl) {
  SDValue MTCTROps[] = {Chain, Callee, Glue};
  EVT ReturnTypes[] = {MVT::Other, MVT::Glue};
  Chain = DAG.getNode(PPCISD::MTCTR, dl, ReturnTypes,
                      ArrayRef(MTCTROps, Glue.getNode() ? 3 : 2));
  // The glue is the second value produced.
  Glue = Chain.getValue(1);
}
 
static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee,
                                          SDValue &Glue, SDValue &Chain,
                                          SDValue CallSeqStart,
                                          const CallBase *CB, const SDLoc &dl,
                                          bool hasNest,
                                          const PPCSubtarget &Subtarget) {
  // Function pointers in the 64-bit SVR4 ABI do not point to the function
  // entry point, but to the function descriptor (the function entry point
  // address is part of the function descriptor though).
  // The function descriptor is a three doubleword structure with the
  // following fields: function entry point, TOC base address and
  // environment pointer.
  // Thus for a call through a function pointer, the following actions need
  // to be performed:
  //   1. Save the TOC of the caller in the TOC save area of its stack
  //      frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
  //   2. Load the address of the function entry point from the function
  //      descriptor.
  //   3. Load the TOC of the callee from the function descriptor into r2.
  //   4. Load the environment pointer from the function descriptor into
  //      r11.
  //   5. Branch to the function entry point address.
  //   6. On return of the callee, the TOC of the caller needs to be
  //      restored (this is done in FinishCall()).
  //
  // The loads are scheduled at the beginning of the call sequence, and the
  // register copies are flagged together to ensure that no other
  // operations can be scheduled in between. E.g. without flagging the
  // copies together, a TOC access in the caller could be scheduled between
  // the assignment of the callee TOC and the branch to the callee, which leads
  // to incorrect code.
 
  // Start by loading the function address from the descriptor.
  SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart);
  auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
                      ? (MachineMemOperand::MODereferenceable |
                         MachineMemOperand::MOInvariant)
                      : MachineMemOperand::MONone;
 
  MachinePointerInfo MPI(CB ? CB->getCalledOperand() : nullptr);
 
  // Registers used in building the DAG.
  const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister();
  const MCRegister TOCReg = Subtarget.getTOCPointerRegister();
 
  // Offsets of descriptor members.
  const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset();
  const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset();
 
  const MVT RegVT = Subtarget.getScalarIntVT();
  const Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
 
  // One load for the functions entry point address.
  SDValue LoadFuncPtr = DAG.getLoad(RegVT, dl, LDChain, Callee, MPI,
                                    Alignment, MMOFlags);
 
  // One for loading the TOC anchor for the module that contains the called
  // function.
  SDValue TOCOff = DAG.getIntPtrConstant(TOCAnchorOffset, dl);
  SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, Callee, TOCOff);
  SDValue TOCPtr =
      DAG.getLoad(RegVT, dl, LDChain, AddTOC,
                  MPI.getWithOffset(TOCAnchorOffset), Alignment, MMOFlags);
 
  // One for loading the environment pointer.
  SDValue PtrOff = DAG.getIntPtrConstant(EnvPtrOffset, dl);
  SDValue AddPtr = DAG.getNode(ISD::ADD, dl, RegVT, Callee, PtrOff);
  SDValue LoadEnvPtr =
      DAG.getLoad(RegVT, dl, LDChain, AddPtr,
                  MPI.getWithOffset(EnvPtrOffset), Alignment, MMOFlags);
 
 
  // Then copy the newly loaded TOC anchor to the TOC pointer.
  SDValue TOCVal = DAG.getCopyToReg(Chain, dl, TOCReg, TOCPtr, Glue);
  Chain = TOCVal.getValue(0);
  Glue = TOCVal.getValue(1);
 
  // If the function call has an explicit 'nest' parameter, it takes the
  // place of the environment pointer.
  assert((!hasNest || !Subtarget.isAIXABI()) &&
         "Nest parameter is not supported on AIX.");
  if (!hasNest) {
    SDValue EnvVal = DAG.getCopyToReg(Chain, dl, EnvPtrReg, LoadEnvPtr, Glue);
    Chain = EnvVal.getValue(0);
    Glue = EnvVal.getValue(1);
  }
 
  // The rest of the indirect call sequence is the same as the non-descriptor
  // DAG.
  prepareIndirectCall(DAG, LoadFuncPtr, Glue, Chain, dl);
}
 
static void prepareOutOfLineGlueCall(SelectionDAG &DAG, SDValue &Callee,
                                     SDValue &Glue, SDValue &Chain,
                                     SDValue CallSeqStart, const CallBase *CB,
                                     const SDLoc &dl, bool hasNest,
                                     const PPCSubtarget &Subtarget) {
  // On AIX there is a feature ("out of line glue code") which uses a special
  // trampoline function ._ptrgl to do the indirect call. If this option is
  // enabled we instead simply load the address of the descriptor into gpr11,
  // with the arguments in the 'normal' registers and branch to the ._ptrgl
  // stub.
  const MCRegister PtrGlueReg = Subtarget.getGlueCodeDescriptorRegister();
  SDValue MoveToPhysicalReg =
      DAG.getCopyToReg(Chain, dl, PtrGlueReg, Callee, Glue);
  Chain = MoveToPhysicalReg.getValue(0);
  Glue = MoveToPhysicalReg.getValue(1);
}
 
static void
buildCallOperands(SmallVectorImpl<SDValue> &Ops,
                  PPCTargetLowering::CallFlags CFlags, const SDLoc &dl,
                  SelectionDAG &DAG,
                  SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass,
                  SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff,
                  const PPCSubtarget &Subtarget) {
  const bool IsPPC64 = Subtarget.isPPC64();
  // MVT for a general purpose register.
  const MVT RegVT = Subtarget.getScalarIntVT();
 
  // First operand is always the chain.
  Ops.push_back(Chain);
 
  // If it's a direct call pass the callee as the second operand.
  if (!CFlags.IsIndirect)
    Ops.push_back(Callee);
  else if (Subtarget.usePointerGlueHelper()) {
    Ops.push_back(Callee);
    // Add the register used to pass the descriptor address.
    Ops.push_back(
        DAG.getRegister(Subtarget.getGlueCodeDescriptorRegister(), RegVT));
  } else {
    assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect.");
 
    // For the TOC based ABIs, we have saved the TOC pointer to the linkage area
    // on the stack (this would have been done in `LowerCall_64SVR4` or
    // `LowerCall_AIX`). The call instruction is a pseudo instruction that
    // represents both the indirect branch and a load that restores the TOC
    // pointer from the linkage area. The operand for the TOC restore is an add
    // of the TOC save offset to the stack pointer. This must be the second
    // operand: after the chain input but before any other variadic arguments.
    // For 64-bit ELFv2 ABI with PCRel, do not restore the TOC as it is not
    // saved or used.
    if (isTOCSaveRestoreRequired(Subtarget)) {
      const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
 
      SDValue StackPtr = DAG.getRegister(StackPtrReg, RegVT);
      unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
      SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
      SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, StackPtr, TOCOff);
      Ops.push_back(AddTOC);
    }
 
    // Add the register used for the environment pointer.
    if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest)
      Ops.push_back(DAG.getRegister(Subtarget.getEnvironmentPointerRegister(),
                                    RegVT));
 
 
    // Add CTR register as callee so a bctr can be emitted later.
    if (CFlags.IsTailCall)
      Ops.push_back(DAG.getRegister(IsPPC64 ? PPC::CTR8 : PPC::CTR, RegVT));
  }
 
  // If this is a tail call add stack pointer delta.
  if (CFlags.IsTailCall)
    Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32));
 
  // Add argument registers to the end of the list so that they are known live
  // into the call.
  for (const auto &[Reg, N] : RegsToPass)
    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));
 
  // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
  // no way to mark dependencies as implicit here.
  // We will add the R2/X2 dependency in EmitInstrWithCustomInserter.
  if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) &&
       !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls())
    Ops.push_back(DAG.getRegister(Subtarget.getTOCPointerRegister(), RegVT));
 
  // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
  if (CFlags.IsVarArg && Subtarget.is32BitELFABI())
    Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32));
 
  // Add a register mask operand representing the call-preserved registers.
  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
  const uint32_t *Mask =
      TRI->getCallPreservedMask(DAG.getMachineFunction(), CFlags.CallConv);
  assert(Mask && "Missing call preserved mask for calling convention");
  Ops.push_back(DAG.getRegisterMask(Mask));
 
  // If the glue is valid, it is the last operand.
  if (Glue.getNode())
    Ops.push_back(Glue);
}
 
SDValue PPCTargetLowering::FinishCall(
    CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG,
    SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue,
    SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
    unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins,
    SmallVectorImpl<SDValue> &InVals, const CallBase *CB) const {
 
  if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) ||
      Subtarget.isAIXABI())
    setUsesTOCBasePtr(DAG);
 
  unsigned CallOpc =
      getCallOpcode(CFlags, DAG.getMachineFunction().getFunction(), Callee,
                    Subtarget, DAG.getTarget(), CB ? CB->isStrictFP() : false);
 
  if (!CFlags.IsIndirect)
    Callee = transformCallee(Callee, DAG, dl, Subtarget);
  else if (Subtarget.usesFunctionDescriptors()) {
    if (Subtarget.usePointerGlueHelper()) {
      prepareOutOfLineGlueCall(DAG, Callee, Glue, Chain, CallSeqStart, CB, dl,
                               CFlags.HasNest, Subtarget);
      SDValue PtrGlueCallee =
          DAG.getExternalSymbol("_ptrgl", getPointerTy(DAG.getDataLayout()));
      Callee = transformCallee(PtrGlueCallee, DAG, dl, Subtarget);
    } else {
      prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CB,
                                    dl, CFlags.HasNest, Subtarget);
    }
  } else {
    prepareIndirectCall(DAG, Callee, Glue, Chain, dl);
  }
 
  // Build the operand list for the call instruction.
  SmallVector<SDValue, 8> Ops;
  buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee,
                    SPDiff, Subtarget);
 
  // Emit tail call.
  if (CFlags.IsTailCall) {
    // Indirect tail call when using PC Relative calls do not have the same
    // constraints.
    assert(((Callee.getOpcode() == ISD::Register &&
             cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) ||
            Callee.getOpcode() == ISD::TargetExternalSymbol ||
            Callee.getOpcode() == ISD::TargetGlobalAddress ||
            isa<ConstantSDNode>(Callee) ||
            (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) &&
           "Expecting a global address, external symbol, absolute value, "
           "register or an indirect tail call when PC Relative calls are "
           "used.");
    // PC Relative calls also use TC_RETURN as the way to mark tail calls.
    assert(CallOpc == PPCISD::TC_RETURN &&
           "Unexpected call opcode for a tail call.");
    DAG.getMachineFunction().getFrameInfo().setHasTailCall();
    SDValue Ret = DAG.getNode(CallOpc, dl, MVT::Other, Ops);
    DAG.addNoMergeSiteInfo(Ret.getNode(), CFlags.NoMerge);
    return Ret;
  }
 
  std::array<EVT, 2> ReturnTypes = {{MVT::Other, MVT::Glue}};
  Chain = DAG.getNode(CallOpc, dl, ReturnTypes, Ops);
  DAG.addNoMergeSiteInfo(Chain.getNode(), CFlags.NoMerge);
  Glue = Chain.getValue(1);
 
  // When performing tail call optimization the callee pops its arguments off
  // the stack. Account for this here so these bytes can be pushed back on in
  // PPCFrameLowering::eliminateCallFramePseudoInstr.
  int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast &&
                         getTargetMachine().Options.GuaranteedTailCallOpt)
                            ? NumBytes
                            : 0;
 
  Chain = DAG.getCALLSEQ_END(Chain, NumBytes, BytesCalleePops, Glue, dl);
  Glue = Chain.getValue(1);
 
  return LowerCallResult(Chain, Glue, CFlags.CallConv, CFlags.IsVarArg, Ins, dl,
                         DAG, InVals);
}
 
bool PPCTargetLowering::supportsTailCallFor(const CallBase *CB) const {
  CallingConv::ID CalleeCC = CB->getCallingConv();
  const Function *CallerFunc = CB->getCaller();
  CallingConv::ID CallerCC = CallerFunc->getCallingConv();
  const Function *CalleeFunc = CB->getCalledFunction();
  if (!CalleeFunc)
    return false;
  const GlobalValue *CalleeGV = dyn_cast<GlobalValue>(CalleeFunc);
 
  SmallVector<ISD::OutputArg, 2> Outs;
  SmallVector<ISD::InputArg, 2> Ins;
 
  GetReturnInfo(CalleeCC, CalleeFunc->getReturnType(),
                CalleeFunc->getAttributes(), Outs, *this,
                CalleeFunc->getDataLayout());
 
  return isEligibleForTCO(CalleeGV, CalleeCC, CallerCC, CB,
                          CalleeFunc->isVarArg(), Outs, Ins, CallerFunc,
                          false /*isCalleeExternalSymbol*/);
}
 
bool PPCTargetLowering::isEligibleForTCO(
    const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
    CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
    bool isCalleeExternalSymbol) const {
  if (Subtarget.useLongCalls() && !(CB && CB->isMustTailCall()))
    return false;
 
  if (Subtarget.isSVR4ABI() && Subtarget.isPPC64())
    return IsEligibleForTailCallOptimization_64SVR4(
        CalleeGV, CalleeCC, CallerCC, CB, isVarArg, Outs, Ins, CallerFunc,
        isCalleeExternalSymbol);
  else
    return IsEligibleForTailCallOptimization(CalleeGV, CalleeCC, CallerCC,
                                             isVarArg, Ins);
}
 
SDValue
PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                             SmallVectorImpl<SDValue> &InVals) const {
  SelectionDAG &DAG                     = CLI.DAG;
  SDLoc &dl                             = CLI.DL;
  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
  SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;
  SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;
  SDValue Chain                         = CLI.Chain;
  SDValue Callee                        = CLI.Callee;
  bool &isTailCall                      = CLI.IsTailCall;
  CallingConv::ID CallConv              = CLI.CallConv;
  bool isVarArg                         = CLI.IsVarArg;
  bool isPatchPoint                     = CLI.IsPatchPoint;
  const CallBase *CB                    = CLI.CB;
 
  if (isTailCall) {
    MachineFunction &MF = DAG.getMachineFunction();
    CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
    auto *G = dyn_cast<GlobalAddressSDNode>(Callee);
    const GlobalValue *GV = G ? G->getGlobal() : nullptr;
    bool IsCalleeExternalSymbol = isa<ExternalSymbolSDNode>(Callee);
 
    isTailCall =
        isEligibleForTCO(GV, CallConv, CallerCC, CB, isVarArg, Outs, Ins,
                         &(MF.getFunction()), IsCalleeExternalSymbol);
    if (isTailCall) {
      ++NumTailCalls;
      if (!getTargetMachine().Options.GuaranteedTailCallOpt)
        ++NumSiblingCalls;
 
      // PC Relative calls no longer guarantee that the callee is a Global
      // Address Node. The callee could be an indirect tail call in which
      // case the SDValue for the callee could be a load (to load the address
      // of a function pointer) or it may be a register copy (to move the
      // address of the callee from a function parameter into a virtual
      // register). It may also be an ExternalSymbolSDNode (ex memcopy).
      assert((Subtarget.isUsingPCRelativeCalls() ||
              isa<GlobalAddressSDNode>(Callee)) &&
             "Callee should be an llvm::Function object.");
 
      LLVM_DEBUG(dbgs() << "TCO caller: " << DAG.getMachineFunction().getName()
                        << "\nTCO callee: ");
      LLVM_DEBUG(Callee.dump());
    }
  }
 
  if (!isTailCall && CB && CB->isMustTailCall())
    report_fatal_error("failed to perform tail call elimination on a call "
                       "site marked musttail");
 
  // When long calls (i.e. indirect calls) are always used, calls are always
  // made via function pointer. If we have a function name, first translate it
  // into a pointer.
  if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Callee) &&
      !isTailCall)
    Callee = LowerGlobalAddress(Callee, DAG);
 
  CallFlags CFlags(
      CallConv, isTailCall, isVarArg, isPatchPoint,
      isIndirectCall(Callee, DAG, Subtarget, isPatchPoint),
      // hasNest
      Subtarget.is64BitELFABI() &&
          any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }),
      CLI.NoMerge);
 
  if (Subtarget.isAIXABI())
    return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
                         InVals, CB);
 
  assert(Subtarget.isSVR4ABI());
  if (Subtarget.isPPC64())
    return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
                            InVals, CB);
  return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
                          InVals, CB);
}
 
SDValue PPCTargetLowering::LowerCall_32SVR4(
    SDValue Chain, SDValue Callee, CallFlags CFlags,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<SDValue> &OutVals,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
    const CallBase *CB) const {
  // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
  // of the 32-bit SVR4 ABI stack frame layout.
 
  const CallingConv::ID CallConv = CFlags.CallConv;
  const bool IsVarArg = CFlags.IsVarArg;
  const bool IsTailCall = CFlags.IsTailCall;
 
  assert((CallConv == CallingConv::C ||
          CallConv == CallingConv::Cold ||
          CallConv == CallingConv::Fast) && "Unknown calling convention!");
 
  const Align PtrAlign(4);
 
  MachineFunction &MF = DAG.getMachineFunction();
 
  // Mark this function as potentially containing a function that contains a
  // tail call. As a consequence the frame pointer will be used for dynamicalloc
  // and restoring the callers stack pointer in this functions epilog. This is
  // done because by tail calling the called function might overwrite the value
  // in this function's (MF) stack pointer stack slot 0(SP).
  if (getTargetMachine().Options.GuaranteedTailCallOpt &&
      CallConv == CallingConv::Fast)
    MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
 
  // Count how many bytes are to be pushed on the stack, including the linkage
  // area, parameter list area and the part of the local variable space which
  // contains copies of aggregates which are passed by value.
 
  // Assign locations to all of the outgoing arguments.
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
 
  // Reserve space for the linkage area on the stack.
  CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(),
                       PtrAlign);
 
  if (IsVarArg) {
    // Handle fixed and variable vector arguments differently.
    // Fixed vector arguments go into registers as long as registers are
    // available. Variable vector arguments always go into memory.
    unsigned NumArgs = Outs.size();
 
    for (unsigned i = 0; i != NumArgs; ++i) {
      MVT ArgVT = Outs[i].VT;
      ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
      bool Result;
 
      if (!ArgFlags.isVarArg()) {
        Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags,
                               Outs[i].OrigTy, CCInfo);
      } else {
        Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full,
                                      ArgFlags, Outs[i].OrigTy, CCInfo);
      }
 
      if (Result) {
#ifndef NDEBUG
        errs() << "Call operand #" << i << " has unhandled type "
               << ArgVT << "\n";
#endif
        llvm_unreachable(nullptr);
      }
    }
  } else {
    // All arguments are treated the same.
    CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4);
  }
 
  // Assign locations to all of the outgoing aggregate by value arguments.
  SmallVector<CCValAssign, 16> ByValArgLocs;
  CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext());
 
  // Reserve stack space for the allocations in CCInfo.
  CCByValInfo.AllocateStack(CCInfo.getStackSize(), PtrAlign);
 
  CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal);
 
  // Size of the linkage area, parameter list area and the part of the local
  // space variable where copies of aggregates which are passed by value are
  // stored.
  unsigned NumBytes = CCByValInfo.getStackSize();
 
  // Calculate by how many bytes the stack has to be adjusted in case of tail
  // call optimization.
  int SPDiff = CalculateTailCallSPDiff(DAG, IsTailCall, NumBytes);
 
  // Adjust the stack pointer for the new arguments...
  // These operations are automatically eliminated by the prolog/epilog pass
  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
  SDValue CallSeqStart = Chain;
 
  // Load the return address and frame pointer so it can be moved somewhere else
  // later.
  SDValue LROp, FPOp;
  Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl);
 
  // Set up a copy of the stack pointer for use loading and storing any
  // arguments that may not fit in the registers available for argument
  // passing.
  SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32);
 
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
  SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
  SmallVector<SDValue, 8> MemOpChains;
 
  bool seenFloatArg = false;
  // Walk the register/memloc assignments, inserting copies/loads.
  // i - Tracks the index into the list of registers allocated for the call
  // RealArgIdx - Tracks the index into the list of actual function arguments
  // j - Tracks the index into the list of byval arguments
  for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size();
       i != e;
       ++i, ++RealArgIdx) {
    CCValAssign &VA = ArgLocs[i];
    SDValue Arg = OutVals[RealArgIdx];
    ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags;
 
    if (Flags.isByVal()) {
      // Argument is an aggregate which is passed by value, thus we need to
      // create a copy of it in the local variable space of the current stack
      // frame (which is the stack frame of the caller) and pass the address of
      // this copy to the callee.
      assert((j < ByValArgLocs.size()) && "Index out of bounds!");
      CCValAssign &ByValVA = ByValArgLocs[j++];
      assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
 
      // Memory reserved in the local variable space of the callers stack frame.
      unsigned LocMemOffset = ByValVA.getLocMemOffset();
 
      SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
      PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()),
                           StackPtr, PtrOff);
 
      // Create a copy of the argument in the local area of the current
      // stack frame.
      SDValue MemcpyCall =
        CreateCopyOfByValArgument(Arg, PtrOff,
                                  CallSeqStart.getNode()->getOperand(0),
                                  Flags, DAG, dl);
 
      // This must go outside the CALLSEQ_START..END.
      SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0,
                                                     SDLoc(MemcpyCall));
      DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
                             NewCallSeqStart.getNode());
      Chain = CallSeqStart = NewCallSeqStart;
 
      // Pass the address of the aggregate copy on the stack either in a
      // physical register or in the parameter list area of the current stack
      // frame to the callee.
      Arg = PtrOff;
    }
 
    // When useCRBits() is true, there can be i1 arguments.
    // It is because getRegisterType(MVT::i1) => MVT::i1,
    // and for other integer types getRegisterType() => MVT::i32.
    // Extend i1 and ensure callee will get i32.
    if (Arg.getValueType() == MVT::i1)
      Arg = DAG.getNode(Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
                        dl, MVT::i32, Arg);
 
    if (VA.isRegLoc()) {
      seenFloatArg |= VA.getLocVT().isFloatingPoint();
      // Put argument in a physical register.
      if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) {
        bool IsLE = Subtarget.isLittleEndian();
        SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
                        DAG.getIntPtrConstant(IsLE ? 0 : 1, dl));
        RegsToPass.push_back(std::make_pair(VA.getLocReg(), SVal.getValue(0)));
        SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
                           DAG.getIntPtrConstant(IsLE ? 1 : 0, dl));
        RegsToPass.push_back(std::make_pair(ArgLocs[++i].getLocReg(),
                             SVal.getValue(0)));
      } else
        RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
    } else {
      // Put argument in the parameter list area of the current stack frame.
      assert(VA.isMemLoc());
      unsigned LocMemOffset = VA.getLocMemOffset();
 
      if (!IsTailCall) {
        SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
        PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()),
                             StackPtr, PtrOff);
 
        MemOpChains.push_back(
            DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()));
      } else {
        // Calculate and remember argument location.
        CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset,
                                 TailCallArguments);
      }
    }
  }
 
  if (!MemOpChains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
 
  // Build a sequence of copy-to-reg nodes chained together with token chain
  // and flag operands which copy the outgoing args into the appropriate regs.
  SDValue InGlue;
  for (const auto &[Reg, N] : RegsToPass) {
    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
    InGlue = Chain.getValue(1);
  }
 
  // Set CR bit 6 to true if this is a vararg call with floating args passed in
  // registers.
  if (IsVarArg) {
    SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
    SDValue Ops[] = { Chain, InGlue };
 
    Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, dl,
                        VTs, ArrayRef(Ops, InGlue.getNode() ? 2 : 1));
 
    InGlue = Chain.getValue(1);
  }
 
  if (IsTailCall)
    PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
                    TailCallArguments);
 
  return FinishCall(CFlags, dl, DAG, RegsToPass, InGlue, Chain, CallSeqStart,
                    Callee, SPDiff, NumBytes, Ins, InVals, CB);
}
 
// Copy an argument into memory, being careful to do this outside the
// call sequence for the call to which the argument belongs.
SDValue PPCTargetLowering::createMemcpyOutsideCallSeq(
    SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
    SelectionDAG &DAG, const SDLoc &dl) const {
  SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff,
                        CallSeqStart.getNode()->getOperand(0),
                        Flags, DAG, dl);
  // The MEMCPY must go outside the CALLSEQ_START..END.
  int64_t FrameSize = CallSeqStart.getConstantOperandVal(1);
  SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0,
                                                 SDLoc(MemcpyCall));
  DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
                         NewCallSeqStart.getNode());
  return NewCallSeqStart;
}
 
SDValue PPCTargetLowering::LowerCall_64SVR4(
    SDValue Chain, SDValue Callee, CallFlags CFlags,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<SDValue> &OutVals,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
    const CallBase *CB) const {
  bool isELFv2ABI = Subtarget.isELFv2ABI();
  bool isLittleEndian = Subtarget.isLittleEndian();
  unsigned NumOps = Outs.size();
  bool IsSibCall = false;
  bool IsFastCall = CFlags.CallConv == CallingConv::Fast;
 
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  unsigned PtrByteSize = 8;
 
  MachineFunction &MF = DAG.getMachineFunction();
 
  if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt)
    IsSibCall = true;
 
  // Mark this function as potentially containing a function that contains a
  // tail call. As a consequence the frame pointer will be used for dynamicalloc
  // and restoring the callers stack pointer in this functions epilog. This is
  // done because by tail calling the called function might overwrite the value
  // in this function's (MF) stack pointer stack slot 0(SP).
  if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
    MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
 
  assert(!(IsFastCall && CFlags.IsVarArg) &&
         "fastcc not supported on varargs functions");
 
  // Count how many bytes are to be pushed on the stack, including the linkage
  // area, and parameter passing area.  On ELFv1, the linkage area is 48 bytes
  // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage
  // area is 32 bytes reserved space for [SP][CR][LR][TOC].
  unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
  unsigned NumBytes = LinkageSize;
  unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
 
  static const MCPhysReg GPR[] = {
    PPC::X3, PPC::X4, PPC::X5, PPC::X6,
    PPC::X7, PPC::X8, PPC::X9, PPC::X10,
  };
  static const MCPhysReg VR[] = {
    PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
    PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
  };
 
  const unsigned NumGPRs = std::size(GPR);
  const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
  const unsigned NumVRs = std::size(VR);
 
  // On ELFv2, we can avoid allocating the parameter area if all the arguments
  // can be passed to the callee in registers.
  // For the fast calling convention, there is another check below.
  // Note: We should keep consistent with LowerFormalArguments_64SVR4()
  bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall;
  if (!HasParameterArea) {
    unsigned ParamAreaSize = NumGPRs * PtrByteSize;
    unsigned AvailableFPRs = NumFPRs;
    unsigned AvailableVRs = NumVRs;
    unsigned NumBytesTmp = NumBytes;
    for (unsigned i = 0; i != NumOps; ++i) {
      if (Outs[i].Flags.isNest()) continue;
      if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags,
                                 PtrByteSize, LinkageSize, ParamAreaSize,
                                 NumBytesTmp, AvailableFPRs, AvailableVRs))
        HasParameterArea = true;
    }
  }
 
  // When using the fast calling convention, we don't provide backing for
  // arguments that will be in registers.
  unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0;
 
  // Avoid allocating parameter area for fastcc functions if all the arguments
  // can be passed in the registers.
  if (IsFastCall)
    HasParameterArea = false;
 
  // Add up all the space actually used.
  for (unsigned i = 0; i != NumOps; ++i) {
    ISD::ArgFlagsTy Flags = Outs[i].Flags;
    EVT ArgVT = Outs[i].VT;
    EVT OrigVT = Outs[i].ArgVT;
 
    if (Flags.isNest())
      continue;
 
    if (IsFastCall) {
      if (Flags.isByVal()) {
        NumGPRsUsed += (Flags.getByValSize()+7)/8;
        if (NumGPRsUsed > NumGPRs)
          HasParameterArea = true;
      } else {
        switch (ArgVT.getSimpleVT().SimpleTy) {
        default: llvm_unreachable("Unexpected ValueType for argument!");
        case MVT::i1:
        case MVT::i32:
        case MVT::i64:
          if (++NumGPRsUsed <= NumGPRs)
            continue;
          break;
        case MVT::v4i32:
        case MVT::v8i16:
        case MVT::v16i8:
        case MVT::v2f64:
        case MVT::v2i64:
        case MVT::v1i128:
        case MVT::f128:
          if (++NumVRsUsed <= NumVRs)
            continue;
          break;
        case MVT::v4f32:
          if (++NumVRsUsed <= NumVRs)
            continue;
          break;
        case MVT::f32:
        case MVT::f64:
          if (++NumFPRsUsed <= NumFPRs)
            continue;
          break;
        }
        HasParameterArea = true;
      }
    }
 
    /* Respect alignment of argument on the stack.  */
    auto Alignement =
        CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
    NumBytes = alignTo(NumBytes, Alignement);
 
    NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
    if (Flags.isInConsecutiveRegsLast())
      NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
  }
 
  unsigned NumBytesActuallyUsed = NumBytes;
 
  // In the old ELFv1 ABI,
  // the prolog code of the callee may store up to 8 GPR argument registers to
  // the stack, allowing va_start to index over them in memory if its varargs.
  // Because we cannot tell if this is needed on the caller side, we have to
  // conservatively assume that it is needed.  As such, make sure we have at
  // least enough stack space for the caller to store the 8 GPRs.
  // In the ELFv2 ABI, we allocate the parameter area iff a callee
  // really requires memory operands, e.g. a vararg function.
  if (HasParameterArea)
    NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize);
  else
    NumBytes = LinkageSize;
 
  // Tail call needs the stack to be aligned.
  if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
    NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes);
 
  int SPDiff = 0;
 
  // Calculate by how many bytes the stack has to be adjusted in case of tail
  // call optimization.
  if (!IsSibCall)
    SPDiff = CalculateTailCallSPDiff(DAG, CFlags.IsTailCall, NumBytes);
 
  // To protect arguments on the stack from being clobbered in a tail call,
  // force all the loads to happen before doing any other lowering.
  if (CFlags.IsTailCall)
    Chain = DAG.getStackArgumentTokenFactor(Chain);
 
  // Adjust the stack pointer for the new arguments...
  // These operations are automatically eliminated by the prolog/epilog pass
  if (!IsSibCall)
    Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
  SDValue CallSeqStart = Chain;
 
  // Load the return address and frame pointer so it can be move somewhere else
  // later.
  SDValue LROp, FPOp;
  Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl);
 
  // Set up a copy of the stack pointer for use loading and storing any
  // arguments that may not fit in the registers available for argument
  // passing.
  SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
 
  // Figure out which arguments are going to go in registers, and which in
  // memory.  Also, if this is a vararg function, floating point operations
  // must be stored to our stack, and loaded into integer regs as well, if
  // any integer regs are available for argument passing.
  unsigned ArgOffset = LinkageSize;
 
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
  SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
 
  SmallVector<SDValue, 8> MemOpChains;
  for (unsigned i = 0; i != NumOps; ++i) {
    SDValue Arg = OutVals[i];
    ISD::ArgFlagsTy Flags = Outs[i].Flags;
    EVT ArgVT = Outs[i].VT;
    EVT OrigVT = Outs[i].ArgVT;
 
    // PtrOff will be used to store the current argument to the stack if a
    // register cannot be found for it.
    SDValue PtrOff;
 
    // We re-align the argument offset for each argument, except when using the
    // fast calling convention, when we need to make sure we do that only when
    // we'll actually use a stack slot.
    auto ComputePtrOff = [&]() {
      /* Respect alignment of argument on the stack.  */
      auto Alignment =
          CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
      ArgOffset = alignTo(ArgOffset, Alignment);
 
      PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType());
 
      PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
    };
 
    if (!IsFastCall) {
      ComputePtrOff();
 
      /* Compute GPR index associated with argument offset.  */
      GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
      GPR_idx = std::min(GPR_idx, NumGPRs);
    }
 
    // Promote integers to 64-bit values.
    if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) {
      // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
      unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
      Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg);
    }
 
    // FIXME memcpy is used way more than necessary.  Correctness first.
    // Note: "by value" is code for passing a structure by value, not
    // basic types.
    if (Flags.isByVal()) {
      // Note: Size includes alignment padding, so
      //   struct x { short a; char b; }
      // will have Size = 4.  With #pragma pack(1), it will have Size = 3.
      // These are the proper values we need for right-justifying the
      // aggregate in a parameter register.
      unsigned Size = Flags.getByValSize();
 
      // An empty aggregate parameter takes up no storage and no
      // registers.
      if (Size == 0)
        continue;
 
      if (IsFastCall)
        ComputePtrOff();
 
      // All aggregates smaller than 8 bytes must be passed right-justified.
      if (Size==1 || Size==2 || Size==4) {
        EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
        if (GPR_idx != NumGPRs) {
          SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
                                        MachinePointerInfo(), VT);
          MemOpChains.push_back(Load.getValue(1));
          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
 
          ArgOffset += PtrByteSize;
          continue;
        }
      }
 
      if (GPR_idx == NumGPRs && Size < 8) {
        SDValue AddPtr = PtrOff;
        if (!isLittleEndian) {
          SDValue Const = DAG.getConstant(PtrByteSize - Size, dl,
                                          PtrOff.getValueType());
          AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
        }
        Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
                                                          CallSeqStart,
                                                          Flags, DAG, dl);
        ArgOffset += PtrByteSize;
        continue;
      }
      // Copy the object to parameter save area if it can not be entirely passed
      // by registers.
      // FIXME: we only need to copy the parts which need to be passed in
      // parameter save area. For the parts passed by registers, we don't need
      // to copy them to the stack although we need to allocate space for them
      // in parameter save area.
      if ((NumGPRs - GPR_idx) * PtrByteSize < Size)
        Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
                                                          CallSeqStart,
                                                          Flags, DAG, dl);
 
      // When a register is available, pass a small aggregate right-justified.
      if (Size < 8 && GPR_idx != NumGPRs) {
        // The easiest way to get this right-justified in a register
        // is to copy the structure into the rightmost portion of a
        // local variable slot, then load the whole slot into the
        // register.
        // FIXME: The memcpy seems to produce pretty awful code for
        // small aggregates, particularly for packed ones.
        // FIXME: It would be preferable to use the slot in the
        // parameter save area instead of a new local variable.
        SDValue AddPtr = PtrOff;
        if (!isLittleEndian) {
          SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType());
          AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
        }
        Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
                                                          CallSeqStart,
                                                          Flags, DAG, dl);
 
        // Load the slot into the register.
        SDValue Load =
            DAG.getLoad(PtrVT, dl, Chain, PtrOff, MachinePointerInfo());
        MemOpChains.push_back(Load.getValue(1));
        RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
 
        // Done with this argument.
        ArgOffset += PtrByteSize;
        continue;
      }
 
      // For aggregates larger than PtrByteSize, copy the pieces of the
      // object that fit into registers from the parameter save area.
      for (unsigned j=0; j<Size; j+=PtrByteSize) {
        SDValue Const = DAG.getConstant(j, dl, PtrOff.getValueType());
        SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
        if (GPR_idx != NumGPRs) {
          unsigned LoadSizeInBits = std::min(PtrByteSize, (Size - j)) * 8;
          EVT ObjType = EVT::getIntegerVT(*DAG.getContext(), LoadSizeInBits);
          SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, AddArg,
                                        MachinePointerInfo(), ObjType);
 
          MemOpChains.push_back(Load.getValue(1));
          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
          ArgOffset += PtrByteSize;
        } else {
          ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
          break;
        }
      }
      continue;
    }
 
    switch (Arg.getSimpleValueType().SimpleTy) {
    default: llvm_unreachable("Unexpected ValueType for argument!");
    case MVT::i1:
    case MVT::i32:
    case MVT::i64:
      if (Flags.isNest()) {
        // The 'nest' parameter, if any, is passed in R11.
        RegsToPass.push_back(std::make_pair(PPC::X11, Arg));
        break;
      }
 
      // These can be scalar arguments or elements of an integer array type
      // passed directly.  Clang may use those instead of "byval" aggregate
      // types to avoid forcing arguments to memory unnecessarily.
      if (GPR_idx != NumGPRs) {
        RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg));
      } else {
        if (IsFastCall)
          ComputePtrOff();
 
        assert(HasParameterArea &&
               "Parameter area must exist to pass an argument in memory.");
        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
                         true, CFlags.IsTailCall, false, MemOpChains,
                         TailCallArguments, dl);
        if (IsFastCall)
          ArgOffset += PtrByteSize;
      }
      if (!IsFastCall)
        ArgOffset += PtrByteSize;
      break;
    case MVT::f32:
    case MVT::f64: {
      // These can be scalar arguments or elements of a float array type
      // passed directly.  The latter are used to implement ELFv2 homogenous
      // float aggregates.
 
      // Named arguments go into FPRs first, and once they overflow, the
      // remaining arguments go into GPRs and then the parameter save area.
      // Unnamed arguments for vararg functions always go to GPRs and
      // then the parameter save area.  For now, put all arguments to vararg
      // routines always in both locations (FPR *and* GPR or stack slot).
      bool NeedGPROrStack = CFlags.IsVarArg || FPR_idx == NumFPRs;
      bool NeededLoad = false;
 
      // First load the argument into the next available FPR.
      if (FPR_idx != NumFPRs)
        RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg));
 
      // Next, load the argument into GPR or stack slot if needed.
      if (!NeedGPROrStack)
        ;
      else if (GPR_idx != NumGPRs && !IsFastCall) {
        // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
        // once we support fp <-> gpr moves.
 
        // In the non-vararg case, this can only ever happen in the
        // presence of f32 array types, since otherwise we never run
        // out of FPRs before running out of GPRs.
        SDValue ArgVal;
 
        // Double values are always passed in a single GPR.
        if (Arg.getValueType() != MVT::f32) {
          ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg);
 
        // Non-array float values are extended and passed in a GPR.
        } else if (!Flags.isInConsecutiveRegs()) {
          ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
          ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal);
 
        // If we have an array of floats, we collect every odd element
        // together with its predecessor into one GPR.
        } else if (ArgOffset % PtrByteSize != 0) {
          SDValue Lo, Hi;
          Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]);
          Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
          if (!isLittleEndian)
            std::swap(Lo, Hi);
          ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
 
        // The final element, if even, goes into the first half of a GPR.
        } else if (Flags.isInConsecutiveRegsLast()) {
          ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg);
          ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal);
          if (!isLittleEndian)
            ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal,
                                 DAG.getConstant(32, dl, MVT::i32));
 
        // Non-final even elements are skipped; they will be handled
        // together the with subsequent argument on the next go-around.
        } else
          ArgVal = SDValue();
 
        if (ArgVal.getNode())
          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal));
      } else {
        if (IsFastCall)
          ComputePtrOff();
 
        // Single-precision floating-point values are mapped to the
        // second (rightmost) word of the stack doubleword.
        if (Arg.getValueType() == MVT::f32 &&
            !isLittleEndian && !Flags.isInConsecutiveRegs()) {
          SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType());
          PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
        }
 
        assert(HasParameterArea &&
               "Parameter area must exist to pass an argument in memory.");
        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
                         true, CFlags.IsTailCall, false, MemOpChains,
                         TailCallArguments, dl);
 
        NeededLoad = true;
      }
      // When passing an array of floats, the array occupies consecutive
      // space in the argument area; only round up to the next doubleword
      // at the end of the array.  Otherwise, each float takes 8 bytes.
      if (!IsFastCall || NeededLoad) {
        ArgOffset += (Arg.getValueType() == MVT::f32 &&
                      Flags.isInConsecutiveRegs()) ? 4 : 8;
        if (Flags.isInConsecutiveRegsLast())
          ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
      }
      break;
    }
    case MVT::v4f32:
    case MVT::v4i32:
    case MVT::v8i16:
    case MVT::v16i8:
    case MVT::v2f64:
    case MVT::v2i64:
    case MVT::v1i128:
    case MVT::f128:
      // These can be scalar arguments or elements of a vector array type
      // passed directly.  The latter are used to implement ELFv2 homogenous
      // vector aggregates.
 
      // For a varargs call, named arguments go into VRs or on the stack as
      // usual; unnamed arguments always go to the stack or the corresponding
      // GPRs when within range.  For now, we always put the value in both
      // locations (or even all three).
      if (CFlags.IsVarArg) {
        assert(HasParameterArea &&
               "Parameter area must exist if we have a varargs call.");
        // We could elide this store in the case where the object fits
        // entirely in R registers.  Maybe later.
        SDValue Store =
            DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
        MemOpChains.push_back(Store);
        if (VR_idx != NumVRs) {
          SDValue Load =
              DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo());
          MemOpChains.push_back(Load.getValue(1));
          RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load));
        }
        ArgOffset += 16;
        for (unsigned i=0; i<16; i+=PtrByteSize) {
          if (GPR_idx == NumGPRs)
            break;
          SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
                                   DAG.getConstant(i, dl, PtrVT));
          SDValue Load =
              DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo());
          MemOpChains.push_back(Load.getValue(1));
          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
        }
        break;
      }
 
      // Non-varargs Altivec params go into VRs or on the stack.
      if (VR_idx != NumVRs) {
        RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg));
      } else {
        if (IsFastCall)
          ComputePtrOff();
 
        assert(HasParameterArea &&
               "Parameter area must exist to pass an argument in memory.");
        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
                         true, CFlags.IsTailCall, true, MemOpChains,
                         TailCallArguments, dl);
        if (IsFastCall)
          ArgOffset += 16;
      }
 
      if (!IsFastCall)
        ArgOffset += 16;
      break;
    }
  }
 
  assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) &&
         "mismatch in size of parameter area");
  (void)NumBytesActuallyUsed;
 
  if (!MemOpChains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
 
  // Check if this is an indirect call (MTCTR/BCTRL).
  // See prepareDescriptorIndirectCall and buildCallOperands for more
  // information about calls through function pointers in the 64-bit SVR4 ABI.
  if (CFlags.IsIndirect) {
    // For 64-bit ELFv2 ABI with PCRel, do not save the TOC of the
    // caller in the TOC save area.
    if (isTOCSaveRestoreRequired(Subtarget)) {
      assert(!CFlags.IsTailCall && "Indirect tails calls not supported");
      // Load r2 into a virtual register and store it to the TOC save area.
      setUsesTOCBasePtr(DAG);
      SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64);
      // TOC save area offset.
      unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
      SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
      SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
      Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr,
                           MachinePointerInfo::getStack(
                               DAG.getMachineFunction(), TOCSaveOffset));
    }
    // In the ELFv2 ABI, R12 must contain the address of an indirect callee.
    // This does not mean the MTCTR instruction must use R12; it's easier
    // to model this as an extra parameter, so do that.
    if (isELFv2ABI && !CFlags.IsPatchPoint)
      RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee));
  }
 
  // Build a sequence of copy-to-reg nodes chained together with token chain
  // and flag operands which copy the outgoing args into the appropriate regs.
  SDValue InGlue;
  for (const auto &[Reg, N] : RegsToPass) {
    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);
    InGlue = Chain.getValue(1);
  }
 
  if (CFlags.IsTailCall && !IsSibCall)
    PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
                    TailCallArguments);
 
  return FinishCall(CFlags, dl, DAG, RegsToPass, InGlue, Chain, CallSeqStart,
                    Callee, SPDiff, NumBytes, Ins, InVals, CB);
}
 
// Returns true when the shadow of a general purpose argument register
// in the parameter save area is aligned to at least 'RequiredAlign'.
static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) {
  assert(RequiredAlign.value() <= 16 &&
         "Required alignment greater than stack alignment.");
  switch (Reg) {
  default:
    report_fatal_error("called on invalid register.");
  case PPC::R5:
  case PPC::R9:
  case PPC::X3:
  case PPC::X5:
  case PPC::X7:
  case PPC::X9:
    // These registers are 16 byte aligned which is the most strict aligment
    // we can support.
    return true;
  case PPC::R3:
  case PPC::R7:
  case PPC::X4:
  case PPC::X6:
  case PPC::X8:
  case PPC::X10:
    // The shadow of these registers in the PSA is 8 byte aligned.
    return RequiredAlign <= 8;
  case PPC::R4:
  case PPC::R6:
  case PPC::R8:
  case PPC::R10:
    return RequiredAlign <= 4;
  }
}
 
static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
                   CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
                   Type *OrigTy, CCState &State) {
  const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(
      State.getMachineFunction().getSubtarget());
  const bool IsPPC64 = Subtarget.isPPC64();
  const unsigned PtrSize = IsPPC64 ? 8 : 4;
  const Align PtrAlign(PtrSize);
  const Align StackAlign(16);
  const MVT RegVT = Subtarget.getScalarIntVT();
 
  if (ValVT == MVT::f128)
    report_fatal_error("f128 is unimplemented on AIX.");
 
  static const MCPhysReg GPR_32[] = {// 32-bit registers.
                                     PPC::R3, PPC::R4, PPC::R5, PPC::R6,
                                     PPC::R7, PPC::R8, PPC::R9, PPC::R10};
  static const MCPhysReg GPR_64[] = {// 64-bit registers.
                                     PPC::X3, PPC::X4, PPC::X5, PPC::X6,
                                     PPC::X7, PPC::X8, PPC::X9, PPC::X10};
 
  static const MCPhysReg VR[] = {// Vector registers.
                                 PPC::V2,  PPC::V3,  PPC::V4,  PPC::V5,
                                 PPC::V6,  PPC::V7,  PPC::V8,  PPC::V9,
                                 PPC::V10, PPC::V11, PPC::V12, PPC::V13};
 
  const ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32;
 
  if (ArgFlags.isNest()) {
    MCRegister EnvReg = State.AllocateReg(IsPPC64 ? PPC::X11 : PPC::R11);
    if (!EnvReg)
      report_fatal_error("More then one nest argument.");
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, EnvReg, RegVT, LocInfo));
    return false;
  }
 
  if (ArgFlags.isByVal()) {
    const Align ByValAlign(ArgFlags.getNonZeroByValAlign());
    if (ByValAlign > StackAlign)
      report_fatal_error("Pass-by-value arguments with alignment greater than "
                         "16 are not supported.");
 
    const unsigned ByValSize = ArgFlags.getByValSize();
    const Align ObjAlign = ByValAlign > PtrAlign ? ByValAlign : PtrAlign;
 
    // An empty aggregate parameter takes up no storage and no registers,
    // but needs a MemLoc for a stack slot for the formal arguments side.
    if (ByValSize == 0) {
      State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE,
                                       State.getStackSize(), RegVT, LocInfo));
      return false;
    }
 
    // Shadow allocate any registers that are not properly aligned.
    unsigned NextReg = State.getFirstUnallocated(GPRs);
    while (NextReg != GPRs.size() &&
           !isGPRShadowAligned(GPRs[NextReg], ObjAlign)) {
      // Shadow allocate next registers since its aligment is not strict enough.
      MCRegister Reg = State.AllocateReg(GPRs);
      // Allocate the stack space shadowed by said register.
      State.AllocateStack(PtrSize, PtrAlign);
      assert(Reg && "Alocating register unexpectedly failed.");
      (void)Reg;
      NextReg = State.getFirstUnallocated(GPRs);
    }
 
    const unsigned StackSize = alignTo(ByValSize, ObjAlign);
    unsigned Offset = State.AllocateStack(StackSize, ObjAlign);
    for (const unsigned E = Offset + StackSize; Offset < E; Offset += PtrSize) {
      if (MCRegister Reg = State.AllocateReg(GPRs))
        State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo));
      else {
        State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE,
                                         Offset, MVT::INVALID_SIMPLE_VALUE_TYPE,
                                         LocInfo));
        break;
      }
    }
    return false;
  }
 
  // Arguments always reserve parameter save area.
  switch (ValVT.SimpleTy) {
  default:
    report_fatal_error("Unhandled value type for argument.");
  case MVT::i64:
    // i64 arguments should have been split to i32 for PPC32.
    assert(IsPPC64 && "PPC32 should have split i64 values.");
    [[fallthrough]];
  case MVT::i1:
  case MVT::i32: {
    const unsigned Offset = State.AllocateStack(PtrSize, PtrAlign);
    // AIX integer arguments are always passed in register width.
    if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits())
      LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt
                                  : CCValAssign::LocInfo::ZExt;
    if (MCRegister Reg = State.AllocateReg(GPRs))
      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo));
    else
      State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, RegVT, LocInfo));
 
    return false;
  }
  case MVT::f32:
  case MVT::f64: {
    // Parameter save area (PSA) is reserved even if the float passes in fpr.
    const unsigned StoreSize = LocVT.getStoreSize();
    // Floats are always 4-byte aligned in the PSA on AIX.
    // This includes f64 in 64-bit mode for ABI compatibility.
    const unsigned Offset =
        State.AllocateStack(IsPPC64 ? 8 : StoreSize, Align(4));
    MCRegister FReg = State.AllocateReg(FPR);
    if (FReg)
      State.addLoc(CCValAssign::getReg(ValNo, ValVT, FReg, LocVT, LocInfo));
 
    // Reserve and initialize GPRs or initialize the PSA as required.
    for (unsigned I = 0; I < StoreSize; I += PtrSize) {
      if (MCRegister Reg = State.AllocateReg(GPRs)) {
        assert(FReg && "An FPR should be available when a GPR is reserved.");
        if (State.isVarArg()) {
          // Successfully reserved GPRs are only initialized for vararg calls.
          // Custom handling is required for:
          //   f64 in PPC32 needs to be split into 2 GPRs.
          //   f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR.
          State.addLoc(
              CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo));
        }
      } else {
        // If there are insufficient GPRs, the PSA needs to be initialized.
        // Initialization occurs even if an FPR was initialized for
        // compatibility with the AIX XL compiler. The full memory for the
        // argument will be initialized even if a prior word is saved in GPR.
        // A custom memLoc is used when the argument also passes in FPR so
        // that the callee handling can skip over it easily.
        State.addLoc(
            FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT,
                                             LocInfo)
                 : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
        break;
      }
    }
 
    return false;
  }
  case MVT::v4f32:
  case MVT::v4i32:
  case MVT::v8i16:
  case MVT::v16i8:
  case MVT::v2i64:
  case MVT::v2f64:
  case MVT::v1i128: {
    const unsigned VecSize = 16;
    const Align VecAlign(VecSize);
 
    if (!State.isVarArg()) {
      // If there are vector registers remaining we don't consume any stack
      // space.
      if (MCRegister VReg = State.AllocateReg(VR)) {
        State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo));
        return false;
      }
      // Vectors passed on the stack do not shadow GPRs or FPRs even though they
      // might be allocated in the portion of the PSA that is shadowed by the
      // GPRs.
      const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
      State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
      return false;
    }
 
    unsigned NextRegIndex = State.getFirstUnallocated(GPRs);
    // Burn any underaligned registers and their shadowed stack space until
    // we reach the required alignment.
    while (NextRegIndex != GPRs.size() &&
           !isGPRShadowAligned(GPRs[NextRegIndex], VecAlign)) {
      // Shadow allocate register and its stack shadow.
      MCRegister Reg = State.AllocateReg(GPRs);
      State.AllocateStack(PtrSize, PtrAlign);
      assert(Reg && "Allocating register unexpectedly failed.");
      (void)Reg;
      NextRegIndex = State.getFirstUnallocated(GPRs);
    }
 
    // Vectors that are passed as fixed arguments are handled differently.
    // They are passed in VRs if any are available (unlike arguments passed
    // through ellipses) and shadow GPRs (unlike arguments to non-vaarg
    // functions)
    if (!ArgFlags.isVarArg()) {
      if (MCRegister VReg = State.AllocateReg(VR)) {
        State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo));
        // Shadow allocate GPRs and stack space even though we pass in a VR.
        for (unsigned I = 0; I != VecSize; I += PtrSize)
          State.AllocateReg(GPRs);
        State.AllocateStack(VecSize, VecAlign);
        return false;
      }
      // No vector registers remain so pass on the stack.
      const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
      State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
      return false;
    }
 
    // If all GPRS are consumed then we pass the argument fully on the stack.
    if (NextRegIndex == GPRs.size()) {
      const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
      State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
      return false;
    }
 
    // Corner case for 32-bit codegen. We have 2 registers to pass the first
    // half of the argument, and then need to pass the remaining half on the
    // stack.
    if (GPRs[NextRegIndex] == PPC::R9) {
      const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
      State.addLoc(
          CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo));
 
      const MCRegister FirstReg = State.AllocateReg(PPC::R9);
      const MCRegister SecondReg = State.AllocateReg(PPC::R10);
      assert(FirstReg && SecondReg &&
             "Allocating R9 or R10 unexpectedly failed.");
      State.addLoc(
          CCValAssign::getCustomReg(ValNo, ValVT, FirstReg, RegVT, LocInfo));
      State.addLoc(
          CCValAssign::getCustomReg(ValNo, ValVT, SecondReg, RegVT, LocInfo));
      return false;
    }
 
    // We have enough GPRs to fully pass the vector argument, and we have
    // already consumed any underaligned registers. Start with the custom
    // MemLoc and then the custom RegLocs.
    const unsigned Offset = State.AllocateStack(VecSize, VecAlign);
    State.addLoc(
        CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo));
    for (unsigned I = 0; I != VecSize; I += PtrSize) {
      const MCRegister Reg = State.AllocateReg(GPRs);
      assert(Reg && "Failed to allocated register for vararg vector argument");
      State.addLoc(
          CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo));
    }
    return false;
  }
  }
  return true;
}
 
// So far, this function is only used by LowerFormalArguments_AIX()
static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT,
                                                    bool IsPPC64,
                                                    bool HasP8Vector,
                                                    bool HasVSX) {
  assert((IsPPC64 || SVT != MVT::i64) &&
         "i64 should have been split for 32-bit codegen.");
 
  switch (SVT) {
  default:
    report_fatal_error("Unexpected value type for formal argument");
  case MVT::i1:
  case MVT::i32:
  case MVT::i64:
    return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
  case MVT::f32:
    return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass;
  case MVT::f64:
    return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass;
  case MVT::v4f32:
  case MVT::v4i32:
  case MVT::v8i16:
  case MVT::v16i8:
  case MVT::v2i64:
  case MVT::v2f64:
  case MVT::v1i128:
    return &PPC::VRRCRegClass;
  }
}
 
static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT,
                                        SelectionDAG &DAG, SDValue ArgValue,
                                        MVT LocVT, const SDLoc &dl) {
  assert(ValVT.isScalarInteger() && LocVT.isScalarInteger());
  assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits());
 
  if (Flags.isSExt())
    ArgValue = DAG.getNode(ISD::AssertSext, dl, LocVT, ArgValue,
                           DAG.getValueType(ValVT));
  else if (Flags.isZExt())
    ArgValue = DAG.getNode(ISD::AssertZext, dl, LocVT, ArgValue,
                           DAG.getValueType(ValVT));
 
  return DAG.getNode(ISD::TRUNCATE, dl, ValVT, ArgValue);
}
 
static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
  const unsigned LASize = FL->getLinkageSize();
 
  if (PPC::GPRCRegClass.contains(Reg)) {
    assert(Reg >= PPC::R3 && Reg <= PPC::R10 &&
           "Reg must be a valid argument register!");
    return LASize + 4 * (Reg - PPC::R3);
  }
 
  if (PPC::G8RCRegClass.contains(Reg)) {
    assert(Reg >= PPC::X3 && Reg <= PPC::X10 &&
           "Reg must be a valid argument register!");
    return LASize + 8 * (Reg - PPC::X3);
  }
 
  llvm_unreachable("Only general purpose registers expected.");
}
 
//   AIX ABI Stack Frame Layout:
//
//   Low Memory +--------------------------------------------+
//   SP   +---> | Back chain                                 | ---+
//        |     +--------------------------------------------+    |
//        |     | Saved Condition Register                   |    |
//        |     +--------------------------------------------+    |
//        |     | Saved Linkage Register                     |    |
//        |     +--------------------------------------------+    | Linkage Area
//        |     | Reserved for compilers                     |    |
//        |     +--------------------------------------------+    |
//        |     | Reserved for binders                       |    |
//        |     +--------------------------------------------+    |
//        |     | Saved TOC pointer                          | ---+
//        |     +--------------------------------------------+
//        |     | Parameter save area                        |
//        |     +--------------------------------------------+
//        |     | Alloca space                               |
//        |     +--------------------------------------------+
//        |     | Local variable space                       |
//        |     +--------------------------------------------+
//        |     | Float/int conversion temporary             |
//        |     +--------------------------------------------+
//        |     | Save area for AltiVec registers            |
//        |     +--------------------------------------------+
//        |     | AltiVec alignment padding                  |
//        |     +--------------------------------------------+
//        |     | Save area for VRSAVE register              |
//        |     +--------------------------------------------+
//        |     | Save area for General Purpose registers    |
//        |     +--------------------------------------------+
//        |     | Save area for Floating Point registers     |
//        |     +--------------------------------------------+
//        +---- | Back chain                                 |
// High Memory  +--------------------------------------------+
//
//  Specifications:
//  AIX 7.2 Assembler Language Reference
//  Subroutine linkage convention
 
SDValue PPCTargetLowering::LowerFormalArguments_AIX(
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
 
  assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold ||
          CallConv == CallingConv::Fast) &&
         "Unexpected calling convention!");
 
  if (getTargetMachine().Options.GuaranteedTailCallOpt)
    report_fatal_error("Tail call support is unimplemented on AIX.");
 
  if (useSoftFloat())
    report_fatal_error("Soft float support is unimplemented on AIX.");
 
  const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
 
  const bool IsPPC64 = Subtarget.isPPC64();
  const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
 
  // Assign locations to all of the incoming arguments.
  SmallVector<CCValAssign, 16> ArgLocs;
  MachineFunction &MF = DAG.getMachineFunction();
  MachineFrameInfo &MFI = MF.getFrameInfo();
  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
  CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
 
  const EVT PtrVT = getPointerTy(MF.getDataLayout());
  // Reserve space for the linkage area on the stack.
  const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
  CCInfo.AllocateStack(LinkageSize, Align(PtrByteSize));
  uint64_t SaveStackPos = CCInfo.getStackSize();
  bool SaveParams = MF.getFunction().hasFnAttribute("save-reg-params");
  CCInfo.AnalyzeFormalArguments(Ins, CC_AIX);
 
  SmallVector<SDValue, 8> MemOps;
 
  for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) {
    CCValAssign &VA = ArgLocs[I++];
    MVT LocVT = VA.getLocVT();
    MVT ValVT = VA.getValVT();
    ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags;
 
    EVT ArgVT = Ins[VA.getValNo()].ArgVT;
    bool ArgSignExt = Ins[VA.getValNo()].Flags.isSExt();
    // For compatibility with the AIX XL compiler, the float args in the
    // parameter save area are initialized even if the argument is available
    // in register.  The caller is required to initialize both the register
    // and memory, however, the callee can choose to expect it in either.
    // The memloc is dismissed here because the argument is retrieved from
    // the register.
    if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint())
      continue;
 
    if (SaveParams && VA.isRegLoc() && !Flags.isByVal() && !VA.needsCustom()) {
      const TargetRegisterClass *RegClass = getRegClassForSVT(
          LocVT.SimpleTy, IsPPC64, Subtarget.hasP8Vector(), Subtarget.hasVSX());
      // On PPC64, debugger assumes extended 8-byte values are stored from GPR.
      MVT SaveVT = RegClass == &PPC::G8RCRegClass ? MVT::i64 : LocVT;
      const Register VReg = MF.addLiveIn(VA.getLocReg(), RegClass);
      SDValue Parm = DAG.getCopyFromReg(Chain, dl, VReg, SaveVT);
      int FI = MFI.CreateFixedObject(SaveVT.getStoreSize(), SaveStackPos, true);
      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
      SDValue StoreReg = DAG.getStore(Chain, dl, Parm, FIN,
                                      MachinePointerInfo(), Align(PtrByteSize));
      SaveStackPos = alignTo(SaveStackPos + SaveVT.getStoreSize(), PtrByteSize);
      MemOps.push_back(StoreReg);
    }
 
    if (SaveParams && (VA.isMemLoc() || Flags.isByVal()) && !VA.needsCustom()) {
      unsigned StoreSize =
          Flags.isByVal() ? Flags.getByValSize() : LocVT.getStoreSize();
      SaveStackPos = alignTo(SaveStackPos + StoreSize, PtrByteSize);
    }
 
    auto HandleMemLoc = [&]() {
      const unsigned LocSize = LocVT.getStoreSize();
      const unsigned ValSize = ValVT.getStoreSize();
      assert((ValSize <= LocSize) &&
             "Object size is larger than size of MemLoc");
      int CurArgOffset = VA.getLocMemOffset();
      // Objects are right-justified because AIX is big-endian.
      if (LocSize > ValSize)
        CurArgOffset += LocSize - ValSize;
      // Potential tail calls could cause overwriting of argument stack slots.
      const bool IsImmutable =
          !(getTargetMachine().Options.GuaranteedTailCallOpt &&
            (CallConv == CallingConv::Fast));
      int FI = MFI.CreateFixedObject(ValSize, CurArgOffset, IsImmutable);
      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
      SDValue ArgValue =
          DAG.getLoad(ValVT, dl, Chain, FIN, MachinePointerInfo());
 
      // While the ABI specifies the argument type is (sign or zero) extended
      // out to register width, not all code is compliant. We truncate and
      // re-extend to be more forgiving of these callers when the argument type
      // is smaller than register width.
      if (!ArgVT.isVector() && !ValVT.isVector() && ArgVT.isInteger() &&
          ValVT.isInteger() &&
          ArgVT.getScalarSizeInBits() < ValVT.getScalarSizeInBits()) {
        // It is possible to have either real integer values
        // or integers that were not originally integers.
        // In the latter case, these could have came from structs,
        // and these integers would not have an extend on the parameter.
        // Since these types of integers do not have an extend specified
        // in the first place, the type of extend that we do should not matter.
        EVT TruncatedArgVT = ArgVT.isSimple() && ArgVT.getSimpleVT() == MVT::i1
                                 ? MVT::i8
                                 : ArgVT;
        SDValue ArgValueTrunc =
            DAG.getNode(ISD::TRUNCATE, dl, TruncatedArgVT, ArgValue);
        SDValue ArgValueExt =
            ArgSignExt ? DAG.getSExtOrTrunc(ArgValueTrunc, dl, ValVT)
                       : DAG.getZExtOrTrunc(ArgValueTrunc, dl, ValVT);
        InVals.push_back(ArgValueExt);
      } else {
        InVals.push_back(ArgValue);
      }
    };
 
    // Vector arguments to VaArg functions are passed both on the stack, and
    // in any available GPRs. Load the value from the stack and add the GPRs
    // as live ins.
    if (VA.isMemLoc() && VA.needsCustom()) {
      assert(ValVT.isVector() && "Unexpected Custom MemLoc type.");
      assert(isVarArg && "Only use custom memloc for vararg.");
      // ValNo of the custom MemLoc, so we can compare it to the ValNo of the
      // matching custom RegLocs.
      const unsigned OriginalValNo = VA.getValNo();
      (void)OriginalValNo;
 
      auto HandleCustomVecRegLoc = [&]() {
        assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
               "Missing custom RegLoc.");
        VA = ArgLocs[I++];
        assert(VA.getValVT().isVector() &&
               "Unexpected Val type for custom RegLoc.");
        assert(VA.getValNo() == OriginalValNo &&
               "ValNo mismatch between custom MemLoc and RegLoc.");
        MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy;
        MF.addLiveIn(VA.getLocReg(),
                     getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(),
                                       Subtarget.hasVSX()));
      };
 
      HandleMemLoc();
      // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
      // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
      // R10.
      HandleCustomVecRegLoc();
      HandleCustomVecRegLoc();
 
      // If we are targeting 32-bit, there might be 2 extra custom RegLocs if
      // we passed the vector in R5, R6, R7 and R8.
      if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) {
        assert(!IsPPC64 &&
               "Only 2 custom RegLocs expected for 64-bit codegen.");
        HandleCustomVecRegLoc();
        HandleCustomVecRegLoc();
      }
 
      continue;
    }
 
    if (VA.isRegLoc()) {
      if (VA.getValVT().isScalarInteger())
        FuncInfo->appendParameterType(PPCFunctionInfo::FixedType);
      else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) {
        switch (VA.getValVT().SimpleTy) {
        default:
          report_fatal_error("Unhandled value type for argument.");
        case MVT::f32:
          FuncInfo->appendParameterType(PPCFunctionInfo::ShortFloatingPoint);
          break;
        case MVT::f64:
          FuncInfo->appendParameterType(PPCFunctionInfo::LongFloatingPoint);
          break;
        }
      } else if (VA.getValVT().isVector()) {
        switch (VA.getValVT().SimpleTy) {
        default:
          report_fatal_error("Unhandled value type for argument.");
        case MVT::v16i8:
          FuncInfo->appendParameterType(PPCFunctionInfo::VectorChar);
          break;
        case MVT::v8i16:
          FuncInfo->appendParameterType(PPCFunctionInfo::VectorShort);
          break;
        case MVT::v4i32:
        case MVT::v2i64:
        case MVT::v1i128:
          FuncInfo->appendParameterType(PPCFunctionInfo::VectorInt);
          break;
        case MVT::v4f32:
        case MVT::v2f64:
          FuncInfo->appendParameterType(PPCFunctionInfo::VectorFloat);
          break;
        }
      }
    }
 
    if (Flags.isByVal() && VA.isMemLoc()) {
      const unsigned Size =
          alignTo(Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize,
                  PtrByteSize);
      const int FI = MF.getFrameInfo().CreateFixedObject(
          Size, VA.getLocMemOffset(), /* IsImmutable */ false,
          /* IsAliased */ true);
      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
      InVals.push_back(FIN);
 
      continue;
    }
 
    if (Flags.isByVal()) {
      assert(VA.isRegLoc() && "MemLocs should already be handled.");
 
      const MCPhysReg ArgReg = VA.getLocReg();
      const PPCFrameLowering *FL = Subtarget.getFrameLowering();
 
      const unsigned StackSize = alignTo(Flags.getByValSize(), PtrByteSize);
      const int FI = MF.getFrameInfo().CreateFixedObject(
          StackSize, mapArgRegToOffsetAIX(ArgReg, FL), /* IsImmutable */ false,
          /* IsAliased */ true);
      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
      InVals.push_back(FIN);
 
      // Add live ins for all the RegLocs for the same ByVal.
      const TargetRegisterClass *RegClass =
          IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
 
      auto HandleRegLoc = [&, RegClass, LocVT](const MCPhysReg PhysReg,
                                               unsigned Offset) {
        const Register VReg = MF.addLiveIn(PhysReg, RegClass);
        // Since the callers side has left justified the aggregate in the
        // register, we can simply store the entire register into the stack
        // slot.
        SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, VReg, LocVT);
        // The store to the fixedstack object is needed becuase accessing a
        // field of the ByVal will use a gep and load. Ideally we will optimize
        // to extracting the value from the register directly, and elide the
        // stores when the arguments address is not taken, but that will need to
        // be future work.
        SDValue Store = DAG.getStore(
            CopyFrom.getValue(1), dl, CopyFrom,
            DAG.getObjectPtrOffset(dl, FIN, TypeSize::getFixed(Offset)),
            MachinePointerInfo::getFixedStack(MF, FI, Offset));
 
        MemOps.push_back(Store);
      };
 
      unsigned Offset = 0;
      HandleRegLoc(VA.getLocReg(), Offset);
      Offset += PtrByteSize;
      for (; Offset != StackSize && ArgLocs[I].isRegLoc();
           Offset += PtrByteSize) {
        assert(ArgLocs[I].getValNo() == VA.getValNo() &&
               "RegLocs should be for ByVal argument.");
 
        const CCValAssign RL = ArgLocs[I++];
        HandleRegLoc(RL.getLocReg(), Offset);
        FuncInfo->appendParameterType(PPCFunctionInfo::FixedType);
      }
 
      if (Offset != StackSize) {
        assert(ArgLocs[I].getValNo() == VA.getValNo() &&
               "Expected MemLoc for remaining bytes.");
        assert(ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes.");
        // Consume the MemLoc.The InVal has already been emitted, so nothing
        // more needs to be done.
        ++I;
      }
 
      continue;
    }
 
    if (VA.isRegLoc() && !VA.needsCustom()) {
      MVT::SimpleValueType SVT = ValVT.SimpleTy;
      Register VReg =
          MF.addLiveIn(VA.getLocReg(),
                       getRegClassForSVT(SVT, IsPPC64, Subtarget.hasP8Vector(),
                                         Subtarget.hasVSX()));
      SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, LocVT);
      if (ValVT.isScalarInteger() &&
          (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) {
        ArgValue =
            truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl);
      }
      InVals.push_back(ArgValue);
      continue;
    }
    if (VA.isMemLoc()) {
      HandleMemLoc();
      continue;
    }
  }
 
  // On AIX a minimum of 8 words is saved to the parameter save area.
  const unsigned MinParameterSaveArea = 8 * PtrByteSize;
  // Area that is at least reserved in the caller of this function.
  unsigned CallerReservedArea = std::max<unsigned>(
      CCInfo.getStackSize(), LinkageSize + MinParameterSaveArea);
 
  // Set the size that is at least reserved in caller of this function. Tail
  // call optimized function's reserved stack space needs to be aligned so
  // that taking the difference between two stack areas will result in an
  // aligned stack.
  CallerReservedArea =
      EnsureStackAlignment(Subtarget.getFrameLowering(), CallerReservedArea);
  FuncInfo->setMinReservedArea(CallerReservedArea);
 
  if (isVarArg) {
    int VAListIndex = 0;
    // If any of the optional arguments are passed in register then the fixed
    // stack object we spill into is not immutable. Create a fixed stack object
    // that overlaps the remainder of the parameter save area.
    if (CCInfo.getStackSize() < (LinkageSize + MinParameterSaveArea)) {
      unsigned FixedStackSize =
          LinkageSize + MinParameterSaveArea - CCInfo.getStackSize();
      VAListIndex =
          MFI.CreateFixedObject(FixedStackSize, CCInfo.getStackSize(),
                                /* IsImmutable */ false, /* IsAliased */ true);
    } else {
      // All the arguments passed through ellipses are on the stack. Create a
      // dummy fixed stack object the same size as a pointer since we don't
      // know the actual size.
      VAListIndex =
          MFI.CreateFixedObject(PtrByteSize, CCInfo.getStackSize(),
                                /* IsImmutable */ true, /* IsAliased */ true);
    }
 
    FuncInfo->setVarArgsFrameIndex(VAListIndex);
    SDValue FIN = DAG.getFrameIndex(VAListIndex, PtrVT);
 
    static const MCPhysReg GPR_32[] = {PPC::R3, PPC::R4, PPC::R5, PPC::R6,
                                       PPC::R7, PPC::R8, PPC::R9, PPC::R10};
 
    static const MCPhysReg GPR_64[] = {PPC::X3, PPC::X4, PPC::X5, PPC::X6,
                                       PPC::X7, PPC::X8, PPC::X9, PPC::X10};
    const unsigned NumGPArgRegs = std::size(IsPPC64 ? GPR_64 : GPR_32);
 
    // The fixed integer arguments of a variadic function are stored to the
    // VarArgsFrameIndex on the stack so that they may be loaded by
    // dereferencing the result of va_next.
    for (unsigned
             GPRIndex = (CCInfo.getStackSize() - LinkageSize) / PtrByteSize,
             Offset = 0;
         GPRIndex < NumGPArgRegs; ++GPRIndex, Offset += PtrByteSize) {
 
      const Register VReg =
          IsPPC64 ? MF.addLiveIn(GPR_64[GPRIndex], &PPC::G8RCRegClass)
                  : MF.addLiveIn(GPR_32[GPRIndex], &PPC::GPRCRegClass);
 
      SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
      MachinePointerInfo MPI =
          MachinePointerInfo::getFixedStack(MF, VAListIndex, Offset);
      SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MPI);
      MemOps.push_back(Store);
      // Increment the address for the next argument to store.
      SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT);
      FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
    }
  }
 
  if (!MemOps.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
 
  return Chain;
}
 
SDValue PPCTargetLowering::LowerCall_AIX(
    SDValue Chain, SDValue Callee, CallFlags CFlags,
    const SmallVectorImpl<ISD::OutputArg> &Outs,
    const SmallVectorImpl<SDValue> &OutVals,
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
    const CallBase *CB) const {
  // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the
  // AIX ABI stack frame layout.
 
  assert((CFlags.CallConv == CallingConv::C ||
          CFlags.CallConv == CallingConv::Cold ||
          CFlags.CallConv == CallingConv::Fast) &&
         "Unexpected calling convention!");
 
  if (CFlags.IsPatchPoint)
    report_fatal_error("This call type is unimplemented on AIX.");
 
  const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
 
  MachineFunction &MF = DAG.getMachineFunction();
  SmallVector<CCValAssign, 16> ArgLocs;
  CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
                 *DAG.getContext());
 
  // Reserve space for the linkage save area (LSA) on the stack.
  // In both PPC32 and PPC64 there are 6 reserved slots in the LSA:
  //   [SP][CR][LR][2 x reserved][TOC].
  // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64.
  const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
  const bool IsPPC64 = Subtarget.isPPC64();
  const EVT PtrVT = getPointerTy(DAG.getDataLayout());
  const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
  CCInfo.AllocateStack(LinkageSize, Align(PtrByteSize));
  CCInfo.AnalyzeCallOperands(Outs, CC_AIX);
 
  // The prolog code of the callee may store up to 8 GPR argument registers to
  // the stack, allowing va_start to index over them in memory if the callee
  // is variadic.
  // Because we cannot tell if this is needed on the caller side, we have to
  // conservatively assume that it is needed.  As such, make sure we have at
  // least enough stack space for the caller to store the 8 GPRs.
  const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize;
  const unsigned NumBytes = std::max<unsigned>(
      LinkageSize + MinParameterSaveAreaSize, CCInfo.getStackSize());
 
  // Adjust the stack pointer for the new arguments...
  // These operations are automatically eliminated by the prolog/epilog pass.
  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
  SDValue CallSeqStart = Chain;
 
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
  SmallVector<SDValue, 8> MemOpChains;
 
  // Set up a copy of the stack pointer for loading and storing any
  // arguments that may not fit in the registers available for argument
  // passing.
  const SDValue StackPtr = IsPPC64 ? DAG.getRegister(PPC::X1, MVT::i64)
                                   : DAG.getRegister(PPC::R1, MVT::i32);
 
  for (unsigned I = 0, E = ArgLocs.size(); I != E;) {
    const unsigned ValNo = ArgLocs[I].getValNo();
    SDValue Arg = OutVals[ValNo];
    ISD::ArgFlagsTy Flags = Outs[ValNo].Flags;
 
    if (Flags.isByVal()) {
      const unsigned ByValSize = Flags.getByValSize();
 
      // Nothing to do for zero-sized ByVals on the caller side.
      if (!ByValSize) {
        ++I;
        continue;
      }
 
      auto GetLoad = [&](EVT VT, unsigned LoadOffset) {
        return DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain,
                              (LoadOffset != 0)
                                  ? DAG.getObjectPtrOffset(
                                        dl, Arg, TypeSize::getFixed(LoadOffset))
                                  : Arg,
                              MachinePointerInfo(), VT);
      };
 
      unsigned LoadOffset = 0;
 
      // Initialize registers, which are fully occupied by the by-val argument.
      while (LoadOffset + PtrByteSize <= ByValSize && ArgLocs[I].isRegLoc()) {
        SDValue Load = GetLoad(PtrVT, LoadOffset);
        MemOpChains.push_back(Load.getValue(1));
        LoadOffset += PtrByteSize;
        const CCValAssign &ByValVA = ArgLocs[I++];
        assert(ByValVA.getValNo() == ValNo &&
               "Unexpected location for pass-by-value argument.");
        RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), Load));
      }
 
      if (LoadOffset == ByValSize)
        continue;
 
      // There must be one more loc to handle the remainder.
      assert(ArgLocs[I].getValNo() == ValNo &&
             "Expected additional location for by-value argument.");
 
      if (ArgLocs[I].isMemLoc()) {
        assert(LoadOffset < ByValSize && "Unexpected memloc for by-val arg.");
        const CCValAssign &ByValVA = ArgLocs[I++];
        ISD::ArgFlagsTy MemcpyFlags = Flags;
        // Only memcpy the bytes that don't pass in register.
        MemcpyFlags.setByValSize(ByValSize - LoadOffset);
        Chain = CallSeqStart = createMemcpyOutsideCallSeq(
            (LoadOffset != 0) ? DAG.getObjectPtrOffset(
                                    dl, Arg, TypeSize::getFixed(LoadOffset))
                              : Arg,
            DAG.getObjectPtrOffset(
                dl, StackPtr, TypeSize::getFixed(ByValVA.getLocMemOffset())),
            CallSeqStart, MemcpyFlags, DAG, dl);
        continue;
      }
 
      // Initialize the final register residue.
      // Any residue that occupies the final by-val arg register must be
      // left-justified on AIX. Loads must be a power-of-2 size and cannot be
      // larger than the ByValSize. For example: a 7 byte by-val arg requires 4,
      // 2 and 1 byte loads.
      const unsigned ResidueBytes = ByValSize % PtrByteSize;
      assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize &&
             "Unexpected register residue for by-value argument.");
      SDValue ResidueVal;
      for (unsigned Bytes = 0; Bytes != ResidueBytes;) {
        const unsigned N = llvm::bit_floor(ResidueBytes - Bytes);
        const MVT VT =
            N == 1 ? MVT::i8
                   : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64));
        SDValue Load = GetLoad(VT, LoadOffset);
        MemOpChains.push_back(Load.getValue(1));
        LoadOffset += N;
        Bytes += N;
 
        // By-val arguments are passed left-justfied in register.
        // Every load here needs to be shifted, otherwise a full register load
        // should have been used.
        assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) &&
               "Unexpected load emitted during handling of pass-by-value "
               "argument.");
        unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8);
        EVT ShiftAmountTy =
            getShiftAmountTy(Load->getValueType(0), DAG.getDataLayout());
        SDValue SHLAmt = DAG.getConstant(NumSHLBits, dl, ShiftAmountTy);
        SDValue ShiftedLoad =
            DAG.getNode(ISD::SHL, dl, Load.getValueType(), Load, SHLAmt);
        ResidueVal = ResidueVal ? DAG.getNode(ISD::OR, dl, PtrVT, ResidueVal,
                                              ShiftedLoad)
                                : ShiftedLoad;
      }
 
      const CCValAssign &ByValVA = ArgLocs[I++];
      RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), ResidueVal));
      continue;
    }
 
    CCValAssign &VA = ArgLocs[I++];
    const MVT LocVT = VA.getLocVT();
    const MVT ValVT = VA.getValVT();
 
    switch (VA.getLocInfo()) {
    default:
      report_fatal_error("Unexpected argument extension type.");
    case CCValAssign::Full:
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    }
 
    if (VA.isRegLoc() && !VA.needsCustom()) {
      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
      continue;
    }
 
    // Vector arguments passed to VarArg functions need custom handling when
    // they are passed (at least partially) in GPRs.
    if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) {
      assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args.");
      // Store value to its stack slot.
      SDValue PtrOff =
          DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType());
      PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
      SDValue Store =
          DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo());
      MemOpChains.push_back(Store);
      const unsigned OriginalValNo = VA.getValNo();
      // Then load the GPRs from the stack
      unsigned LoadOffset = 0;
      auto HandleCustomVecRegLoc = [&]() {
        assert(I != E && "Unexpected end of CCvalAssigns.");
        assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
               "Expected custom RegLoc.");
        CCValAssign RegVA = ArgLocs[I++];
        assert(RegVA.getValNo() == OriginalValNo &&
               "Custom MemLoc ValNo and custom RegLoc ValNo must match.");
        SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
                                  DAG.getConstant(LoadOffset, dl, PtrVT));
        SDValue Load = DAG.getLoad(PtrVT, dl, Store, Add, MachinePointerInfo());
        MemOpChains.push_back(Load.getValue(1));
        RegsToPass.push_back(std::make_pair(RegVA.getLocReg(), Load));
        LoadOffset += PtrByteSize;
      };
 
      // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
      // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
      // R10.
      HandleCustomVecRegLoc();
      HandleCustomVecRegLoc();
 
      if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
          ArgLocs[I].getValNo() == OriginalValNo) {
        assert(!IsPPC64 &&
               "Only 2 custom RegLocs expected for 64-bit codegen.");
        HandleCustomVecRegLoc();
        HandleCustomVecRegLoc();
      }
 
      continue;
    }
 
    if (VA.isMemLoc()) {
      SDValue PtrOff =
          DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType());
      PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
      MemOpChains.push_back(
          DAG.getStore(Chain, dl, Arg, PtrOff,
                       MachinePointerInfo::getStack(MF, VA.getLocMemOffset()),
                       Subtarget.getFrameLowering()->getStackAlign()));
 
      continue;
    }
 
    if (!ValVT.isFloatingPoint())
      report_fatal_error(
          "Unexpected register handling for calling convention.");
 
    // Custom handling is used for GPR initializations for vararg float
    // arguments.
    assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg &&
           LocVT.isInteger() &&
           "Custom register handling only expected for VarArg.");
 
    SDValue ArgAsInt =
        DAG.getBitcast(MVT::getIntegerVT(ValVT.getSizeInBits()), Arg);
 
    if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize())
      // f32 in 32-bit GPR
      // f64 in 64-bit GPR
      RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgAsInt));
    else if (Arg.getValueType().getFixedSizeInBits() <
             LocVT.getFixedSizeInBits())
      // f32 in 64-bit GPR.
      RegsToPass.push_back(std::make_pair(
          VA.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, LocVT)));
    else {
      // f64 in two 32-bit GPRs
      // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs.
      assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 &&
             "Unexpected custom register for argument!");
      CCValAssign &GPR1 = VA;
      SDValue MSWAsI64 = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgAsInt,
                                     DAG.getConstant(32, dl, MVT::i8));
      RegsToPass.push_back(std::make_pair(
          GPR1.getLocReg(), DAG.getZExtOrTrunc(MSWAsI64, dl, MVT::i32)));
 
      if (I != E) {
        // If only 1 GPR was available, there will only be one custom GPR and
        // the argument will also pass in memory.
        CCValAssign &PeekArg = ArgLocs[I];
        if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) {
          assert(PeekArg.needsCustom() && "A second custom GPR is expected.");
          CCValAssign &GPR2 = ArgLocs[I++];
          RegsToPass.push_back(std::make_pair(
              GPR2.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, MVT::i32)));
        }
      }
    }
  }
 
  if (!MemOpChains.empty())
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
 
  // For indirect calls, we need to save the TOC base to the stack for
  // restoration after the call.
  if (CFlags.IsIndirect && !Subtarget.usePointerGlueHelper()) {
    assert(!CFlags.IsTailCall && "Indirect tail-calls not supported.");
    const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister();
    const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
    const MVT PtrVT = Subtarget.getScalarIntVT();
    const unsigned TOCSaveOffset =
        Subtarget.getFrameLowering()->getTOCSaveOffset();
 
    setUsesTOCBasePtr(DAG);
    SDValue Val = DAG.getCopyFromReg(Chain, dl, TOCBaseReg, PtrVT);
    SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl);
    SDValue StackPtr = DAG.getRegister(StackPtrReg, PtrVT);
    SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
    Chain = DAG.getStore(
        Val.getValue(1), dl, Val, AddPtr,
        MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset));
  }
 
  // Build a sequence of copy-to-reg nodes chained together with token chain
  // and flag operands which copy the outgoing args into the appropriate regs.
  SDValue InGlue;
  for (auto Reg : RegsToPass) {
    Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, InGlue);
    InGlue = Chain.getValue(1);
  }
 
  const int SPDiff = 0;
  return FinishCall(CFlags, dl, DAG, RegsToPass, InGlue, Chain, CallSeqStart,
                    Callee, SPDiff, NumBytes, Ins, InVals, CB);
}
 
bool
PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
                                  MachineFunction &MF, bool isVarArg,
                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
                                  LLVMContext &Context,
                                  const Type *RetTy) const {
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
  return CCInfo.CheckReturn(
      Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
                ? RetCC_PPC_Cold
                : RetCC_PPC);
}
 
SDValue
PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                               bool isVarArg,
                               const SmallVectorImpl<ISD::OutputArg> &Outs,
                               const SmallVectorImpl<SDValue> &OutVals,
                               const SDLoc &dl, SelectionDAG &DAG) const {
  SmallVector<CCValAssign, 16> RVLocs;
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
                 *DAG.getContext());
  CCInfo.AnalyzeReturn(Outs,
                       (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
                           ? RetCC_PPC_Cold
                           : RetCC_PPC);
 
  SDValue Glue;
  SmallVector<SDValue, 4> RetOps(1, Chain);
 
  // Copy the result values into the output registers.
  for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) {
    CCValAssign &VA = RVLocs[i];
    assert(VA.isRegLoc() && "Can only return in registers!");
 
    SDValue Arg = OutVals[RealResIdx];
 
    switch (VA.getLocInfo()) {
    default: llvm_unreachable("Unknown loc info!");
    case CCValAssign::Full: break;
    case CCValAssign::AExt:
      Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    case CCValAssign::ZExt:
      Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    case CCValAssign::SExt:
      Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
      break;
    }
    if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
      bool isLittleEndian = Subtarget.isLittleEndian();
      // Legalize ret f64 -> ret 2 x i32.
      SDValue SVal =
          DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
                      DAG.getIntPtrConstant(isLittleEndian ? 0 : 1, dl));
      Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Glue);
      RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
      SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg,
                         DAG.getIntPtrConstant(isLittleEndian ? 1 : 0, dl));
      Glue = Chain.getValue(1);
      VA = RVLocs[++i]; // skip ahead to next loc
      Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Glue);
    } else
      Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Glue);
    Glue = Chain.getValue(1);
    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
  }
 
  RetOps[0] = Chain;  // Update chain.
 
  // Add the glue if we have it.
  if (Glue.getNode())
    RetOps.push_back(Glue);
 
  return DAG.getNode(PPCISD::RET_GLUE, dl, MVT::Other, RetOps);
}
 
SDValue
PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDLoc dl(Op);
 
  // Get the correct type for integers.
  EVT IntVT = Op.getValueType();
 
  // Get the inputs.
  SDValue Chain = Op.getOperand(0);
  SDValue FPSIdx = getFramePointerFrameIndex(DAG);
  // Build a DYNAREAOFFSET node.
  SDValue Ops[2] = {Chain, FPSIdx};
  SDVTList VTs = DAG.getVTList(IntVT);
  return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops);
}
 
SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
                                             SelectionDAG &DAG) const {
  // When we pop the dynamic allocation we need to restore the SP link.
  SDLoc dl(Op);
 
  // Get the correct type for pointers.
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
  // Construct the stack pointer operand.
  bool isPPC64 = Subtarget.isPPC64();
  unsigned SP = isPPC64 ? PPC::X1 : PPC::R1;
  SDValue StackPtr = DAG.getRegister(SP, PtrVT);
 
  // Get the operands for the STACKRESTORE.
  SDValue Chain = Op.getOperand(0);
  SDValue SaveSP = Op.getOperand(1);
 
  // Load the old link SP.
  SDValue LoadLinkSP =
      DAG.getLoad(PtrVT, dl, Chain, StackPtr, MachinePointerInfo());
 
  // Restore the stack pointer.
  Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP);
 
  // Store the old link SP.
  return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo());
}
 
SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  bool isPPC64 = Subtarget.isPPC64();
  EVT PtrVT = getPointerTy(MF.getDataLayout());
 
  // Get current frame pointer save index.  The users of this index will be
  // primarily DYNALLOC instructions.
  PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
  int RASI = FI->getReturnAddrSaveIndex();
 
  // If the frame pointer save index hasn't been defined yet.
  if (!RASI) {
    // Find out what the fix offset of the frame pointer save area.
    int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset();
    // Allocate the frame index for frame pointer save area.
    RASI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, LROffset, false);
    // Save the result.
    FI->setReturnAddrSaveIndex(RASI);
  }
  return DAG.getFrameIndex(RASI, PtrVT);
}
 
SDValue
PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  bool isPPC64 = Subtarget.isPPC64();
  EVT PtrVT = getPointerTy(MF.getDataLayout());
 
  // Get current frame pointer save index.  The users of this index will be
  // primarily DYNALLOC instructions.
  PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
  int FPSI = FI->getFramePointerSaveIndex();
 
  // If the frame pointer save index hasn't been defined yet.
  if (!FPSI) {
    // Find out what the fix offset of the frame pointer save area.
    int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset();
    // Allocate the frame index for frame pointer save area.
    FPSI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, FPOffset, true);
    // Save the result.
    FI->setFramePointerSaveIndex(FPSI);
  }
  return DAG.getFrameIndex(FPSI, PtrVT);
}
 
SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
                                                   SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
  // Get the inputs.
  SDValue Chain = Op.getOperand(0);
  SDValue Size  = Op.getOperand(1);
  SDLoc dl(Op);
 
  // Get the correct type for pointers.
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
  // Negate the size.
  SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT,
                                DAG.getConstant(0, dl, PtrVT), Size);
  // Construct a node for the frame pointer save index.
  SDValue FPSIdx = getFramePointerFrameIndex(DAG);
  SDValue Ops[3] = { Chain, NegSize, FPSIdx };
  SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other);
  if (hasInlineStackProbe(MF))
    return DAG.getNode(PPCISD::PROBED_ALLOCA, dl, VTs, Ops);
  return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops);
}
 
SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
                                                     SelectionDAG &DAG) const {
  MachineFunction &MF = DAG.getMachineFunction();
 
  bool isPPC64 = Subtarget.isPPC64();
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
 
  int FI = MF.getFrameInfo().CreateFixedObject(isPPC64 ? 8 : 4, 0, false);
  return DAG.getFrameIndex(FI, PtrVT);
}
 
SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
                                               SelectionDAG &DAG) const {
  SDLoc DL(Op);
  return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL,
                     DAG.getVTList(MVT::i32, MVT::Other),
                     Op.getOperand(0), Op.getOperand(1));
}
 
SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
                                                SelectionDAG &DAG) const {
  SDLoc DL(Op);
  return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
                     Op.getOperand(0), Op.getOperand(1));
}
 
SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
  if (Op.getValueType().isVector())
    return LowerVectorLoad(Op, DAG);
 
  assert(Op.getValueType() == MVT::i1 &&
         "Custom lowering only for i1 loads");
 
  // First, load 8 bits into 32 bits, then truncate to 1 bit.
 
  SDLoc dl(Op);
  LoadSDNode *LD = cast<LoadSDNode>(Op);
 
  SDValue Chain = LD->getChain();
  SDValue BasePtr = LD->getBasePtr();
  MachineMemOperand *MMO = LD->getMemOperand();
 
  SDValue NewLD =
      DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain,
                     BasePtr, MVT::i8, MMO);
  SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD);
 
  SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
  return DAG.getMergeValues(Ops, dl);
}
 
SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
  if (Op.getOperand(1).getValueType().isVector())
    return LowerVectorStore(Op, DAG);
 
  assert(Op.getOperand(1).getValueType() == MVT::i1 &&
         "Custom lowering only for i1 stores");
 
  // First, zero extend to 32 bits, then use a truncating store to 8 bits.
 
  SDLoc dl(Op);
  StoreSDNode *ST = cast<StoreSDNode>(Op);
 
  SDValue Chain = ST->getChain();
  SDValue BasePtr = ST->getBasePtr();
  SDValue Value = ST->getValue();
  MachineMemOperand *MMO = ST->getMemOperand();
 
  Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()),
                      Value);
  return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO);
}
 
// FIXME: Remove this once the ANDI glue bug is fixed:
SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
  assert(Op.getValueType() == MVT::i1 &&
         "Custom lowering only for i1 results");
 
  SDLoc DL(Op);
  return DAG.getNode(PPCISD::ANDI_rec_1_GT_BIT, DL, MVT::i1, Op.getOperand(0));
}
 
SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
                                               SelectionDAG &DAG) const {
 
  // Implements a vector truncate that fits in a vector register as a shuffle.
  // We want to legalize vector truncates down to where the source fits in
  // a vector register (and target is therefore smaller than vector register
  // size).  At that point legalization will try to custom lower the sub-legal
  // result and get here - where we can contain the truncate as a single target
  // operation.
 
  // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows:
  //   <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2>
  //
  // We will implement it for big-endian ordering as this (where x denotes
  // undefined):
  //   < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to
  //   < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u>
  //
  // The same operation in little-endian ordering will be:
  //   <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to
  //   <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1>
 
  EVT TrgVT = Op.getValueType();
  assert(TrgVT.isVector() && "Vector type expected.");
  unsigned TrgNumElts = TrgVT.getVectorNumElements();
  EVT EltVT = TrgVT.getVectorElementType();
  if (!isOperationCustom(Op.getOpcode(), TrgVT) ||
      TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(TrgNumElts) ||
      !llvm::has_single_bit<uint32_t>(EltVT.getSizeInBits()))
    return SDValue();
 
  SDValue N1 = Op.getOperand(0);
  EVT SrcVT = N1.getValueType();
  unsigned SrcSize = SrcVT.getSizeInBits();
  if (SrcSize > 256 || !isPowerOf2_32(SrcVT.getVectorNumElements()) ||
      !llvm::has_single_bit<uint32_t>(
          SrcVT.getVectorElementType().getSizeInBits()))
    return SDValue();
  if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2)
    return SDValue();
 
  unsigned WideNumElts = 128 / EltVT.getSizeInBits();
  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts);
 
  SDLoc DL(Op);
  SDValue Op1, Op2;
  if (SrcSize == 256) {
    EVT VecIdxTy = getVectorIdxTy(DAG.getDataLayout());
    EVT SplitVT =
        N1.getValueType().getHalfNumVectorElementsVT(*DAG.getContext());
    unsigned SplitNumElts = SplitVT.getVectorNumElements();
    Op1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1,
                      DAG.getConstant(0, DL, VecIdxTy));
    Op2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SplitVT, N1,
                      DAG.getConstant(SplitNumElts, DL, VecIdxTy));
  }
  else {
    Op1 = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL);
    Op2 = DAG.getUNDEF(WideVT);
  }
 
  // First list the elements we want to keep.
  unsigned SizeMult = SrcSize / TrgVT.getSizeInBits();
  SmallVector<int, 16> ShuffV;
  if (Subtarget.isLittleEndian())
    for (unsigned i = 0; i < TrgNumElts; ++i)
      ShuffV.push_back(i * SizeMult);
  else
    for (unsigned i = 1; i <= TrgNumElts; ++i)
      ShuffV.push_back(i * SizeMult - 1);
 
  // Populate the remaining elements with undefs.
  for (unsigned i = TrgNumElts; i < WideNumElts; ++i)
    // ShuffV.push_back(i + WideNumElts);
    ShuffV.push_back(WideNumElts + 1);
 
  Op1 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op1);
  Op2 = DAG.getNode(ISD::BITCAST, DL, WideVT, Op2);
  return DAG.getVectorShuffle(WideVT, DL, Op1, Op2, ShuffV);
}
 
/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
/// possible.
SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
  EVT ResVT = Op.getValueType();
  EVT CmpVT = Op.getOperand(0).getValueType();
  SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1);
  SDValue TV  = Op.getOperand(2), FV  = Op.getOperand(3);
  SDLoc dl(Op);
 
  // Without power9-vector, we don't have native instruction for f128 comparison.
  // Following transformation to libcall is needed for setcc:
  // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE
  if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) {
    SDValue Z = DAG.getSetCC(
        dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT),
        LHS, RHS, CC);
    SDValue Zero = DAG.getConstant(0, dl, Z.getValueType());
    return DAG.getSelectCC(dl, Z, Zero, TV, FV, ISD::SETNE);
  }
 
  // Not FP, or using SPE? Not a fsel.
  if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() ||
      Subtarget.hasSPE())
    return Op;
 
  SDNodeFlags Flags = Op.getNode()->getFlags();
 
  // We have xsmaxc[dq]p/xsminc[dq]p which are OK to emit even in the
  // presence of infinities.
  if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) {
    switch (CC) {
    default:
      break;
    case ISD::SETOGT:
    case ISD::SETGT:
      return DAG.getNode(PPCISD::XSMAXC, dl, Op.getValueType(), LHS, RHS);
    case ISD::SETOLT:
    case ISD::SETLT:
      return DAG.getNode(PPCISD::XSMINC, dl, Op.getValueType(), LHS, RHS);
    }
  }
 
  // We might be able to do better than this under some circumstances, but in
  // general, fsel-based lowering of select is a finite-math-only optimization.
  // For more information, see section F.3 of the 2.06 ISA specification.
  // With ISA 3.0
  if (!Flags.hasNoInfs() || !Flags.hasNoNaNs() || ResVT == MVT::f128)
    return Op;
 
  // If the RHS of the comparison is a 0.0, we don't need to do the
  // subtraction at all.
  SDValue Sel1;
  if (isFloatingPointZero(RHS))
    switch (CC) {
    default: break;       // SETUO etc aren't handled by fsel.
    case ISD::SETNE:
      std::swap(TV, FV);
      [[fallthrough]];
    case ISD::SETEQ:
      if (LHS.getValueType() == MVT::f32)   // Comparison is always 64-bits
        LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
      Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV);
      if (Sel1.getValueType() == MVT::f32)   // Comparison is always 64-bits
        Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1);
      return DAG.getNode(PPCISD::FSEL, dl, ResVT,
                         DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV);
    case ISD::SETULT:
    case ISD::SETLT:
      std::swap(TV, FV);  // fsel is natively setge, swap operands for setlt
      [[fallthrough]];
    case ISD::SETOGE:
    case ISD::SETGE:
      if (LHS.getValueType() == MVT::f32)   // Comparison is always 64-bits
        LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
      return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV);
    case ISD::SETUGT:
    case ISD::SETGT:
      std::swap(TV, FV);  // fsel is natively setge, swap operands for setlt
      [[fallthrough]];
    case ISD::SETOLE:
    case ISD::SETLE:
      if (LHS.getValueType() == MVT::f32)   // Comparison is always 64-bits
        LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS);
      return DAG.getNode(PPCISD::FSEL, dl, ResVT,
                         DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV);
    }
 
  SDValue Cmp;
  switch (CC) {
  default: break;       // SETUO etc aren't handled by fsel.
  case ISD::SETNE:
    std::swap(TV, FV);
    [[fallthrough]];
  case ISD::SETEQ:
    Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags);
    if (Cmp.getValueType() == MVT::f32)   // Comparison is always 64-bits
      Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
    Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
    if (Sel1.getValueType() == MVT::f32)   // Comparison is always 64-bits
      Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1);
    return DAG.getNode(PPCISD::FSEL, dl, ResVT,
                       DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV);
  case ISD::SETULT:
  case ISD::SETLT:
    Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags);
    if (Cmp.getValueType() == MVT::f32)   // Comparison is always 64-bits
      Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
    return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV);
  case ISD::SETOGE:
  case ISD::SETGE:
    Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags);
    if (Cmp.getValueType() == MVT::f32)   // Comparison is always 64-bits
      Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
    return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
  case ISD::SETUGT:
  case ISD::SETGT:
    Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags);
    if (Cmp.getValueType() == MVT::f32)   // Comparison is always 64-bits
      Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
    return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV);
  case ISD::SETOLE:
  case ISD::SETLE:
    Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags);
    if (Cmp.getValueType() == MVT::f32)   // Comparison is always 64-bits
      Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp);
    return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV);
  }
  return Op;
}
 
static unsigned getPPCStrictOpcode(unsigned Opc) {
  switch (Opc) {
  default:
    llvm_unreachable("No strict version of this opcode!");
  case PPCISD::FCTIDZ:
    return PPCISD::STRICT_FCTIDZ;
  case PPCISD::FCTIWZ:
    return PPCISD::STRICT_FCTIWZ;
  case PPCISD::FCTIDUZ:
    return PPCISD::STRICT_FCTIDUZ;
  case PPCISD::FCTIWUZ:
    return PPCISD::STRICT_FCTIWUZ;
  case PPCISD::FCFID:
    return PPCISD::STRICT_FCFID;
  case PPCISD::FCFIDU:
    return PPCISD::STRICT_FCFIDU;
  case PPCISD::FCFIDS:
    return PPCISD::STRICT_FCFIDS;
  case PPCISD::FCFIDUS:
    return PPCISD::STRICT_FCFIDUS;
  }
}
 
static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG,
                              const PPCSubtarget &Subtarget) {
  SDLoc dl(Op);
  bool IsStrict = Op->isStrictFPOpcode();
  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
                  Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
 
  // TODO: Any other flags to propagate?
  SDNodeFlags Flags;
  Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
 
  // For strict nodes, source is the second operand.
  SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
  SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
  MVT DestTy = Op.getSimpleValueType();
  assert(Src.getValueType().isFloatingPoint() &&
         (DestTy == MVT::i8 || DestTy == MVT::i16 || DestTy == MVT::i32 ||
          DestTy == MVT::i64) &&
         "Invalid FP_TO_INT types");
  if (Src.getValueType() == MVT::f32) {
    if (IsStrict) {
      Src =
          DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
                      DAG.getVTList(MVT::f64, MVT::Other), {Chain, Src}, Flags);
      Chain = Src.getValue(1);
    } else
      Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src);
  }
  if ((DestTy == MVT::i8 || DestTy == MVT::i16) && Subtarget.hasP9Vector())
    DestTy = Subtarget.getScalarIntVT();
  unsigned Opc = ISD::DELETED_NODE;
  switch (DestTy.SimpleTy) {
  default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
  case MVT::i32:
    Opc = IsSigned ? PPCISD::FCTIWZ
                   : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ);
    break;
  case MVT::i64:
    assert((IsSigned || Subtarget.hasFPCVT()) &&
           "i64 FP_TO_UINT is supported only with FPCVT");
    Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ;
  }
  EVT ConvTy = Src.getValueType() == MVT::f128 ? MVT::f128 : MVT::f64;
  SDValue Conv;
  if (IsStrict) {
    Opc = getPPCStrictOpcode(Opc);
    Conv = DAG.getNode(Opc, dl, DAG.getVTList(ConvTy, MVT::Other), {Chain, Src},
                       Flags);
  } else {
    Conv = DAG.getNode(Opc, dl, ConvTy, Src);
  }
  return Conv;
}
 
void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
                                               SelectionDAG &DAG,
                                               const SDLoc &dl) const {
  SDValue Tmp = convertFPToInt(Op, DAG, Subtarget);
  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
                  Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
  bool IsStrict = Op->isStrictFPOpcode();
 
  // Convert the FP value to an int value through memory.
  bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
                  (IsSigned || Subtarget.hasFPCVT());
  SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64);
  int FI = cast<FrameIndexSDNode>(FIPtr)->getIndex();
  MachinePointerInfo MPI =
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
 
  // Emit a store to the stack slot.
  SDValue Chain = IsStrict ? Tmp.getValue(1) : DAG.getEntryNode();
  Align Alignment(DAG.getEVTAlign(Tmp.getValueType()));
  if (i32Stack) {
    MachineFunction &MF = DAG.getMachineFunction();
    Alignment = Align(4);
    MachineMemOperand *MMO =
        MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Alignment);
    SDValue Ops[] = { Chain, Tmp, FIPtr };
    Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl,
              DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO);
  } else
    Chain = DAG.getStore(Chain, dl, Tmp, FIPtr, MPI, Alignment);
 
  // Result is a load from the stack slot.  If loading 4 bytes, make sure to
  // add in a bias on big endian.
  if (Op.getValueType() == MVT::i32 && !i32Stack &&
      !Subtarget.isLittleEndian()) {
    FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr,
                        DAG.getConstant(4, dl, FIPtr.getValueType()));
    MPI = MPI.getWithOffset(4);
  }
 
  RLI.Chain = Chain;
  RLI.Ptr = FIPtr;
  RLI.MPI = MPI;
  RLI.Alignment = Alignment;
}
 
/// Custom lowers floating point to integer conversions to use
/// the direct move instructions available in ISA 2.07 to avoid the
/// need for load/store combinations.
SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
                                                    SelectionDAG &DAG,
                                                    const SDLoc &dl) const {
  SDValue Conv = convertFPToInt(Op, DAG, Subtarget);
  SDValue Mov = DAG.getNode(PPCISD::MFVSR, dl, Op.getValueType(), Conv);
  if (Op->isStrictFPOpcode())
    return DAG.getMergeValues({Mov, Conv.getValue(1)}, dl);
  else
    return Mov;
}
 
SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
                                          const SDLoc &dl) const {
  bool IsStrict = Op->isStrictFPOpcode();
  bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
                  Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
  SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
  EVT SrcVT = Src.getValueType();
  EVT DstVT = Op.getValueType();